unrealon 2.0.7__tar.gz → 2.0.9__tar.gz

{unrealon-2.0.7/unrealon.egg-info → unrealon-2.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.7
+Version: 2.0.9
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT

{unrealon-2.0.7 → unrealon-2.0.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unrealon"
-version = "2.0.7"
+version = "2.0.9"
 description = "Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities"
 authors = [
     {name = "UnrealOn Team", email = "team@unrealon.com"}

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-browser/src/unrealon_browser/core/browser_manager.py

@@ -30,6 +30,7 @@ from unrealon_browser.managers import (
     CaptchaDetector,
     create_browser_logger_bridge,
     PageWaitManager,
+    ScriptManager,
 )
 
 
@@ -66,6 +67,7 @@ class BrowserManager:
         self.cookie_manager = None
         self.captcha_manager = CaptchaDetector()
         self.page_wait = PageWaitManager(None, self.logger_bridge)
+        self.script_manager = ScriptManager(None, self.logger_bridge)
 
         # Signal handlers for graceful shutdown
         self._setup_signal_handlers()
@@ -231,6 +233,9 @@ class BrowserManager:
 
             # Update page wait manager with new page
             self.page_wait.update_page(self._page)
+            
+            # Update script manager with new page
+            self.script_manager.update_page(self._page)
 
             # 🔥 STEALTH ALWAYS APPLIED TO EVERY PAGE!
             stealth_success = await self.stealth_manager.apply_stealth(self._page)
@@ -644,6 +649,7 @@ class BrowserManager:
                 finally:
                     self._page = None
                     self.page_wait.update_page(None)
+                    self.script_manager.update_page(None)
 
             # Close context with safety checks
             if self._context:

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-browser/src/unrealon_browser/managers/__init__.py

@@ -8,6 +8,7 @@ from .logger_bridge import BrowserLoggerBridge, create_browser_logger_bridge
 from .cookies import CookieManager
 from .captcha import CaptchaDetector
 from .page_wait_manager import PageWaitManager
+from .script_manager import ScriptManager
 
 
 __all__ = [
@@ -18,4 +19,5 @@ __all__ = [
     "CookieManager",
     "CaptchaDetector",
     "PageWaitManager",
+    "ScriptManager",
 ]

unrealon-2.0.9/unrealon-browser/src/unrealon_browser/managers/script_manager.py

@@ -0,0 +1,314 @@
+"""
+Script Manager - JavaScript execution and evaluation manager
+Layer 2.5: JavaScript Integration - Handles script execution, API calls, and result processing
+"""
+
+import asyncio
+import json
+from typing import Any, Dict, Optional, Union, List
+from datetime import datetime, timezone
+from playwright.async_api import Page
+
+from .logger_bridge import BrowserLoggerBridge as LoggingBridge
+
+
+class ScriptManager:
+    """Manager for JavaScript execution and evaluation"""
+    
+    def __init__(self, page: Optional[Page], logger_bridge: LoggingBridge):
+        self._page = page
+        self.logger_bridge = logger_bridge
+        
+        # Statistics
+        self._scripts_executed = 0
+        self._scripts_successful = 0
+        self._scripts_failed = 0
+        self._api_calls_made = 0
+        self._execution_history: List[Dict[str, Any]] = []
+    
+    def update_page(self, page: Optional[Page]):
+        """Update the page reference"""
+        self._page = page
+    
+    async def execute_script(self, script: str, timeout: int = 30000) -> Any:
+        """
+        Execute JavaScript code and return result
+        
+        Args:
+            script: JavaScript code to execute
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            Script execution result
+        """
+        if not self._page:
+            raise RuntimeError("No page available for script execution")
+        
+        start_time = datetime.now()
+        self._scripts_executed += 1
+        
+        try:
+            self.logger_bridge.log_info(f"🔧 Executing JavaScript (timeout: {timeout}ms)")
+            self.logger_bridge.log_debug(f"Script preview: {script[:100]}...")
+            
+            # Execute script with timeout
+            result = await asyncio.wait_for(
+                self._page.evaluate(script),
+                timeout=timeout / 1000
+            )
+            
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_successful += 1
+            
+            # Log execution details
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": True,
+                "result_type": type(result).__name__,
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_info(f"✅ Script executed successfully ({duration_ms:.1f}ms)")
+            self.logger_bridge.log_debug(f"Result type: {type(result).__name__}")
+            
+            return result
+            
+        except asyncio.TimeoutError:
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_failed += 1
+            
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": False,
+                "error": "Timeout",
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_error(f"⏰ Script execution timeout ({timeout}ms)")
+            raise
+            
+        except Exception as e:
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_failed += 1
+            
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": False,
+                "error": str(e),
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_error(f"❌ Script execution failed: {e}")
+            raise
+    
+    async def execute_api_call(self, api_url: str, headers: Dict[str, str], method: str = "GET", timeout: int = 30000) -> Dict[str, Any]:
+        """
+        Execute API call via JavaScript fetch
+        
+        Args:
+            api_url: API endpoint URL
+            headers: HTTP headers
+            method: HTTP method
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            API response data
+        """
+        self._api_calls_made += 1
+        
+        # Build fetch script
+        headers_json = json.dumps(headers)
+        
+        script = f"""
+        (async function() {{
+            try {{
+                const response = await fetch('{api_url}', {{
+                    method: '{method}',
+                    headers: {headers_json}
+                }});
+                
+                if (!response.ok) {{
+                    throw new Error('HTTP error! status: ' + response.status);
+                }}
+                
+                const data = await response.json();
+                return data;
+            }} catch (error) {{
+                return {{ error: error.message }};
+            }}
+        }})()
+        """
+        
+        self.logger_bridge.log_info(f"🌐 Making API call: {method} {api_url}")
+        
+        result = await self.execute_script(script, timeout)
+        
+        if isinstance(result, dict) and 'error' in result:
+            self.logger_bridge.log_error(f"❌ API call failed: {result['error']}")
+        else:
+            self.logger_bridge.log_info(f"✅ API call successful")
+        
+        return result
+    
+    
+    async def wait_for_element(self, selector: str, timeout: int = 10000) -> bool:
+        """
+        Wait for element using JavaScript
+        
+        Args:
+            selector: CSS selector
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            True if element found
+        """
+        script = f"""
+        (function() {{
+            return new Promise((resolve) => {{
+                const element = document.querySelector('{selector}');
+                if (element) {{
+                    resolve(true);
+                    return;
+                }}
+                
+                const observer = new MutationObserver(() => {{
+                    const element = document.querySelector('{selector}');
+                    if (element) {{
+                        observer.disconnect();
+                        resolve(true);
+                    }}
+                }});
+                
+                observer.observe(document.body, {{
+                    childList: true,
+                    subtree: true
+                }});
+                
+                setTimeout(() => {{
+                    observer.disconnect();
+                    resolve(false);
+                }}, {timeout});
+            }});
+        }})()
+        """
+        
+        self.logger_bridge.log_info(f"🎯 Waiting for element: {selector}")
+        
+        try:
+            result = await self.execute_script(script, timeout + 1000)
+            if result:
+                self.logger_bridge.log_info(f"✅ Element found: {selector}")
+            else:
+                self.logger_bridge.log_warning(f"⏰ Element timeout: {selector}")
+            return result
+        except Exception as e:
+            self.logger_bridge.log_error(f"❌ Element wait failed: {selector} - {e}")
+            return False
+    
+    async def inject_helper_functions(self) -> bool:
+        """
+        Inject helper JavaScript functions into page
+        
+        Returns:
+            True if injection successful
+        """
+        helper_script = """
+        window.unrealonHelpers = {
+            // Wait for element with promise
+            waitForElement: function(selector, timeout = 10000) {
+                return new Promise((resolve) => {
+                    const element = document.querySelector(selector);
+                    if (element) {
+                        resolve(element);
+                        return;
+                    }
+                    
+                    const observer = new MutationObserver(() => {
+                        const element = document.querySelector(selector);
+                        if (element) {
+                            observer.disconnect();
+                            resolve(element);
+                        }
+                    });
+                    
+                    observer.observe(document.body, {
+                        childList: true,
+                        subtree: true
+                    });
+                    
+                    setTimeout(() => {
+                        observer.disconnect();
+                        resolve(null);
+                    }, timeout);
+                });
+            },
+            
+            // Get page info
+            getPageInfo: function() {
+                return {
+                    url: window.location.href,
+                    title: document.title,
+                    readyState: document.readyState,
+                    timestamp: new Date().toISOString()
+                };
+            },
+            
+            // Check if SPA loaded
+            isSPAReady: function() {
+                return document.readyState === 'complete' && 
+                       document.querySelector('body').children.length > 0;
+            }
+        };
+        
+        console.log('🔧 UnrealOn helper functions injected');
+        """
+        
+        try:
+            await self.execute_script(helper_script)
+            self.logger_bridge.log_info("🔧 Helper functions injected successfully")
+            return True
+        except Exception as e:
+            self.logger_bridge.log_error(f"❌ Failed to inject helper functions: {e}")
+            return False
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get script execution statistics"""
+        success_rate = (self._scripts_successful / self._scripts_executed * 100) if self._scripts_executed > 0 else 0
+        
+        return {
+            "scripts_executed": self._scripts_executed,
+            "scripts_successful": self._scripts_successful,
+            "scripts_failed": self._scripts_failed,
+            "success_rate": success_rate,
+            "api_calls_made": self._api_calls_made,
+            "execution_history_count": len(self._execution_history),
+        }
+    
+    def print_statistics(self) -> None:
+        """Print script execution statistics"""
+        stats = self.get_statistics()
+        
+        self.logger_bridge.log_info("\n🔧 Script Manager Statistics:")
+        self.logger_bridge.log_info(f"   Scripts executed: {stats['scripts_executed']}")
+        self.logger_bridge.log_info(f"   Successful: {stats['scripts_successful']}")
+        self.logger_bridge.log_info(f"   Failed: {stats['scripts_failed']}")
+        self.logger_bridge.log_info(f"   Success rate: {stats['success_rate']:.1f}%")
+        self.logger_bridge.log_info(f"   API calls made: {stats['api_calls_made']}")
+        
+        # Show recent executions
+        if self._execution_history:
+            self.logger_bridge.log_info("   Recent executions:")
+            for execution in self._execution_history[-3:]:  # Show last 3
+                status = "✅" if execution["success"] else "❌"
+                self.logger_bridge.log_info(f"     {status} {execution['duration_ms']:.1f}ms")
+    
+    def clear_history(self) -> None:
+        """Clear execution history"""
+        self._execution_history.clear()
+        self.logger_bridge.log_info("🧹 Cleared script execution history")

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-core/src/unrealon_core/config/urls.py

@@ -24,6 +24,7 @@ class URLConfig(BaseModel):
     
     # Cloud platform URLs  
     cloud_base_url: str = Field(
+        default="https://cloud.unrealon.com",
         description="Base URL for UnrealOn Cloud platform"
     )
     
@@ -41,7 +42,7 @@ class URLConfig(BaseModel):
         if environment == Environment.PRODUCTION:
             return cls(
                 # scanner_url="https://cloud.unrealon.com/scanner",
-                cloud_base_url="https://cloud.unrealon.com",
+                # cloud_base_url="https://cloud.unrealon.com",
                 api_base_url="https://api.unrealon.com"
             )
         # elif environment == Environment.TESTING:

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-core/src/unrealon_core/enums/__init__.py

@@ -9,6 +9,8 @@ Phase 1: Foundation enums with strict validation
 
 from .status import DriverStatus, TaskStatus, ProxyStatus, LogLevel
 from .types import MessageType, ProxyType, TaskPriority
+from .events import EventType, SystemEventType, RedisEventType
+from .jobs import ARQJobName
 
 __all__ = [
     # Status enums
@@ -21,4 +23,12 @@ __all__ = [
     "MessageType",
     "ProxyType",
     "TaskPriority",
+    
+    # Event enums
+    "EventType",
+    "SystemEventType",
+    "RedisEventType",
+    
+    # Job enums
+    "ARQJobName",
 ]

unrealon-2.0.9/unrealon-core/src/unrealon_core/enums/events.py

@@ -0,0 +1,98 @@
+"""
+Event type enums for UnrealOn system.
+
+Centralized event type definitions used across all services.
+"""
+from enum import Enum
+
+
+class EventType(str, Enum):
+    """
+    Event types used in Redis PubSub and system events.
+    
+    These events are used for communication between services:
+    - RPC server publishes events
+    - Django consumes events via Redis PubSub
+    - System events are stored in database
+    """
+    
+    # Driver lifecycle events
+    DRIVER_REGISTER = "driver_register"
+    DRIVER_DISCONNECT = "driver_disconnect"
+    
+    # Parser events (Django-specific naming)
+    PARSER_REGISTERED = "parser_registered"
+    PARSER_HEARTBEAT = "parser_heartbeat"
+    PARSER_DISCONNECTED = "parser_disconnected"
+    PARSER_ERROR = "parser_error"
+    PARSER_LOG = "parser_log"
+    
+    # Session events
+    SESSION_STARTED = "session_started"
+    SESSION_COMPLETED = "session_completed"
+    SESSION_FAILED = "session_failed"
+    
+    # Command events
+    COMMAND_ISSUED = "command_issued"
+    COMMAND_COMPLETED = "command_completed"
+    COMMAND_FAILED = "command_failed"
+    
+    # System events
+    SYSTEM_STARTUP = "system_startup"
+    SYSTEM_SHUTDOWN = "system_shutdown"
+    SYSTEM_ERROR = "system_error"
+
+
+class SystemEventType(str, Enum):
+    """
+    System event types for Django database storage.
+    
+    These match the choices in Django's SystemEvent model.
+    """
+    
+    # Parser events
+    PARSER_REGISTERED = "parser_registered"
+    PARSER_HEARTBEAT = "parser_heartbeat"
+    PARSER_DISCONNECTED = "parser_disconnected"
+    PARSER_ERROR = "parser_error"
+    
+    # Session events
+    SESSION_STARTED = "session_started"
+    SESSION_COMPLETED = "session_completed"
+    SESSION_FAILED = "session_failed"
+    
+    # Command events
+    COMMAND_ISSUED = "command_issued"
+    COMMAND_COMPLETED = "command_completed"
+    COMMAND_FAILED = "command_failed"
+    
+    # System events
+    SYSTEM_STARTUP = "system_startup"
+    SYSTEM_SHUTDOWN = "system_shutdown"
+    SYSTEM_ERROR = "system_error"
+
+
+class RedisEventType(str, Enum):
+    """
+    Redis PubSub event types.
+    
+    These are the event types published to Redis channels
+    for inter-service communication.
+    """
+    
+    # Driver lifecycle (from RPC to Django)
+    DRIVER_REGISTER = "driver_register"
+    DRIVER_DISCONNECT = "driver_disconnect"
+    
+    # Parser events (internal Django events)
+    PARSER_HEARTBEAT = "parser_heartbeat"
+    PARSER_LOG = "parser_log"
+    
+    # Session events
+    SESSION_STARTED = "session_started"
+    SESSION_COMPLETED = "session_completed"
+    SESSION_FAILED = "session_failed"
+    
+    # Command events
+    COMMAND_COMPLETED = "command_completed"
+    COMMAND_FAILED = "command_failed"

unrealon-2.0.9/unrealon-core/src/unrealon_core/enums/jobs.py

@@ -0,0 +1,32 @@
+"""
+ARQ job name enums for UnrealOn system.
+
+Centralized job name definitions used across all services.
+"""
+from enum import Enum
+
+
+class ARQJobName(str, Enum):
+    """
+    ARQ job names used in the system.
+    
+    These are the job function names that can be enqueued
+    and processed by ARQ workers.
+    """
+    
+    # Driver management jobs
+    REGISTER_DRIVER = "register_driver"
+    DRIVER_DISCONNECT = "driver_disconnect"
+    GET_DRIVER_STATUS = "get_driver_status"
+    LIST_AVAILABLE_DRIVERS = "list_available_drivers"
+    
+    # Task management jobs
+    ASSIGN_TASK_TO_DRIVER = "assign_task_to_driver"
+    
+    # System jobs
+    PING = "ping"
+    PROCESS_DRIVER_HEARTBEAT = "process_driver_heartbeat"
+    
+    # Lifecycle jobs
+    STARTUP = "startup"
+    SHUTDOWN = "shutdown"

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-core/src/unrealon_core/enums/types.py

@@ -58,6 +58,7 @@ class MessageType(str, Enum):
     PONG = "pong"                                 # Pong response
     ERROR = "error"                               # Error message
     ACK = "ack"                                   # Acknowledgment
+    MONITOR_REGISTER = "monitor_register"         # Monitor client registration
     
     def is_driver_lifecycle(self) -> bool:
         """Check if message is related to driver lifecycle."""

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-core/src/unrealon_core/models/__init__.py

@@ -46,6 +46,11 @@ from .logging import (
     LogMetrics
 )
 
+from .authentication import (
+    APIKeyAuthRequest,
+    APIKeyAuthResponse
+)
+
 __all__ = [
     # Base models
     "UnrealOnBaseModel",
@@ -76,4 +81,8 @@ __all__ = [
     "LogEntry",
     "LogQuery",
     "LogMetrics",
+    
+    # Authentication models
+    "APIKeyAuthRequest",
+    "APIKeyAuthResponse",
 ]

unrealon-2.0.9/unrealon-core/src/unrealon_core/models/authentication.py

@@ -0,0 +1,24 @@
+"""
+Authentication models for UnrealOn system.
+
+Provides Pydantic models for API key authentication between services.
+"""
+from typing import Optional, List
+from pydantic import Field
+
+from .base import UnrealOnBaseModel
+
+
+class APIKeyAuthRequest(UnrealOnBaseModel):
+    """API key authentication request."""
+    api_key: str = Field(min_length=1, description="API key for authentication")
+    parser_id: str = Field(min_length=1, description="Parser requesting authentication")
+
+
+class APIKeyAuthResponse(UnrealOnBaseModel):
+    """API key authentication response."""
+    success: bool = Field(description="Whether authentication was successful")
+    user_id: Optional[int] = Field(default=None, description="Authenticated user ID")
+    username: Optional[str] = Field(default=None, description="Authenticated username")
+    permissions: List[str] = Field(default_factory=list, description="User permissions")
+    error: Optional[str] = Field(default=None, description="Error message if failed")

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon-driver/src/unrealon_driver/managers/browser.py

@@ -14,6 +14,7 @@ class BrowserManagerConfig(ManagerConfig):
 
     headless: bool = Field(default=True, description="Run headless")
     parser_name: str = Field(..., description="Parser name for browser")
+    stealth_warmup_enabled: bool = Field(default=False, description="Enable stealth warmup")
 
 
 class BrowserManager(BaseManager):
@@ -51,7 +52,10 @@ class BrowserManager(BaseManager):
             self.logger.info("🚀 Starting browser (lazy initialization)...")
             
             # Create browser config
-            browser_config = BrowserConfig(parser_name=self.config.parser_name)
+            browser_config = BrowserConfig(
+                parser_name=self.config.parser_name,
+                stealth_warmup_enabled=self.config.stealth_warmup_enabled  # Disable stealth warmup to avoid scanner navigation
+            )
 
             # Create browser manager
             self.browser = CoreBrowserManager(browser_config)
@@ -96,3 +100,19 @@ class BrowserManager(BaseManager):
             self.logger.error(f"Get HTML failed: {e}")
             self.stats.record_operation(False, 0.0)
             return None
+    
+    async def execute_script_async(self, script: str) -> any:
+        """Execute JavaScript on current page via ScriptManager."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+        
+        try:
+            # Use ScriptManager from CoreBrowserManager
+            result = await self.browser.script_manager.execute_script(script)
+            self.stats.record_operation(True, 0.0)
+            return result
+        except Exception as e:
+            self.logger.error(f"Script execution failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            raise

{unrealon-2.0.7 → unrealon-2.0.9/unrealon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.7
+Version: 2.0.9
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT

{unrealon-2.0.7 → unrealon-2.0.9}/unrealon.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ unrealon-browser/src/unrealon_browser/managers/cookies.py
 unrealon-browser/src/unrealon_browser/managers/logger_bridge.py
 unrealon-browser/src/unrealon_browser/managers/page_wait_manager.py
 unrealon-browser/src/unrealon_browser/managers/profile.py
+unrealon-browser/src/unrealon_browser/managers/script_manager.py
 unrealon-browser/src/unrealon_browser/stealth/__init__.py
 unrealon-browser/src/unrealon_browser/stealth/bypass_techniques.pyc
 unrealon-browser/src/unrealon_browser/stealth/manager.pyc
@@ -38,6 +39,8 @@ unrealon-core/src/unrealon_core/config/__init__.py
 unrealon-core/src/unrealon_core/config/environment.py
 unrealon-core/src/unrealon_core/config/urls.py
 unrealon-core/src/unrealon_core/enums/__init__.py
+unrealon-core/src/unrealon_core/enums/events.py
+unrealon-core/src/unrealon_core/enums/jobs.py
 unrealon-core/src/unrealon_core/enums/status.py
 unrealon-core/src/unrealon_core/enums/types.py
 unrealon-core/src/unrealon_core/error_handling/__init__.py
@@ -55,6 +58,7 @@ unrealon-core/src/unrealon_core/exceptions/validation.py
 unrealon-core/src/unrealon_core/models/__init__.py
 unrealon-core/src/unrealon_core/models/arq_context.py
 unrealon-core/src/unrealon_core/models/arq_responses.py
+unrealon-core/src/unrealon_core/models/authentication.py
 unrealon-core/src/unrealon_core/models/base.py
 unrealon-core/src/unrealon_core/models/bridge_stats.py
 unrealon-core/src/unrealon_core/models/communication.py