unrealon 2.0.8__py3-none-any.whl → 2.0.10__py3-none-any.whl

{unrealon-2.0.8.dist-info → unrealon-2.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.8
+Version: 2.0.10
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT

{unrealon-2.0.8.dist-info → unrealon-2.0.10.dist-info}/RECORD

@@ -6,7 +6,7 @@ unrealon_browser/cli/cookies_cli.py,sha256=yhZvGrg8bknlH4zlySdi8ue-25Ue-1rI_u1G0
 unrealon_browser/cli/interactive_mode.py,sha256=gLn9bMH0h0tPX3dP4i4QQxQK4Htkyg5r4KcqdMBaP6Q,12125
 unrealon_browser/cli/main.py,sha256=XCYcTxJUqaz320KCU_JPKizYMk6bdljb8Boyok3uO-4,1353
 unrealon_browser/core/__init__.py,sha256=uVL_t4sZelUzflWPdgrwoXGnAkSV1WNQ98-eu0QB2eM,151
-unrealon_browser/core/browser_manager.py,sha256=9xxo0kqbUcssFBNpvIXavWBya2E3TmgHKplQQ2kkZeU,29598
+unrealon_browser/core/browser_manager.py,sha256=G7jZ9490j4yAsfW-VM1QXyRJGCsESX4dgaSO4-5EHko,29864
 unrealon_browser/dto/__init__.py,sha256=bApqcLz-KanEi0_MCiFPrQmGBoX3VBijP7XtBUyIfjo,1636
 unrealon_browser/dto/bot_detection.py,sha256=qXfC0HghV7m4L6qA87t3STi-166jM-QgoP6OYbCb4o4,6884
 unrealon_browser/dto/models/config.py,sha256=Why5H3rtFclmwbdczuDfhlgf-LDz72Aa8LhDX4_ayfw,1752
@@ -15,15 +15,16 @@ unrealon_browser/dto/models/dataclasses.py,sha256=zqhJVyzp4CvtuTBsZwm6n6TodVWrZf
 unrealon_browser/dto/models/detection.py,sha256=ma9ZNIjPR7HnjqZaAj6ZoskiewPFiSn_FgFXSkgiQc8,2715
 unrealon_browser/dto/models/enums.py,sha256=Q4WzHdfSKf7dhKyX00i_Pvl2U8w3lBsxOYfSIoaQY3Q,1219
 unrealon_browser/dto/models/statistics.py,sha256=aIzJNV5r23VBxjhEoja4tXwI1Z7_UCw5zOaxuPya2E8,2728
-unrealon_browser/managers/__init__.py,sha256=WMzrYGrCXn467XWBr8msrvT43ckX_EQSe9Ye9cfMozg,528
+unrealon_browser/managers/__init__.py,sha256=lpa93ggEN93ucoi4FqnCG_sn-_aRlP1As7DBRogDSsQ,591
 unrealon_browser/managers/captcha.py,sha256=KGBO7sfq9XusAlcPByUFdIg-v6rlruzS2oHx-Zx28wo,21453
 unrealon_browser/managers/cookies.py,sha256=r4VVnKLXH82vhU7qgtY-dF7KPf0Ie3QxGD3FEi6geFA,15085
 unrealon_browser/managers/logger_bridge.py,sha256=I3KNUmxSV9xL1JHpc6WOVCzn_kLlOxJzfZy76rsxr48,10912
 unrealon_browser/managers/page_wait_manager.py,sha256=UyZqiSfkjzahrxp9x1odXFIT_sFhZGvdECxWuIMCVBY,7876
 unrealon_browser/managers/profile.py,sha256=HjddlSeUry_65WPtF8CMkT7cfJ6X3Jap9kJaaZpwtAA,18956
+unrealon_browser/managers/script_manager.py,sha256=z2TxYKlhXIsXNz6bCStLF7cxthBOZEESkhoFI1TCu_Y,11453
 unrealon_browser/stealth/__init__.py,sha256=zUfkPPafYlPANLVQIy-Se11R_UjcJakUb3krCxxUK5Q,842
 unrealon_browser/stealth/bypass_techniques.pyc,sha256=Tys_I4tnJmL9aQLB1k1mL-4OtzedhpYTmW4XPEIb3cI,25790
-unrealon_browser/stealth/manager.pyc,sha256=prGC5fWE-T4ROGSVrc65CJ_qFpb8dScuN83FotZ1oj4,21183
+unrealon_browser/stealth/manager.pyc,sha256=snye2EmZWo36PRy-dG9MrB837gVhVRjt6MgawA1mAF4,21540
 unrealon_browser/stealth/nodriver_stealth.pyc,sha256=SnDMdeG_W2LDK_3UfrjDeolMpqxTMDNjRbId_YC_cKA,15286
 unrealon_browser/stealth/playwright_stealth.pyc,sha256=Y-IiRTu136R6LIstGdKbns1j-CODX3sTGVT0IGx_MF4,6529
 unrealon_browser/stealth/scanner_tester.pyc,sha256=ki_Lp7zBp12iWgj3LGPnXTwemwgaGG1UrWeM1IXB6Gs,21711
@@ -32,7 +33,7 @@ unrealon_core/__init__.py,sha256=ZAinyQDsAS63xqnl5uqUKTna9M-xFDs6fLwy8-Hh-X0,559
 unrealon_core/version.py,sha256=qZOlKA_Hsz7_KXsCLO_0l9Mf0u_iNTxvHZPV21DwCqs,5803
 unrealon_core/config/__init__.py,sha256=57-KZaTDya0oyfOCfZ3pU1xLLbnZPBgqwfP9VrfhcKE,395
 unrealon_core/config/environment.py,sha256=TACbyjr3lxrA9R_Ve0LLqyLnRk3FShm2imQjhiP9i98,4589
-unrealon_core/config/urls.py,sha256=7ScpWEhSu1kewhMaKhuqgiMHIv_fyI1TyRiS-f2wrLo,2821
+unrealon_core/config/urls.py,sha256=FjdXI5WJzdveNCYS8FryRQX9gxjSSH-Qu30dT-USUts,2869
 unrealon_core/enums/__init__.py,sha256=wOH3DK3E1xUO3jD4Cu0UFzTL3HaHq9UEdwP70YdnyI0,736
 unrealon_core/enums/events.py,sha256=rlidf5M2-EDZCRLv4tBkBQ0--qsrgKFt5WrAgw7FRGg,2704
 unrealon_core/enums/jobs.py,sha256=sTSjhgnwE9O6tkgecYSI2BFxECEW0Ro5lkPF1oiEy40,787
@@ -114,7 +115,7 @@ unrealon_driver/driver/utilities/logging.py,sha256=2my2QnkAa6Hdw-TfO4oOQ94yGc-Cj
 unrealon_driver/driver/utilities/serialization.py,sha256=wTCSVrEloykiGN4K1JXbk2aqNKm7W90aWXmzhcLyAZc,2123
 unrealon_driver/managers/__init__.py,sha256=zJJsOb6Oodg7l00v4ncKUytnyeaZM887pHY8-eSuWdU,981
 unrealon_driver/managers/base.py,sha256=GkuXillg9uqqnx6RL682fmKgK-7JyqYlH6DFUgyN4F8,5445
-unrealon_driver/managers/browser.py,sha256=9kwdmWPxWFpNQ1KejtpzdN615V8PBJu8Y2nUOewQaeQ,3489
+unrealon_driver/managers/browser.py,sha256=v2FjgtgzTM94Qobll9_7TGRVbxsMlVvirP_sj3u152Q,4608
 unrealon_driver/managers/cache.py,sha256=c0tPKQ5KFd_Un1U8mw3j1WPuycxg863MMWNMveVF_2I,3506
 unrealon_driver/managers/http.py,sha256=EjlpoTRuhpsgzzrEARxRlbGczzua7hnKFVq06bvCgTM,3624
 unrealon_driver/managers/logger.py,sha256=PL3rA9ZQl12jJU0EiPAkLwJ6eDHQfIzr8-nc8bVivKQ,10526
@@ -124,9 +125,9 @@ unrealon_driver/managers/threading.py,sha256=djw5cSC99dfBKmep3IJ_8IgxQceMXtNvCp5
 unrealon_driver/managers/update.py,sha256=-hohVxGXpj5bZ6ZTQN6NH1RK9Pd6GVzCMtu3GS2SdcQ,3582
 unrealon_driver/utils/__init__.py,sha256=qxXVoQJVdLJhaLBXk_LZV_062AhrvBrMPXWAKfEc3C4,104
 unrealon_driver/utils/time.py,sha256=Oxk1eicKeZl8ZWbf7gu1Ll716k6CpXmVj67FHSnPIsA,184
-unrealon-2.0.8.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
-unrealon-2.0.8.dist-info/METADATA,sha256=sCydlGvYhaf2-tzj-0twQUib9nTG5LR-4bHPAqwuDWI,15688
-unrealon-2.0.8.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
-unrealon-2.0.8.dist-info/entry_points.txt,sha256=k0qM-eotpajkKUq-almJmxj9afhXprZ6IkvQkSdcKhI,104
-unrealon-2.0.8.dist-info/top_level.txt,sha256=Gu8IeIfIVfUxdi-h-F0nKMQxo15pjhHZ0aTadXTpRE8,47
-unrealon-2.0.8.dist-info/RECORD,,
+unrealon-2.0.10.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
+unrealon-2.0.10.dist-info/METADATA,sha256=fatBlrWFUSdwc5LjMflYHXNVlalO65qX482hwKToNpE,15689
+unrealon-2.0.10.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
+unrealon-2.0.10.dist-info/entry_points.txt,sha256=k0qM-eotpajkKUq-almJmxj9afhXprZ6IkvQkSdcKhI,104
+unrealon-2.0.10.dist-info/top_level.txt,sha256=Gu8IeIfIVfUxdi-h-F0nKMQxo15pjhHZ0aTadXTpRE8,47
+unrealon-2.0.10.dist-info/RECORD,,

unrealon_browser/core/browser_manager.py

@@ -30,6 +30,7 @@ from unrealon_browser.managers import (
     CaptchaDetector,
     create_browser_logger_bridge,
     PageWaitManager,
+    ScriptManager,
 )
 
 
@@ -66,6 +67,7 @@ class BrowserManager:
         self.cookie_manager = None
         self.captcha_manager = CaptchaDetector()
         self.page_wait = PageWaitManager(None, self.logger_bridge)
+        self.script_manager = ScriptManager(None, self.logger_bridge)
 
         # Signal handlers for graceful shutdown
         self._setup_signal_handlers()
@@ -231,6 +233,9 @@ class BrowserManager:
 
             # Update page wait manager with new page
             self.page_wait.update_page(self._page)
+            
+            # Update script manager with new page
+            self.script_manager.update_page(self._page)
 
             # 🔥 STEALTH ALWAYS APPLIED TO EVERY PAGE!
             stealth_success = await self.stealth_manager.apply_stealth(self._page)
@@ -644,6 +649,7 @@ class BrowserManager:
                 finally:
                     self._page = None
                     self.page_wait.update_page(None)
+                    self.script_manager.update_page(None)
 
             # Close context with safety checks
             if self._context:

unrealon_browser/managers/__init__.py

@@ -8,6 +8,7 @@ from .logger_bridge import BrowserLoggerBridge, create_browser_logger_bridge
 from .cookies import CookieManager
 from .captcha import CaptchaDetector
 from .page_wait_manager import PageWaitManager
+from .script_manager import ScriptManager
 
 
 __all__ = [
@@ -18,4 +19,5 @@ __all__ = [
     "CookieManager",
     "CaptchaDetector",
     "PageWaitManager",
+    "ScriptManager",
 ]

unrealon_browser/managers/script_manager.py

@@ -0,0 +1,314 @@
+"""
+Script Manager - JavaScript execution and evaluation manager
+Layer 2.5: JavaScript Integration - Handles script execution, API calls, and result processing
+"""
+
+import asyncio
+import json
+from typing import Any, Dict, Optional, Union, List
+from datetime import datetime, timezone
+from playwright.async_api import Page
+
+from .logger_bridge import BrowserLoggerBridge as LoggingBridge
+
+
+class ScriptManager:
+    """Manager for JavaScript execution and evaluation"""
+    
+    def __init__(self, page: Optional[Page], logger_bridge: LoggingBridge):
+        self._page = page
+        self.logger_bridge = logger_bridge
+        
+        # Statistics
+        self._scripts_executed = 0
+        self._scripts_successful = 0
+        self._scripts_failed = 0
+        self._api_calls_made = 0
+        self._execution_history: List[Dict[str, Any]] = []
+    
+    def update_page(self, page: Optional[Page]):
+        """Update the page reference"""
+        self._page = page
+    
+    async def execute_script(self, script: str, timeout: int = 30000) -> Any:
+        """
+        Execute JavaScript code and return result
+        
+        Args:
+            script: JavaScript code to execute
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            Script execution result
+        """
+        if not self._page:
+            raise RuntimeError("No page available for script execution")
+        
+        start_time = datetime.now()
+        self._scripts_executed += 1
+        
+        try:
+            self.logger_bridge.log_info(f"🔧 Executing JavaScript (timeout: {timeout}ms)")
+            self.logger_bridge.log_debug(f"Script preview: {script[:100]}...")
+            
+            # Execute script with timeout
+            result = await asyncio.wait_for(
+                self._page.evaluate(script),
+                timeout=timeout / 1000
+            )
+            
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_successful += 1
+            
+            # Log execution details
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": True,
+                "result_type": type(result).__name__,
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_info(f"✅ Script executed successfully ({duration_ms:.1f}ms)")
+            self.logger_bridge.log_debug(f"Result type: {type(result).__name__}")
+            
+            return result
+            
+        except asyncio.TimeoutError:
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_failed += 1
+            
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": False,
+                "error": "Timeout",
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_error(f"⏰ Script execution timeout ({timeout}ms)")
+            raise
+            
+        except Exception as e:
+            duration_ms = (datetime.now() - start_time).total_seconds() * 1000
+            self._scripts_failed += 1
+            
+            execution_record = {
+                "timestamp": start_time.isoformat(),
+                "duration_ms": duration_ms,
+                "success": False,
+                "error": str(e),
+                "script_length": len(script),
+            }
+            self._execution_history.append(execution_record)
+            
+            self.logger_bridge.log_error(f"❌ Script execution failed: {e}")
+            raise
+    
+    async def execute_api_call(self, api_url: str, headers: Dict[str, str], method: str = "GET", timeout: int = 30000) -> Dict[str, Any]:
+        """
+        Execute API call via JavaScript fetch
+        
+        Args:
+            api_url: API endpoint URL
+            headers: HTTP headers
+            method: HTTP method
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            API response data
+        """
+        self._api_calls_made += 1
+        
+        # Build fetch script
+        headers_json = json.dumps(headers)
+        
+        script = f"""
+        (async function() {{
+            try {{
+                const response = await fetch('{api_url}', {{
+                    method: '{method}',
+                    headers: {headers_json}
+                }});
+                
+                if (!response.ok) {{
+                    throw new Error('HTTP error! status: ' + response.status);
+                }}
+                
+                const data = await response.json();
+                return data;
+            }} catch (error) {{
+                return {{ error: error.message }};
+            }}
+        }})()
+        """
+        
+        self.logger_bridge.log_info(f"🌐 Making API call: {method} {api_url}")
+        
+        result = await self.execute_script(script, timeout)
+        
+        if isinstance(result, dict) and 'error' in result:
+            self.logger_bridge.log_error(f"❌ API call failed: {result['error']}")
+        else:
+            self.logger_bridge.log_info(f"✅ API call successful")
+        
+        return result
+    
+    
+    async def wait_for_element(self, selector: str, timeout: int = 10000) -> bool:
+        """
+        Wait for element using JavaScript
+        
+        Args:
+            selector: CSS selector
+            timeout: Timeout in milliseconds
+            
+        Returns:
+            True if element found
+        """
+        script = f"""
+        (function() {{
+            return new Promise((resolve) => {{
+                const element = document.querySelector('{selector}');
+                if (element) {{
+                    resolve(true);
+                    return;
+                }}
+                
+                const observer = new MutationObserver(() => {{
+                    const element = document.querySelector('{selector}');
+                    if (element) {{
+                        observer.disconnect();
+                        resolve(true);
+                    }}
+                }});
+                
+                observer.observe(document.body, {{
+                    childList: true,
+                    subtree: true
+                }});
+                
+                setTimeout(() => {{
+                    observer.disconnect();
+                    resolve(false);
+                }}, {timeout});
+            }});
+        }})()
+        """
+        
+        self.logger_bridge.log_info(f"🎯 Waiting for element: {selector}")
+        
+        try:
+            result = await self.execute_script(script, timeout + 1000)
+            if result:
+                self.logger_bridge.log_info(f"✅ Element found: {selector}")
+            else:
+                self.logger_bridge.log_warning(f"⏰ Element timeout: {selector}")
+            return result
+        except Exception as e:
+            self.logger_bridge.log_error(f"❌ Element wait failed: {selector} - {e}")
+            return False
+    
+    async def inject_helper_functions(self) -> bool:
+        """
+        Inject helper JavaScript functions into page
+        
+        Returns:
+            True if injection successful
+        """
+        helper_script = """
+        window.unrealonHelpers = {
+            // Wait for element with promise
+            waitForElement: function(selector, timeout = 10000) {
+                return new Promise((resolve) => {
+                    const element = document.querySelector(selector);
+                    if (element) {
+                        resolve(element);
+                        return;
+                    }
+                    
+                    const observer = new MutationObserver(() => {
+                        const element = document.querySelector(selector);
+                        if (element) {
+                            observer.disconnect();
+                            resolve(element);
+                        }
+                    });
+                    
+                    observer.observe(document.body, {
+                        childList: true,
+                        subtree: true
+                    });
+                    
+                    setTimeout(() => {
+                        observer.disconnect();
+                        resolve(null);
+                    }, timeout);
+                });
+            },
+            
+            // Get page info
+            getPageInfo: function() {
+                return {
+                    url: window.location.href,
+                    title: document.title,
+                    readyState: document.readyState,
+                    timestamp: new Date().toISOString()
+                };
+            },
+            
+            // Check if SPA loaded
+            isSPAReady: function() {
+                return document.readyState === 'complete' && 
+                       document.querySelector('body').children.length > 0;
+            }
+        };
+        
+        console.log('🔧 UnrealOn helper functions injected');
+        """
+        
+        try:
+            await self.execute_script(helper_script)
+            self.logger_bridge.log_info("🔧 Helper functions injected successfully")
+            return True
+        except Exception as e:
+            self.logger_bridge.log_error(f"❌ Failed to inject helper functions: {e}")
+            return False
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get script execution statistics"""
+        success_rate = (self._scripts_successful / self._scripts_executed * 100) if self._scripts_executed > 0 else 0
+        
+        return {
+            "scripts_executed": self._scripts_executed,
+            "scripts_successful": self._scripts_successful,
+            "scripts_failed": self._scripts_failed,
+            "success_rate": success_rate,
+            "api_calls_made": self._api_calls_made,
+            "execution_history_count": len(self._execution_history),
+        }
+    
+    def print_statistics(self) -> None:
+        """Print script execution statistics"""
+        stats = self.get_statistics()
+        
+        self.logger_bridge.log_info("\n🔧 Script Manager Statistics:")
+        self.logger_bridge.log_info(f"   Scripts executed: {stats['scripts_executed']}")
+        self.logger_bridge.log_info(f"   Successful: {stats['scripts_successful']}")
+        self.logger_bridge.log_info(f"   Failed: {stats['scripts_failed']}")
+        self.logger_bridge.log_info(f"   Success rate: {stats['success_rate']:.1f}%")
+        self.logger_bridge.log_info(f"   API calls made: {stats['api_calls_made']}")
+        
+        # Show recent executions
+        if self._execution_history:
+            self.logger_bridge.log_info("   Recent executions:")
+            for execution in self._execution_history[-3:]:  # Show last 3
+                status = "✅" if execution["success"] else "❌"
+                self.logger_bridge.log_info(f"     {status} {execution['duration_ms']:.1f}ms")
+    
+    def clear_history(self) -> None:
+        """Clear execution history"""
+        self._execution_history.clear()
+        self.logger_bridge.log_info("🧹 Cleared script execution history")

unrealon_core/config/urls.py

@@ -24,6 +24,7 @@ class URLConfig(BaseModel):
     
     # Cloud platform URLs  
     cloud_base_url: str = Field(
+        default="https://cloud.unrealon.com",
         description="Base URL for UnrealOn Cloud platform"
     )
     
@@ -41,7 +42,7 @@ class URLConfig(BaseModel):
         if environment == Environment.PRODUCTION:
             return cls(
                 # scanner_url="https://cloud.unrealon.com/scanner",
-                cloud_base_url="https://cloud.unrealon.com",
+                # cloud_base_url="https://cloud.unrealon.com",
                 api_base_url="https://api.unrealon.com"
             )
         # elif environment == Environment.TESTING:

unrealon_driver/managers/browser.py

@@ -6,6 +6,8 @@ from typing import Optional
 from pydantic import Field
 
 from unrealon_browser import BrowserManager as CoreBrowserManager, BrowserConfig
+from unrealon_browser.dto.models.enums import BrowserMode
+
 from .base import BaseManager, ManagerConfig
 
 
@@ -14,6 +16,7 @@ class BrowserManagerConfig(ManagerConfig):
 
     headless: bool = Field(default=True, description="Run headless")
     parser_name: str = Field(..., description="Parser name for browser")
+    stealth_warmup_enabled: bool = Field(default=False, description="Enable stealth warmup")
 
 
 class BrowserManager(BaseManager):
@@ -50,8 +53,14 @@ class BrowserManager(BaseManager):
         try:
             self.logger.info("🚀 Starting browser (lazy initialization)...")
             
-            # Create browser config
-            browser_config = BrowserConfig(parser_name=self.config.parser_name)
+            # Create browser config with proper headless mode
+            browser_mode = BrowserMode.HEADLESS if self.config.headless else BrowserMode.HEADED
+            
+            browser_config = BrowserConfig(
+                parser_name=self.config.parser_name,
+                mode=browser_mode,
+                stealth_warmup_enabled=self.config.stealth_warmup_enabled
+            )
 
             # Create browser manager
             self.browser = CoreBrowserManager(browser_config)
@@ -96,3 +105,19 @@ class BrowserManager(BaseManager):
             self.logger.error(f"Get HTML failed: {e}")
             self.stats.record_operation(False, 0.0)
             return None
+    
+    async def execute_script_async(self, script: str) -> any:
+        """Execute JavaScript on current page via ScriptManager."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+        
+        try:
+            # Use ScriptManager from CoreBrowserManager
+            result = await self.browser.script_manager.execute_script(script)
+            self.stats.record_operation(True, 0.0)
+            return result
+        except Exception as e:
+            self.logger.error(f"Script execution failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            raise