unrealon 2.0.34__tar.gz → 2.0.35__tar.gz

{unrealon-2.0.34/unrealon.egg-info → unrealon-2.0.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.34
+Version: 2.0.35
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT

{unrealon-2.0.34 → unrealon-2.0.35}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unrealon"
-version = "2.0.34"
+version = "2.0.35"
 description = "Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities"
 authors = [
     {name = "UnrealOn Team", email = "team@unrealon.com"}

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-browser/src/unrealon_browser/core/browser_manager.py

@@ -31,6 +31,7 @@ from unrealon_browser.managers import (
     create_browser_logger_bridge,
     PageWaitManager,
     ScriptManager,
+    DataExtractionManager,
 )
 
 
@@ -71,6 +72,7 @@ class BrowserManager:
         self.captcha_manager = CaptchaDetector()
         self.page_wait = PageWaitManager(None, self.logger_bridge)
         self.script_manager = ScriptManager(None, self.logger_bridge)
+        self.data_extraction = DataExtractionManager(None, self.logger_bridge)
 
         # Signal handlers for graceful shutdown
         self._setup_signal_handlers()
@@ -251,6 +253,9 @@ class BrowserManager:
             
             # Update script manager with new page
             self.script_manager.update_page(self._page)
+            
+            # Update data extraction manager with new page
+            self.data_extraction.update_page(self._page)
 
             # 🔥 STEALTH ALWAYS APPLIED TO EVERY PAGE!
             stealth_success = await self.stealth_manager.apply_stealth(self._page)
@@ -476,17 +481,6 @@ class BrowserManager:
                 "error": str(e),
             }
 
-    async def get_page_content_async(self) -> Optional[str]:
-        """Get current page content"""
-        if not self._page:
-            return None
-
-        try:
-            return await self._page.content()
-        except Exception as e:
-            self.logger_bridge.log_error(f"❌ Failed to get page content: {e}")
-            return None
-
     async def execute_script_async(self, script: str) -> Any:
         """Execute JavaScript on current page"""
         if not self._page:
@@ -603,11 +597,6 @@ class BrowserManager:
             resolution_result = await self.captcha_manager.handle_captcha_interactive(self, detection_result, timeout_seconds)
 
             if resolution_result["success"]:
-                # Log successful captcha resolution
-                if hasattr(self, "_current_proxy") and self._current_proxy:
-                    proxy_host = self._current_proxy.get("host", "unknown")
-                    proxy_port = self._current_proxy.get("port", 0)
-                    self.logger_bridge.log_captcha_solved(proxy_host, proxy_port, manual=True)
 
                 # Update session status back to active
                 self.session_metadata.current_status = BrowserSessionStatus.ACTIVE
@@ -708,6 +697,7 @@ class BrowserManager:
                     self._page = None
                     self.page_wait.update_page(None)
                     self.script_manager.update_page(None)
+                    self.data_extraction.update_page(None)
 
             # Close context with safety checks
             if self._context:

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-browser/src/unrealon_browser/dto/models/statistics.py

@@ -62,7 +62,6 @@ class BrowserStatistics(BaseModel):
     
     # Captcha metrics
     captcha_encounters: int = Field(default=0)
-    captcha_solved: int = Field(default=0)
     captcha_timeouts: int = Field(default=0)
     
     # Performance

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-browser/src/unrealon_browser/managers/__init__.py

@@ -9,6 +9,7 @@ from .cookies import CookieManager
 from .captcha import CaptchaDetector
 from .page_wait_manager import PageWaitManager
 from .script_manager import ScriptManager
+from .data_extraction_manager import DataExtractionManager
 
 
 __all__ = [
@@ -20,4 +21,5 @@ __all__ = [
     "CaptchaDetector",
     "PageWaitManager",
     "ScriptManager",
+    "DataExtractionManager",
 ]

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-browser/src/unrealon_browser/managers/captcha.py

@@ -310,7 +310,7 @@ class CaptchaDetector:
 
                 if cookies_saved:
                     self._logger("💾 Cookies saved after captcha resolution", "info")
-                    self._captchas_solved += 1
+                    # self._captchas_solved += 1  # Solving disabled
                 else:
                     self._logger("⚠️ Failed to save cookies after captcha resolution", "warning")
 
@@ -499,8 +499,8 @@ class CaptchaDetector:
 
         print(f"\n🤖 Captcha Detection Statistics:")
         print(f"   Captchas detected: {stats['captchas_detected']}")
-        print(f"   Captchas solved: {stats['captchas_solved']}")
-        print(f"   Success rate: {stats['success_rate']:.1f}%")
+        print(f"   Captchas solved: {stats['captchas_solved']} (solving disabled)")
+        print(f"   Success rate: N/A (solving disabled)")
         print(f"   Detection history: {stats['detection_history_count']} events")
         print(f"   Supported types: {', '.join(stats['supported_types'])}")
 

unrealon-2.0.35/unrealon-browser/src/unrealon_browser/managers/data_extraction_manager.py

@@ -0,0 +1,266 @@
+"""
+Data Extraction Manager - Extract different types of data from pages
+"""
+import json
+from typing import Optional, Dict, Any, Union
+from playwright.async_api import Page
+
+from .logger_bridge import BrowserLoggerBridge as LoggingBridge
+
+
+class DataExtractionManager:
+    """Manager for extracting different types of data from web pages"""
+    
+    def __init__(self, page: Optional[Page], logger_bridge: LoggingBridge):
+        self._page = page
+        self.logger_bridge = logger_bridge
+    
+    def update_page(self, page: Optional[Page]):
+        """Update the page reference"""
+        self._page = page
+    
+    async def get_json_content(self) -> Optional[Dict[str, Any]]:
+        """Extract JSON content from current page (for API endpoints)."""
+        if not self._page:
+            self.logger_bridge.log_error("No page available for JSON extraction")
+            return None
+        
+        try:
+            self.logger_bridge.log_info("🔍 Extracting JSON content from page...")
+            
+            # JavaScript to extract JSON from different page formats
+            script = """
+            (() => {
+                try {
+                    // Method 1: Try to get from document.body.textContent (for API responses)
+                    const bodyText = document.body.textContent || document.body.innerText || '';
+                    const cleanBodyText = bodyText.trim();
+                    
+                    if (cleanBodyText && (cleanBodyText.startsWith('{') || cleanBodyText.startsWith('['))) {
+                        return {
+                            success: true,
+                            data: JSON.parse(cleanBodyText),
+                            method: 'body_text'
+                        };
+                    }
+                    
+                    // Method 2: Try to get from <pre> tag (common for JSON APIs)
+                    const preElement = document.querySelector('pre');
+                    if (preElement) {
+                        const preText = (preElement.textContent || preElement.innerText || '').trim();
+                        if (preText && (preText.startsWith('{') || preText.startsWith('['))) {
+                            return {
+                                success: true,
+                                data: JSON.parse(preText),
+                                method: 'pre_element'
+                            };
+                        }
+                    }
+                    
+                    // Method 3: Check if entire document is JSON
+                    const docText = (document.documentElement.textContent || document.documentElement.innerText || '').trim();
+                    if (docText && (docText.startsWith('{') || docText.startsWith('['))) {
+                        return {
+                            success: true,
+                            data: JSON.parse(docText),
+                            method: 'document_text'
+                        };
+                    }
+                    
+                    // Method 4: Look for JSON in script tags
+                    const scriptTags = document.querySelectorAll('script[type="application/json"]');
+                    for (const script of scriptTags) {
+                        const scriptText = (script.textContent || script.innerText || '').trim();
+                        if (scriptText && (scriptText.startsWith('{') || scriptText.startsWith('['))) {
+                            return {
+                                success: true,
+                                data: JSON.parse(scriptText),
+                                method: 'script_tag'
+                            };
+                        }
+                    }
+                    
+                    return {
+                        success: false,
+                        error: 'No JSON content found',
+                        page_text_preview: cleanBodyText.substring(0, 200)
+                    };
+                    
+                } catch (e) {
+                    return {
+                        success: false,
+                        error: 'JSON parse failed: ' + e.message,
+                        page_text_preview: (document.body.textContent || '').substring(0, 200)
+                    };
+                }
+            })();
+            """
+            
+            result = await self._page.evaluate(script)
+            
+            if result.get('success'):
+                method = result.get('method', 'unknown')
+                self.logger_bridge.log_info(f"✅ JSON extracted successfully using method: {method}")
+                return result.get('data')
+            else:
+                error = result.get('error', 'Unknown error')
+                preview = result.get('page_text_preview', '')
+                self.logger_bridge.log_warning(f"❌ JSON extraction failed: {error}")
+                if preview:
+                    self.logger_bridge.log_info(f"📄 Page preview: {preview}...")
+                return None
+                
+        except Exception as e:
+            self.logger_bridge.log_error(f"JSON extraction error: {e}")
+            return None
+    
+    async def get_page_text(self) -> Optional[str]:
+        """Get plain text content from current page."""
+        if not self._page:
+            return None
+        
+        try:
+            self.logger_bridge.log_info("📄 Extracting plain text content...")
+            
+            script = """
+            (() => {
+                return {
+                    body_text: document.body.textContent || document.body.innerText || '',
+                    title: document.title || '',
+                    url: window.location.href
+                };
+            })();
+            """
+            
+            result = await self._page.evaluate(script)
+            text = result.get('body_text', '').strip()
+            
+            if text:
+                self.logger_bridge.log_info(f"✅ Text extracted: {len(text)} characters")
+                return text
+            else:
+                self.logger_bridge.log_warning("❌ No text content found")
+                return None
+                
+        except Exception as e:
+            self.logger_bridge.log_error(f"Text extraction error: {e}")
+            return None
+    
+    async def get_structured_data(self) -> Optional[Dict[str, Any]]:
+        """Get structured data including JSON, text, and metadata."""
+        if not self._page:
+            return None
+        
+        try:
+            self.logger_bridge.log_info("🔍 Extracting structured data...")
+            
+            # Try JSON first
+            json_data = await self.get_json_content()
+            
+            # Get page metadata
+            script = """
+            (() => {
+                return {
+                    url: window.location.href,
+                    title: document.title || '',
+                    content_type: document.contentType || '',
+                    charset: document.characterSet || '',
+                    ready_state: document.readyState,
+                    has_pre_element: !!document.querySelector('pre'),
+                    body_text_length: (document.body.textContent || '').length
+                };
+            })();
+            """
+            
+            metadata = await self._page.evaluate(script)
+            
+            result = {
+                "extraction_success": json_data is not None,
+                "json_data": json_data,
+                "metadata": metadata,
+                "extracted_at": self._get_timestamp()
+            }
+            
+            if json_data:
+                self.logger_bridge.log_info("✅ Structured data extraction successful")
+            else:
+                self.logger_bridge.log_warning("⚠️ No JSON data found, but metadata extracted")
+            
+            return result
+            
+        except Exception as e:
+            self.logger_bridge.log_error(f"Structured data extraction error: {e}")
+            return None
+    
+    async def detect_content_type(self) -> str:
+        """Detect the type of content on the current page."""
+        if not self._page:
+            return "unknown"
+        
+        try:
+            script = """
+            (() => {
+                const bodyText = (document.body.textContent || '').trim();
+                const contentType = document.contentType || '';
+                const hasPreElement = !!document.querySelector('pre');
+                
+                // Check for JSON
+                if (bodyText.startsWith('{') || bodyText.startsWith('[')) {
+                    return 'json';
+                }
+                
+                // Check for XML
+                if (bodyText.startsWith('<') && contentType.includes('xml')) {
+                    return 'xml';
+                }
+                
+                // Check for HTML
+                if (document.querySelector('html') && document.querySelector('body') && !hasPreElement) {
+                    return 'html';
+                }
+                
+                // Check for plain text
+                if (hasPreElement || contentType.includes('text/plain')) {
+                    return 'text';
+                }
+                
+                return 'unknown';
+            })();
+            """
+            
+            content_type = await self._page.evaluate(script)
+            self.logger_bridge.log_info(f"🔍 Detected content type: {content_type}")
+            return content_type
+            
+        except Exception as e:
+            self.logger_bridge.log_error(f"Content type detection error: {e}")
+            return "unknown"
+    
+    async def get_page_html(self) -> Optional[str]:
+        """Get full HTML content from current page."""
+        if not self._page:
+            return None
+        
+        try:
+            self.logger_bridge.log_info("📄 Extracting HTML content...")
+            html = await self._page.content()
+            
+            if html:
+                self.logger_bridge.log_info(f"✅ HTML extracted: {len(html)} characters")
+                return html
+            else:
+                self.logger_bridge.log_warning("❌ No HTML content found")
+                return None
+                
+        except Exception as e:
+            self.logger_bridge.log_error(f"HTML extraction error: {e}")
+            return None
+    
+    def _get_timestamp(self) -> str:
+        """Get current timestamp in ISO format."""
+        from datetime import datetime
+        return datetime.now().isoformat()
+
+
+# Export
+__all__ = ["DataExtractionManager"]

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-browser/src/unrealon_browser/managers/logger_bridge.py

@@ -81,7 +81,6 @@ class BrowserLoggerBridge:
             "navigation_failed": 0,
             "stealth_applied": 0,
             "captcha_detected": 0,
-            "captcha_solved": 0,
             "profile_created": 0,
             "cookies_saved": 0,
         }
@@ -194,17 +193,6 @@ class BrowserLoggerBridge:
             detected_at=result.detected_at.isoformat(),
         )
 
-    def log_captcha_solved(self, proxy_host: str, proxy_port: int, manual: bool = True) -> None:
-        """Log captcha resolution"""
-        self._browser_events["captcha_solved"] += 1
-        self._log_info(
-            f"Captcha solved for proxy {proxy_host}:{proxy_port}",
-            proxy_host=proxy_host,
-            proxy_port=proxy_port,
-            resolution_method="manual" if manual else "automatic",
-            cookies_will_be_saved=True,
-        )
-
     def log_profile_created(self, profile_name: str, proxy_info: Optional[Dict[str, Any]] = None) -> None:
         """Log profile creation"""
         self._browser_events["profile_created"] += 1

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-driver/src/unrealon_driver/driver/core/driver.py

@@ -21,7 +21,7 @@ from ..utilities.logging import LoggingUtility
 from ..utilities.serialization import SerializationUtility
 
 from ...managers import (
-    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    LoggerManager, HttpManager, HttpxManager, BrowserManager, CacheManager,
     ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
 )
 
@@ -77,6 +77,7 @@ class UniversalDriver:
         self.manager_registry: Optional[ManagerRegistry] = None
         self.logger_manager: Optional[LoggerManager] = None
         self.http: Optional[HttpManager] = None
+        self.httpx: Optional[HttpxManager] = None
         self.browser: Optional[BrowserManager] = None
         self.cache: Optional[CacheManager] = None
         self.proxy: Optional[ProxyManager] = None

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-driver/src/unrealon_driver/driver/factory/manager_factory.py

@@ -8,11 +8,12 @@ import logging
 from typing import TYPE_CHECKING
 
 from ...managers import (
-    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    LoggerManager, HttpManager, HttpxManager, BrowserManager, CacheManager,
     ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
 )
 from ...managers.logger import LoggerManagerConfig
 from ...managers.http import HttpManagerConfig
+from ...managers.http import HttpxManagerConfig
 from ...managers.browser import BrowserManagerConfig
 from ...managers.cache import CacheManagerConfig
 from ...managers.proxy import ProxyManagerConfig
@@ -45,6 +46,7 @@ class ManagerFactory:
         # Setup each manager
         ManagerFactory._setup_logger_manager(driver, manager_registry)
         ManagerFactory._setup_http_manager(driver, manager_registry)
+        ManagerFactory._setup_httpx_manager(driver, manager_registry)
         ManagerFactory._setup_browser_manager(driver, manager_registry)
         ManagerFactory._setup_cache_manager(driver, manager_registry)
         ManagerFactory._setup_proxy_manager(driver, manager_registry)
@@ -78,6 +80,17 @@ class ManagerFactory:
         driver.http = HttpManager(http_config)
         registry.register(driver.http)
     
+    @staticmethod
+    def _setup_httpx_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup HTTPx manager."""
+        httpx_config = HttpxManagerConfig(
+            enabled=True,
+            timeout=driver.config.http_timeout,
+            max_retries=driver.config.max_retries
+        )
+        driver.httpx = HttpxManager(httpx_config)
+        registry.register(driver.httpx)
+    
     @staticmethod
     def _setup_browser_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
         """Setup browser manager."""

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-driver/src/unrealon_driver/managers/__init__.py

@@ -4,7 +4,7 @@ Clean manager system for UnrealOn Driver.
 
 from .base import BaseManager, ManagerConfig, ManagerStatus
 from .logger import LoggerManager, LoggerManagerConfig
-from .http import HttpManager, HttpManagerConfig
+from .http import HttpManager, HttpManagerConfig, HttpxManager, HttpxManagerConfig
 from .browser import BrowserManager, BrowserManagerConfig
 from .cache import CacheManager, CacheManagerConfig
 from .proxy import ProxyManager, ProxyManagerConfig
@@ -21,6 +21,7 @@ __all__ = [
     # Managers
     "LoggerManager", "LoggerManagerConfig",
     "HttpManager", "HttpManagerConfig",
+    "HttpxManager", "HttpxManagerConfig",
     "BrowserManager", "BrowserManagerConfig",
     "CacheManager", "CacheManagerConfig",
     "ProxyManager", "ProxyManagerConfig",

{unrealon-2.0.34 → unrealon-2.0.35}/unrealon-driver/src/unrealon_driver/managers/browser.py

@@ -113,7 +113,7 @@ class BrowserManager(BaseManager):
             raise RuntimeError("Failed to initialize browser")
 
         try:
-            html = await self.browser.get_page_content_async()
+            html = await self.browser.data_extraction.get_page_html()
             self.stats.record_operation(True, 0.0)
             return html
         except Exception as e:
@@ -121,6 +121,36 @@ class BrowserManager(BaseManager):
             self.stats.record_operation(False, 0.0)
             return None
     
+    async def get_json_content(self) -> Optional[dict]:
+        """Extract JSON content from current page via DataExtractionManager."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+        
+        try:
+            result = await self.browser.data_extraction.get_json_content()
+            self.stats.record_operation(True, 0.0)
+            return result
+        except Exception as e:
+            self.logger.error(f"JSON extraction failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            return None
+    
+    async def get_page_text(self) -> Optional[str]:
+        """Get plain text content from current page via DataExtractionManager."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+        
+        try:
+            result = await self.browser.data_extraction.get_page_text()
+            self.stats.record_operation(True, 0.0)
+            return result
+        except Exception as e:
+            self.logger.error(f"Text extraction failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            return None
+    
     async def execute_script_async(self, script: str) -> any:
         """Execute JavaScript on current page via ScriptManager."""
         # Ensure browser is initialized
@@ -136,3 +166,9 @@ class BrowserManager(BaseManager):
             self.logger.error(f"Script execution failed: {e}")
             self.stats.record_operation(False, 0.0)
             raise
+    
+    async def cleanup(self) -> None:
+        """Cleanup browser resources."""
+        if self.browser:
+            await self.browser.close_async()
+            self.browser = None