scitex 2.5.0__py3-none-any.whl → 2.7.3__py3-none-any.whl

scitex/browser/remote/ZenRowsBrowserManager.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -32,9 +33,7 @@ class ZenRowsRemoteScholarBrowserManager:
     def __init__(
         self,
         auth_manager=None,
-        zenrows_api_key: Optional[str] = os.getenv(
-            "SCITEX_SCHOLAR_ZENROWS_API_KEY"
-        ),
+        zenrows_api_key: Optional[str] = os.getenv("SCITEX_SCHOLAR_ZENROWS_API_KEY"),
         proxy_country: Optional[str] = os.getenv(
             "SCITEX_SCHOLAR_ZENROWS_PROXY_COUNTRY"
         ),
@@ -62,11 +61,10 @@ class ZenRowsRemoteScholarBrowserManager:
         self._browser: Optional[Browser] = None
         self._context: Optional[BrowserContext] = None
         self.cookie_acceptor = CookieAutoAcceptor()
-        
+
         # Also initialize API browser for reliable screenshots
         self._api_browser = ZenRowsAPIBrowser(
-            api_key=self.zenrows_api_key,
-            proxy_country=self.proxy_country or "au"
+            api_key=self.zenrows_api_key, proxy_country=self.proxy_country or "au"
         )
 
     async def get_browser_async(self) -> Browser:
@@ -80,26 +78,26 @@ class ZenRowsRemoteScholarBrowserManager:
 
         # Build connection URL with optional country parameter
         connection_url = f"wss://browser.zenrows.com?apikey={self.zenrows_api_key}"
-        
+
         # Note: Country routing via WebSocket URL is not documented
         # but we can try appending it as a parameter
         if self.proxy_country:
             connection_url += f"&proxy_country={self.proxy_country}"
             logger.debug(f"Requesting proxy country: {self.proxy_country.upper()}")
-        
+
         try:
             self._browser = await self._playwright.chromium.connect_over_cdp(
                 connection_url
             )
             logger.debug("Successfully connected to ZenRows browser")
-            
+
             # Log a note about country routing
             if self.proxy_country:
                 logger.debug(
                     "Note: Country routing via Scraping Browser is experimental. "
                     "Use API mode for guaranteed country-specific IPs."
                 )
-            
+
             return self._browser
         except Exception as e:
             logger.error(f"Failed to connect to ZenRows browser: {e}")
@@ -135,9 +133,7 @@ class ZenRowsRemoteScholarBrowserManager:
             try:
                 cookies = await self.auth_manager.get_auth_cookies_async()
                 await context.add_cookies(cookies)
-                logger.success(
-                    f"Injected {len(cookies)} authentication cookies"
-                )
+                logger.success(f"Injected {len(cookies)} authentication cookies")
             except Exception as e:
                 logger.error(f"Failed to inject auth cookies: {e}")
 
@@ -168,27 +164,23 @@ class ZenRowsRemoteScholarBrowserManager:
         self._browser = None
         self._context = None
         self._playwright = None
-    
+
     async def take_screenshot_reliable_async(
-        self,
-        url: str,
-        output_path: str,
-        use_api: bool = True,
-        wait_ms: int = 5000
+        self, url: str, output_path: str, use_api: bool = True, wait_ms: int = 5000
     ) -> Dict[str, Any]:
         """Take a screenshot with automatic CAPTCHA handling.
-        
+
         This method provides reliable screenshot capture by:
         1. Using the API approach by default (more reliable)
         2. Falling back to WebSocket browser if needed
         3. Automatically handling CAPTCHAs via ZenRows
-        
+
         Args:
             url: URL to screenshot
             output_path: Path to save screenshot
             use_api: Use API browser (recommended) vs WebSocket
             wait_ms: Additional wait time
-            
+
         Returns:
             Dict with success status and details
         """
@@ -196,9 +188,7 @@ class ZenRowsRemoteScholarBrowserManager:
             # Use API browser for reliability
             logger.debug("Using ZenRows API for screenshot (recommended)")
             return await self._api_browser.navigate_and_screenshot_async(
-                url=url,
-                screenshot_path=output_path,
-                wait_ms=wait_ms
+                url=url, screenshot_path=output_path, wait_ms=wait_ms
             )
         else:
             # Use WebSocket browser (less reliable for captchas)
@@ -207,52 +197,46 @@ class ZenRowsRemoteScholarBrowserManager:
                 browser = await self.get_browser_async()
                 context = await browser.new_context()
                 page = await context.new_page()
-                
+
                 # Navigate
                 await page.goto(url, wait_until="domcontentloaded", timeout=30000)
-                
+
                 # Wait for content
                 await page.wait_for_load_state("networkidle", timeout=10000)
                 await page.wait_for_timeout(wait_ms)
-                
+
                 # Take screenshot
                 await page.screenshot(path=output_path, full_page=True)
-                
+
                 await page.close()
                 await context.close()
-                
+
                 return {
                     "success": True,
-                    "screenshot": {
-                        "saved": True,
-                        "path": output_path
-                    }
+                    "screenshot": {"saved": True, "path": output_path},
                 }
             except Exception as e:
                 logger.error(f"WebSocket screenshot failed: {e}")
-                return {
-                    "success": False,
-                    "error": str(e)
-                }
-    
+                return {"success": False, "error": str(e)}
+
     async def navigate_and_extract_async(
         self,
         url: str,
         extract_pdf_url: bool = True,
         take_screenshot: bool = False,
-        screenshot_path: Optional[str] = None
+        screenshot_path: Optional[str] = None,
     ) -> Dict[str, Any]:
         """Navigate to URL and extract information.
-        
+
         This combines navigation, screenshot, and data extraction.
         Uses the API approach for better reliability.
-        
+
         Args:
             url: Target URL
             extract_pdf_url: Try to find PDF URL
             take_screenshot: Whether to capture screenshot
             screenshot_path: Where to save screenshot
-            
+
         Returns:
             Dict with extracted data
         """
@@ -260,28 +244,29 @@ class ZenRowsRemoteScholarBrowserManager:
             url=url,
             screenshot_path=screenshot_path if take_screenshot else None,
             return_html=extract_pdf_url,
-            wait_ms=8000  # Longer wait for academic sites
+            wait_ms=8000,  # Longer wait for academic sites
         )
-        
+
         if extract_pdf_url and result.get("html"):
             # Try to extract PDF URL
             import re
+
             html = result["html"]
-            
+
             pdf_patterns = [
                 r'href="([^"]+\.pdf[^"]*)"',
                 r'content="([^"]+\.pdf[^"]*)"',
                 r'data-pdf-url="([^"]+)"',
                 r'pdfUrl["\']?\s*:\s*["\']([^"\']+)',
             ]
-            
+
             for pattern in pdf_patterns:
                 match = re.search(pattern, html, re.IGNORECASE)
                 if match:
                     result["pdf_url"] = match.group(1)
                     logger.debug(f"Found PDF URL: {result['pdf_url']}")
                     break
-        
+
         return result
 
     async def __aenter__(self):
@@ -302,192 +287,233 @@ if __name__ == "__main__":
         import json
         from pathlib import Path
         from datetime import datetime
-        
+
         # Create screenshots directory
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         screenshots_dir = Path(f"./screenshots_remote_{timestamp}")
         screenshots_dir.mkdir(exist_ok=True)
-        
+
         # Test sites for comprehensive evaluation
         test_sites = [
             ("ip", "https://httpbin.org/ip", "Shows your public IP address"),
             ("headers", "https://httpbin.org/headers", "HTTP headers sent by browser"),
-            ("bot_detection", "https://bot.sannysoft.com/", "Bot tests - green=good, red=detected"),
-            ("fingerprint", "https://pixelscan.net/", "Browser fingerprinting analysis"),
+            (
+                "bot_detection",
+                "https://bot.sannysoft.com/",
+                "Bot tests - green=good, red=detected",
+            ),
+            (
+                "fingerprint",
+                "https://pixelscan.net/",
+                "Browser fingerprinting analysis",
+            ),
             ("webrtc", "https://browserleaks.com/webrtc", "WebRTC IP leak test"),
         ]
-        
+
         async def test_browser_async(browser_type, browser_manager, use_auth=False):
             """Test a browser with all test sites."""
-            print(f"\n{'='*60}")
+            print(f"\n{'=' * 60}")
             print(f"Testing: {browser_type}")
-            print('='*60)
-            
+            print("=" * 60)
+
             results = {}
-            
+
             try:
-                if use_auth and hasattr(browser_manager, 'get_authenticated_browser_and_context_async'):
+                if use_auth and hasattr(
+                    browser_manager, "get_authenticated_browser_and_context_async"
+                ):
                     # For managers with auth support
-                    browser, context = await browser_manager.get_authenticated_browser_and_context_async()
+                    (
+                        browser,
+                        context,
+                    ) = await browser_manager.get_authenticated_browser_and_context_async()
                     pages_via_context = True
                 else:
                     # Direct browser access
                     browser = await browser_manager.get_browser_async()
                     pages_via_context = False
-                
+
                 for test_name, url, description in test_sites:
                     print(f"\n{test_name}: {description}")
-                    
+
                     page = None
                     try:
                         if pages_via_context:
                             page = await context.new_page()
                         else:
                             page = await browser.new_page()
-                        
+
                         # Navigate with timeout
-                        await page.goto(url, wait_until="domcontentloaded", timeout=30000)
-                        
+                        await page.goto(
+                            url, wait_until="domcontentloaded", timeout=30000
+                        )
+
                         if test_name in ["ip", "headers"]:
                             # Extract text content
                             content = await page.text_content("pre")
                             print(f"Result: {content.strip()[:200]}...")
-                            
+
                             # Parse IP if available
                             if test_name == "ip":
                                 try:
                                     ip_data = json.loads(content)
-                                    results['ip'] = ip_data.get('origin', 'Unknown')
+                                    results["ip"] = ip_data.get("origin", "Unknown")
                                     print(f"Detected IP: {results['ip']}")
                                 except:
-                                    results['ip'] = 'Parse error'
+                                    results["ip"] = "Parse error"
                         else:
                             # Wait for dynamic content
                             await page.wait_for_timeout(5000)
-                            
+
                             # For fingerprint test, try to click start button
                             if test_name == "fingerprint":
                                 try:
-                                    await page.click('button:has-text("Start")', timeout=3000)
+                                    await page.click(
+                                        'button:has-text("Start")', timeout=3000
+                                    )
                                     await page.wait_for_timeout(5000)
                                 except:
                                     pass
-                        
+
                         # Take screenshot
-                        screenshot_path = screenshots_dir / f"{browser_type.lower().replace(' ', '_')}_{test_name}.png"
+                        screenshot_path = (
+                            screenshots_dir
+                            / f"{browser_type.lower().replace(' ', '_')}_{test_name}.png"
+                        )
                         await page.screenshot(path=screenshot_path, full_page=True)
                         print(f"Screenshot saved: {screenshot_path}")
-                        
+
                         results[test_name] = "Success"
-                        
+
                     except Exception as e:
                         print(f"Failed: {str(e)[:100]}...")
                         results[test_name] = f"Failed: {str(e)[:50]}"
                     finally:
                         if page:
                             await page.close()
-                
+
                 # Clean up
-                if hasattr(browser_manager, 'close'):
+                if hasattr(browser_manager, "close"):
                     await browser_manager.close()
-                    
+
             except Exception as e:
                 print(f"Browser initialization failed: {str(e)}")
-                results['error'] = str(e)
-            
+                results["error"] = str(e)
+
             return results
-        
+
         # Store all results
         all_results = {}
-        
+
         # Test 1: Regular browser (baseline) - if available
         print("\nChecking if we can import local browser for comparison...")
         try:
             from scitex.scholar.browser import ScholarBrowserManager
-            
+
             print("Initializing regular browser for baseline comparison...")
             regular_manager = ScholarBrowserManager(headless=False)
-            regular_results = await test_browser_async("Regular Browser", regular_manager)
+            regular_results = await test_browser_async(
+                "Regular Browser", regular_manager
+            )
             all_results["Regular Browser"] = regular_results
         except Exception as e:
             print(f"Regular browser not available for comparison: {e}")
             all_results["Regular Browser"] = {"error": "Not available"}
-        
+
         # Test 2: ZenRows Remote Browser (default settings)
         print("\nInitializing ZenRows Remote Browser...")
         try:
             zenrows_manager = ZenRowsRemoteScholarBrowserManager()
-            zenrows_results = await test_browser_async("ZenRows Remote", zenrows_manager)
+            zenrows_results = await test_browser_async(
+                "ZenRows Remote", zenrows_manager
+            )
             all_results["ZenRows Remote"] = zenrows_results
         except Exception as e:
             print(f"ZenRows Remote test failed: {e}")
             all_results["ZenRows Remote"] = {"error": str(e)}
-        
+
         # Test 3: ZenRows Remote Browser with country (if supported)
         print("\nInitializing ZenRows Remote Browser with AU country...")
         try:
-            zenrows_au_manager = ZenRowsRemoteScholarBrowserManager(proxy_country='au')
-            zenrows_au_results = await test_browser_async("ZenRows Remote AU", zenrows_au_manager)
+            zenrows_au_manager = ZenRowsRemoteScholarBrowserManager(proxy_country="au")
+            zenrows_au_results = await test_browser_async(
+                "ZenRows Remote AU", zenrows_au_manager
+            )
             all_results["ZenRows Remote AU"] = zenrows_au_results
         except Exception as e:
             print(f"ZenRows Remote AU test failed: {e}")
             all_results["ZenRows Remote AU"] = {"error": str(e)}
-        
+
         # Test 4: Test the API client as well
         print("\nTesting ZenRows API Client for comparison...")
         try:
             from ._ZenRowsAPIClient import ZenRowsAPIClient
-            
+
             print("Testing basic API request...")
             api_client = ZenRowsAPIClient()
             response = api_client.request("https://httpbin.org/ip")
             if response.status_code == 200:
                 ip_data = json.loads(response.text)
                 print(f"API Client IP (Basic): {ip_data.get('origin', 'Unknown')}")
-                print(f"API Cost: {response.headers.get('X-Request-Cost', 'Unknown')} credits")
-                all_results["API Client Basic"] = {"ip": ip_data.get('origin', 'Unknown')}
-            
+                print(
+                    f"API Cost: {response.headers.get('X-Request-Cost', 'Unknown')} credits"
+                )
+                all_results["API Client Basic"] = {
+                    "ip": ip_data.get("origin", "Unknown")
+                }
+
             print("\nTesting API with Australian proxy...")
-            api_client_au = ZenRowsAPIClient(default_country='au')
+            api_client_au = ZenRowsAPIClient(default_country="au")
             response_au = api_client_au.request("https://httpbin.org/ip")
             if response_au.status_code == 200:
                 ip_data_au = json.loads(response_au.text)
                 print(f"API Client IP (AU): {ip_data_au.get('origin', 'Unknown')}")
-                print(f"API Cost: {response_au.headers.get('X-Request-Cost', 'Unknown')} credits")
-                all_results["API Client AU"] = {"ip": ip_data_au.get('origin', 'Unknown')}
+                print(
+                    f"API Cost: {response_au.headers.get('X-Request-Cost', 'Unknown')} credits"
+                )
+                all_results["API Client AU"] = {
+                    "ip": ip_data_au.get("origin", "Unknown")
+                }
         except Exception as e:
             print(f"API Client test failed: {e}")
             all_results["API Client"] = {"error": str(e)}
-        
+
         # Print summary
-        print("\n" + "="*60)
+        print("\n" + "=" * 60)
         print("SUMMARY REPORT")
-        print("="*60)
-        
+        print("=" * 60)
+
         print("\nIP Addresses detected:")
         for method, data in all_results.items():
             if isinstance(data, dict):
-                ip = data.get('ip', 'Not tested')
+                ip = data.get("ip", "Not tested")
             else:
-                ip = 'Error'
+                ip = "Error"
             print(f"  {method:.<35} {ip}")
-        
+
         print(f"\nScreenshots saved in: {screenshots_dir.absolute()}")
-        
+
         # Save summary report
         summary_path = screenshots_dir / "test_summary.json"
-        with open(summary_path, 'w') as f:
-            json.dump({
-                'timestamp': timestamp,
-                'results': all_results,
-                'test_sites': [{"name": t[0], "url": t[1], "description": t[2]} for t in test_sites]
-            }, f, indent=2)
+        with open(summary_path, "w") as f:
+            json.dump(
+                {
+                    "timestamp": timestamp,
+                    "results": all_results,
+                    "test_sites": [
+                        {"name": t[0], "url": t[1], "description": t[2]}
+                        for t in test_sites
+                    ],
+                },
+                f,
+                indent=2,
+            )
         print(f"Summary report saved: {summary_path}")
-        
+
         # Comparison notes
-        print("\n" + "="*60)
+        print("\n" + "=" * 60)
         print("COMPARISON NOTES:")
-        print("="*60)
+        print("=" * 60)
         print("1. Regular Browser: Uses your local IP, no proxy")
         print("2. ZenRows Remote: Cloud browser with built-in anti-bot")
         print("3. ZenRows Remote AU: Attempts Australian IP (experimental)")

scitex/browser/stealth/HumanBehavior.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/browser/stealth/HumanBehavior.py"
-)
+
+__FILE__ = "./src/scitex/browser/stealth/HumanBehavior.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -93,9 +92,7 @@ class HumanBehavior:
         self, page: Page, selector: str = None, element=None
     ) -> None:
         """Hover over element before clicking with human-like timing."""
-        await browser_logger.debug(
-            page, f"{self.name}: Hovering and clicking..."
-        )
+        await browser_logger.debug(page, f"{self.name}: Hovering and clicking...")
         if selector:
             element = page.locator(selector)
 
@@ -111,9 +108,7 @@ class HumanBehavior:
             target_x = box["x"] + box["width"] / 2 + offset_x
             target_y = box["y"] + box["height"] / 2 + offset_y
 
-            await HumanBehavior.mouse_move_async(
-                page, int(target_x), int(target_y)
-            )
+            await HumanBehavior.mouse_move_async(page, int(target_x), int(target_y))
 
         # Hover
         await element.hover()

scitex/browser/stealth/StealthManager.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/browser/stealth/StealthManager.py"
-)
+
+__FILE__ = "./src/scitex/browser/stealth/StealthManager.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -57,9 +56,7 @@ class StealthManager:
             # "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
         ]
         user_agent = random.choice(user_agents)
-        logger.debug(
-            f"{self.name}: User Agent randomly selected: {user_agent}"
-        )
+        logger.debug(f"{self.name}: User Agent randomly selected: {user_agent}")
         return user_agent
 
     def get_random_viewport(self) -> dict:
@@ -90,9 +87,7 @@ class StealthManager:
                     {"width": 1280, "height": 720},
                 ]
             )
-            logger.debug(
-                f"{self.name}: Viewport randomly selected: {viewport}"
-            )
+            logger.debug(f"{self.name}: Viewport randomly selected: {viewport}")
             return viewport
 
     def get_stealth_options(self) -> dict:
@@ -224,9 +219,7 @@ class StealthManager:
         else:
             config_desc = "Default (1920x1080)"
 
-        logger.debug(
-            f"{self.name}: Browser window configuration: {config_desc}"
-        )
+        logger.debug(f"{self.name}: Browser window configuration: {config_desc}")
         return stealth_args
 
     def get_network_evasion_headers(self) -> dict:
@@ -246,17 +239,15 @@ class StealthManager:
             "Sec-Fetch-Site": "none",
             "Sec-Fetch-User": "?1",
             "Upgrade-Insecure-Requests": "1",
-            "X-Forwarded-For": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
-            "X-Real-IP": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
+            "X-Forwarded-For": f"{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}",
+            "X-Real-IP": f"{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}",
         }
 
     async def add_human_behavior_async(self, page: Page):
         """Add human-like behavior patterns to avoid detection."""
         # Random delay before starting interactions
         delay = random.uniform(2, 5)
-        logger.debug(
-            f"{self.name}: Adding human behavior delay: {delay:.2f} seconds"
-        )
+        logger.debug(f"{self.name}: Adding human behavior delay: {delay:.2f} seconds")
         await asyncio.sleep(delay)
 
         # Simulate scrolling behavior
@@ -273,9 +264,7 @@ class StealthManager:
         except Exception as e:
             logger.debug(f"{self.name}: Human behavior simulation failed: {e}")
 
-    async def handle_cloudflare_challenge_async(
-        self, page: Page, max_wait: int = 45
-    ):
+    async def handle_cloudflare_challenge_async(self, page: Page, max_wait: int = 45):
         """Enhanced Cloudflare challenge detection and handling."""
         logger.debug(f"{self.name}: Checking for Cloudflare challenge...")
 
@@ -342,9 +331,7 @@ class StealthManager:
             # Additional wait to ensure page is fully loaded
             await asyncio.sleep(random.uniform(2, 4))
 
-            logger.debug(
-                f"{self.name}: Cloudflare challenge passed successfully"
-            )
+            logger.debug(f"{self.name}: Cloudflare challenge passed successfully")
             return True
 
         except Exception as e:
@@ -910,9 +897,7 @@ class StealthManager:
         await element.click()
 
     async def human_mouse_move_async(self, page: Page):
-        await page.mouse.move(
-            random.randint(100, 800), random.randint(100, 600)
-        )
+        await page.mouse.move(random.randint(100, 800), random.randint(100, 600))
 
     async def human_scroll_async(self, page: Page):
         scroll_distance = random.randint(300, 800)