unrealon 1.1.6__py3-none-any.whl → 2.0.5__py3-none-any.whl

unrealon_browser/cli/browser_cli.py

@@ -13,11 +13,19 @@ from rich.panel import Panel
 from rich.table import Table
 
 # Use existing unrealon_browser API
-from unrealon_browser import BrowserManager, BrowserConfig, BrowserType, BrowserMode
+from unrealon_browser.core.browser_manager import BrowserManager
+from unrealon_browser.dto.models.config import BrowserConfig, BrowserType, BrowserMode
+
+from unrealon_core.config.urls import get_url_config
 
 console = Console()
 
 
+def _get_default_test_url() -> str:
+    """Get default test URL from configuration."""
+    return get_url_config().stealth_test_url
+
+
 @click.group()
 def browser():
     """🌐 Browser automation commands."""
@@ -46,9 +54,10 @@ def launch(parser, browser_type, headless, stealth, url):
 
     # Interactive mode if no URL provided
     if not url:
-        url = questionary.text("Enter URL to navigate:", default="https://bot.sannysoft.com").ask()
+        default_url = _get_default_test_url()
+        url = questionary.text("Enter URL to navigate:", default=default_url).ask()
         if not url:  # If user just pressed Enter with empty input
-            url = "https://bot.sannysoft.com"
+            url = default_url
 
     # Run browser session
     asyncio.run(_run_browser_session(parser, browser_type, headless, stealth, url))
@@ -56,7 +65,7 @@ def launch(parser, browser_type, headless, stealth, url):
 
 @browser.command()
 @click.option("--parser", default="default_parser", help="Parser name")
-@click.option("--url", default="https://bot.sannysoft.com", help="Test URL")
+@click.option("--url", default=None, help="Test URL")
 def stealth_test(parser, url):
     """🕵️ Test stealth effectiveness."""
     console.print("[bold blue]🕵️ Testing stealth capabilities...[/bold blue]")
@@ -109,9 +118,10 @@ def _interactive_launch():
     browser_type = questionary.select("Browser type:", choices=["chromium", "firefox", "webkit"]).ask()
     headless = questionary.confirm("Headless mode?", default=False).ask()
     stealth = questionary.select("Stealth level:", choices=["disabled", "basic", "advanced"]).ask()
-    url = questionary.text("URL to navigate:", default="https://bot.sannysoft.com").ask()
+    default_url = _get_default_test_url()
+    url = questionary.text("URL to navigate:", default=default_url).ask()
     if not url:
-        url = "https://bot.sannysoft.com"
+        url = default_url
 
     asyncio.run(_run_browser_session(parser, browser_type, headless, stealth, url))
 
@@ -119,7 +129,8 @@ def _interactive_launch():
 def _interactive_stealth_test():
     """Interactive stealth test."""
     parser = questionary.text("Parser name:", default="default_parser").ask()
-    url = questionary.text("Test URL:", default="https://bot.sannysoft.com").ask()
+    default_url = _get_default_test_url()
+    url = questionary.text("Test URL:", default=default_url).ask()
 
     asyncio.run(_run_stealth_test(parser, url))
 
@@ -157,7 +168,6 @@ async def _run_browser_session(parser: str, browser_type: str, headless: bool, s
             parser_name=parser,
             browser_type=BrowserType(browser_type.lower()),
             mode=BrowserMode.HEADLESS if headless else BrowserMode.HEADED,
-            # stealth_level removed - STEALTH ALWAYS ON!
         )
 
         # Use existing BrowserManager - no duplication!
@@ -194,7 +204,6 @@ async def _run_stealth_test(parser: str, url: str):
             parser_name=parser,
             browser_type=BrowserType.CHROMIUM,
             mode=BrowserMode.HEADLESS,
-            # stealth_level removed - STEALTH ALWAYS ON!
         )
 
         browser_manager = BrowserManager(config)

unrealon_browser/cli/interactive_mode.py

@@ -11,9 +11,16 @@ from rich.panel import Panel
 from rich.table import Table
 from typing import Dict, Any
 
+from unrealon_core.config.urls import get_url_config
+
 console = Console()
 
 
+def _get_default_test_url() -> str:
+    """Get default test URL from configuration."""
+    return get_url_config().stealth_test_url
+
+
 async def run_interactive_mode(parser: str, verbose: bool = False) -> None:
     """
     Run fully interactive browser management mode.
@@ -100,12 +107,13 @@ async def _interactive_browser_launch(parser: str, verbose: bool) -> None:
     # 🔥 STEALTH ALWAYS ON - NO CONFIG NEEDED!
     stealth_info = questionary.select(
         "Stealth is always enabled. Select stealth verification:",
-        choices=["None", "Test on bot.sannysoft.com", "Test on fingerprint.com"]
+        choices=["None", "Test on detection service", "Test on fingerprint.com"]
     ).ask()
     
+    default_url = _get_default_test_url()
     url = questionary.text(
         "Enter URL to navigate:",
-        default="https://bot.sannysoft.com"
+        default=default_url
     ).ask()
     
     # Show configuration summary
@@ -136,9 +144,10 @@ async def _interactive_stealth_test(parser: str, verbose: bool) -> None:
     """Interactive stealth testing."""
     console.print("\n[bold blue]🕵️ Stealth Testing[/bold blue]")
     
+    default_url = _get_default_test_url()
     test_url = questionary.text(
         "Enter test URL:",
-        default="https://bot.sannysoft.com"
+        default=default_url
     ).ask()
     
     test_levels = questionary.checkbox(
@@ -330,7 +339,7 @@ def _show_help_information() -> None:
 
 [yellow]Examples:[/yellow]
 [dim]cli-browser launch --parser my_parser --headless --url https://example.com
-cli-browser stealth-test --url https://bot.sannysoft.com  
+cli-browser stealth-test --url {_get_default_test_url()}  
 cli-browser workflow --parser scraper --url https://target-site.com[/dim]
 
 [yellow]For more information:[/yellow]

unrealon_browser/core/browser_manager.py

@@ -55,14 +55,16 @@ class BrowserManager:
         self._initialized = False
         self._statistics = BrowserManagerStatistics()
 
+        # Initialize logger bridge first
+        self.logger_bridge = create_browser_logger_bridge(session_id=self._generate_session_id(), parser_id=self.parser_id, enable_console=True)  # Use resolved parser_id
+        
         # Initialize managers
-        self.stealth_manager = StealthManager()
+        self.stealth_manager = StealthManager(logger_bridge=self.logger_bridge)
         # ✅ FIX: Don't create default managers here - they will be set by Browser service
         # This prevents duplicate directory creation with wrong paths
         self.profile_manager = None
         self.cookie_manager = None
         self.captcha_manager = CaptchaDetector()
-        self.logger_bridge = create_browser_logger_bridge(session_id=self._generate_session_id(), parser_id=self.parser_id, enable_console=True)  # Use resolved parser_id
         self.page_wait = PageWaitManager(None, self.logger_bridge)
 
         # Signal handlers for graceful shutdown
@@ -226,13 +228,13 @@ class BrowserManager:
 
             # Create page
             self._page = await self._context.new_page()
-            
+
             # Update page wait manager with new page
             self.page_wait.update_page(self._page)
 
             # 🔥 STEALTH ALWAYS APPLIED TO EVERY PAGE!
             stealth_success = await self.stealth_manager.apply_stealth(self._page)
-            self.logger_bridge.log_stealth_applied("ALWAYS_ON", stealth_success)
+            self.logger_bridge.log_stealth_applied(stealth_success)
 
             # 🔥 CRITICAL: If stealth fails, CLOSE BROWSER WITH ERROR!
             if not stealth_success:
@@ -302,34 +304,34 @@ class BrowserManager:
     async def _navigate_with_stealth_retry(self, url: str, wait_for: Optional[str] = None) -> Dict[str, Any]:
         """Navigate with stealth retry logic - universal for all sites"""
         self.logger_bridge.log_info(f"🥷 Stealth navigation to: {url}")
-        
+
         # Stealth warmup if enabled
         if self.config.stealth_warmup_enabled:
             self.logger_bridge.log_info(f"🧪 First visiting stealth test page: {self.config.stealth_test_url}")
             await self._navigate_basic(self.config.stealth_test_url, None)
             self.logger_bridge.log_info(f"⏳ Waiting {self.config.stealth_warmup_delay} seconds for stealth establishment...")
             await asyncio.sleep(self.config.stealth_warmup_delay)
-        
+
         # Now navigate to the actual target URL with retry logic
         max_retries = self.config.stealth_retry_attempts
         for attempt in range(max_retries + 1):
             if attempt > 0:
                 self.logger_bridge.log_info(f"🔄 Stealth retry attempt {attempt}/{max_retries}")
                 await asyncio.sleep(self.config.stealth_retry_delay)
-            
+
             result = await self._navigate_basic(url, wait_for)
-            
+
             # If successful, return result
             if result["success"]:
                 if attempt > 0:
                     self.logger_bridge.log_info(f"✅ Stealth retry successful on attempt {attempt + 1}")
                 return result
-            
+
             # If failed and we have retries left, continue
             if attempt < max_retries:
                 self.logger_bridge.log_info(f"⚠️ Navigation failed, will retry in {self.config.stealth_retry_delay} seconds...")
                 continue
-        
+
         # All retries failed
         self.logger_bridge.log_warning(f"❌ Stealth navigation failed after {max_retries + 1} attempts")
         return result
@@ -411,10 +413,6 @@ class BrowserManager:
                 "error": str(e),
             }
 
-    async def wait_for_page_ready_async(self, wait_type: str = "networkidle", timeout: int = 10000) -> bool:
-        """Wait for page to be ready for parsing (legacy method - use page_wait.* methods instead)"""
-        return await self.page_wait.wait_custom(wait_type, timeout)
-
     async def get_page_content_async(self) -> Optional[str]:
         """Get current page content"""
         if not self._page:
@@ -483,8 +481,23 @@ class BrowserManager:
         self.logger_bridge.print_statistics()
 
     async def test_stealth_async(self) -> Dict[str, Any]:
-        """Test stealth effectiveness on bot.sannysoft.com"""
-        return await self.stealth_manager.test_stealth_on_sannysoft(self)
+        """Test stealth effectiveness on detection service"""
+        self.logger_bridge.log_info("🔍 DEBUG: test_stealth_async called")
+        self.logger_bridge.log_info(f"🔍 DEBUG: stealth_manager={self.stealth_manager}")
+        try:
+            self.logger_bridge.log_info("🔍 DEBUG: About to call stealth_manager.test_stealth_effectiveness")
+            result = await self.stealth_manager.test_stealth_effectiveness(self)
+            self.logger_bridge.log_info(f"🔍 DEBUG: stealth_manager.test_stealth_effectiveness returned: {result}")
+            return result
+        except Exception as e:
+            self.logger_bridge.log_error(f"❌ DEBUG: Exception in test_stealth_async: {e}")
+            import traceback
+            self.logger_bridge.log_error(f"❌ DEBUG: Traceback: {traceback.format_exc()}")
+            return {
+                "success": False,
+                "error": str(e),
+                "skipped": False,
+            }
 
     async def detect_captcha_async(self) -> Dict[str, Any]:
         """Detect captcha on current page"""

unrealon_browser/dto/__init__.py

@@ -39,6 +39,18 @@ from .models.detection import (
     CookieMetadata,
 )
 
+# Bot detection models
+from .bot_detection import (
+    TestResult,
+    BotDetectionSummary,
+    BotDetectionResults,
+    BotDetectionError,
+    BotDetectionResponse,
+    BotTestResult,
+    BotSummary,
+    BotResults,
+)
+
 
 # Exports
 __all__ = [
@@ -62,4 +74,13 @@ __all__ = [
     "CaptchaDetection",
     "CaptchaDetectionResult",
     "CookieMetadata",
+    # Bot detection models
+    "TestResult",
+    "BotDetectionSummary",
+    "BotDetectionResults",
+    "BotDetectionError",
+    "BotDetectionResponse",
+    "BotTestResult",
+    "BotSummary",
+    "BotResults",
 ]

unrealon_browser/dto/bot_detection.py

@@ -0,0 +1,175 @@
+"""
+Bot Detection Results DTOs
+
+Pydantic models for bot detection results from the frontend scanner.
+Based on TypeScript interfaces from botDetection.ts
+"""
+
+from typing import Any, Dict, List, Literal, Optional, Union
+from pydantic import BaseModel, Field
+
+
+class TestResult(BaseModel):
+    """Individual test result from bot detection"""
+    
+    name: str = Field(..., description="Name of the test")
+    status: Literal["passed", "failed", "warn"] = Field(..., description="Test status")
+    value: Any = Field(..., description="Test result value (can be any type)")
+    description: str = Field(..., description="Human-readable test description")
+    score: int = Field(..., ge=0, le=100, description="Suspicion score (0-100, higher = more suspicious)")
+
+
+class BotDetectionSummary(BaseModel):
+    """Summary statistics for bot detection tests"""
+    
+    passed: int = Field(..., ge=0, description="Number of tests that passed")
+    failed: int = Field(..., ge=0, description="Number of tests that failed")
+    warnings: int = Field(..., ge=0, description="Number of tests with warnings")
+    total: int = Field(..., ge=0, description="Total number of tests run")
+
+
+class BotDetectionResults(BaseModel):
+    """Complete bot detection results from frontend scanner"""
+    
+    tests: List[TestResult] = Field(..., description="List of individual test results")
+    overall_score: int = Field(..., ge=0, le=100, description="Overall suspicion score (0-100)", alias="overallScore")
+    is_bot: bool = Field(..., description="Whether the browser is detected as a bot", alias="isBot")
+    confidence: Literal["low", "medium", "high"] = Field(..., description="Confidence level of detection")
+    summary: BotDetectionSummary = Field(..., description="Summary statistics")
+    
+    model_config = {
+        "json_encoders": {
+            # Handle Any type serialization
+            Any: lambda v: v
+        },
+        # Allow population by field name or alias
+        "populate_by_name": True
+    }
+        
+    @property
+    def failed_tests(self) -> List[TestResult]:
+        """Get list of failed tests"""
+        return [test for test in self.tests if test.status == "failed"]
+    
+    @property
+    def critical_failures(self) -> List[TestResult]:
+        """Get list of critical failures that strongly indicate bot detection"""
+        critical_test_names = [
+            "BotD Detection",
+            "WebDriver Property", 
+            "Headless Chrome",
+            "Chrome Object Consistency",
+            "Navigator WebDriver",
+            "Advanced Automation Detection"
+        ]
+        
+        return [
+            test for test in self.failed_tests 
+            if any(critical in test.name for critical in critical_test_names)
+        ]
+    
+    @property
+    def warning_tests(self) -> List[TestResult]:
+        """Get list of tests with warnings"""
+        return [test for test in self.tests if test.status == "warn"]
+    
+    @property
+    def passed_tests(self) -> List[TestResult]:
+        """Get list of passed tests"""
+        return [test for test in self.tests if test.status == "passed"]
+    
+    def get_effectiveness_rating(self) -> str:
+        """Get stealth effectiveness rating based on results"""
+        if self.overall_score <= 10 and not self.is_bot:
+            return "excellent"
+        elif self.overall_score <= 25 and not self.is_bot:
+            return "good"
+        elif self.overall_score <= 50:
+            return "moderate"
+        else:
+            return "poor"
+    
+    def get_recommendations(self) -> List[str]:
+        """Generate recommendations based on test results"""
+        recommendations = []
+        
+        failed_test_names = [test.name for test in self.failed_tests]
+        
+        if any("BotD" in name for name in failed_test_names):
+            recommendations.append("Consider using headed mode instead of headless for better BotD bypass")
+            
+        if any("WebDriver" in name for name in failed_test_names):
+            recommendations.append("Enhance webdriver property removal techniques")
+            
+        if any("Chrome Object" in name for name in failed_test_names):
+            recommendations.append("Improve window.chrome object spoofing")
+            
+        if any("Plugin" in name for name in failed_test_names):
+            recommendations.append("Add more realistic browser plugin simulation")
+            
+        if any("WebGL" in name for name in failed_test_names):
+            recommendations.append("Enhance WebGL vendor/renderer spoofing")
+            
+        if self.overall_score > 30:
+            recommendations.append("Consider using undetected-chromedriver or NoDriver for better results")
+            
+        if len(self.critical_failures) > 2:
+            recommendations.append("Multiple critical failures detected - review stealth configuration")
+        
+        return recommendations
+
+
+class BotDetectionError(BaseModel):
+    """Error information when bot detection fails"""
+    
+    error: str = Field(..., description="Error message")
+    error_type: str = Field(default="detection_error", description="Type of error")
+    timestamp: Optional[str] = Field(None, description="When the error occurred")
+    
+    
+class BotDetectionResponse(BaseModel):
+    """Response wrapper for bot detection results"""
+    
+    success: bool = Field(..., description="Whether detection was successful")
+    results: Optional[BotDetectionResults] = Field(None, description="Detection results if successful")
+    error: Optional[BotDetectionError] = Field(None, description="Error information if failed")
+    scanner_url: Optional[str] = Field(None, description="URL of the scanner used")
+    method: Optional[str] = Field(None, description="Stealth method used")
+    timestamp: Optional[str] = Field(None, description="When the detection was performed")
+    
+    @classmethod
+    def success_response(
+        cls, 
+        results: BotDetectionResults, 
+        scanner_url: Optional[str] = None,
+        method: Optional[str] = None
+    ) -> "BotDetectionResponse":
+        """Create a successful response"""
+        return cls(
+            success=True,
+            results=results,
+            scanner_url=scanner_url,
+            method=method
+        )
+    
+    @classmethod
+    def error_response(
+        cls, 
+        error_message: str, 
+        error_type: str = "detection_error",
+        scanner_url: Optional[str] = None,
+        method: Optional[str] = None
+    ) -> "BotDetectionResponse":
+        """Create an error response"""
+        return cls(
+            success=False,
+            error=BotDetectionError(error=error_message, error_type=error_type),
+            scanner_url=scanner_url,
+            method=method
+        )
+
+
+# Type aliases for convenience
+BotTestResult = TestResult
+BotSummary = BotDetectionSummary
+BotResults = BotDetectionResults

unrealon_browser/dto/models/config.py

@@ -8,6 +8,13 @@ from typing import Optional
 from pydantic import BaseModel, Field, ConfigDict
 from .enums import BrowserType, BrowserMode
 
+from unrealon_core.config.urls import get_url_config
+
+
+def _get_default_stealth_url() -> str:
+    """Get default stealth test URL based on environment configuration."""
+    return get_url_config().stealth_test_url
+
 
 class BrowserConfig(BaseModel):
     """Simplified browser configuration."""
@@ -30,11 +37,10 @@ class BrowserConfig(BaseModel):
 
     # Performance
     disable_images: bool = Field(default=False)
-    enable_stealth_check: bool = Field(default=False)
     
     # Stealth settings
     stealth_warmup_enabled: bool = Field(default=True, description="Enable stealth warmup before target navigation")
-    stealth_test_url: str = Field(default="https://bot.sannysoft.com", description="URL for stealth warmup")
+    stealth_test_url: str = Field(default_factory=_get_default_stealth_url, description="URL for stealth warmup")
     stealth_warmup_delay: float = Field(default=3.0, description="Delay in seconds after stealth warmup")
     stealth_retry_attempts: int = Field(default=2, description="Maximum retry attempts for failed navigation")
-    stealth_retry_delay: float = Field(default=3.0, description="Delay between retry attempts")
+    stealth_retry_delay: float = Field(default=3.0, description="Delay between retry attempts")

unrealon_browser/managers/__init__.py

@@ -2,7 +2,7 @@
 Browser Managers Package - Specialized management components
 """
 
-from .stealth import StealthManager
+from ..stealth import StealthManager
 from .profile import ProfileManager
 from .logger_bridge import BrowserLoggerBridge, create_browser_logger_bridge
 from .cookies import CookieManager

unrealon_browser/managers/logger_bridge.py

@@ -141,7 +141,6 @@ class BrowserLoggerBridge:
             session_id=metadata.session_id,
             parser_name=metadata.parser_name,
             browser_type=metadata.browser_type or "unknown",
-            stealth_level="ALWAYS_ON",  # 🔥 STEALTH ALWAYS ON!
             proxy_host=getattr(metadata.proxy, "host", None) if metadata.proxy else None,
             proxy_port=getattr(metadata.proxy, "port", None) if metadata.proxy else None,
         )
@@ -168,20 +167,18 @@ class BrowserLoggerBridge:
             navigation_type="browser_navigation",
         )
 
-    def log_stealth_applied(self, stealth_level: str = "ALWAYS_ON", success: bool = True) -> None:
+    def log_stealth_applied(self, success: bool = True) -> None:
         """Log stealth application - 🔥 STEALTH ALWAYS ON!"""
         self._browser_events["stealth_applied"] += 1
 
         if success:
             self._log_info(
                 "Stealth measures applied: ALWAYS_ON",
-                stealth_level="ALWAYS_ON",
                 stealth_success=True,
             )
         else:
             self._log_warning(
                 "Stealth application failed: ALWAYS_ON",
-                stealth_level="ALWAYS_ON",
                 stealth_success=False,
             )
 

unrealon_browser/stealth/__init__.py

@@ -0,0 +1,27 @@
+"""
+UnrealOn Stealth Package - Advanced anti-detection system
+
+Modular bot detection bypass system:
+- PlaywrightStealth: playwright-stealth integration
+- UndetectedChrome: undetected-chromedriver support  
+- NoDriverStealth: NoDriver integration
+- BypassTechniques: advanced BotD bypass techniques
+- StealthManager: main coordinator for all techniques
+- ScannerTester: real-world testing through UnrealOn scanner
+"""
+
+from .manager import StealthManager
+from .playwright_stealth import PlaywrightStealth
+from .undetected_chrome import UndetectedChrome
+from .nodriver_stealth import NoDriverStealth
+from .bypass_techniques import BypassTechniques
+from .scanner_tester import ScannerTester
+
+__all__ = [
+    "StealthManager",
+    "PlaywrightStealth", 
+    "UndetectedChrome",
+    "NoDriverStealth",
+    "BypassTechniques",
+    "ScannerTester"
+]