scrapling 0.2.99__py3-none-any.whl → 0.3__py3-none-any.whl

scrapling/engines/_browsers/_page.py

@@ -0,0 +1,93 @@
+from threading import RLock
+from dataclasses import dataclass
+
+from playwright.sync_api import Page as SyncPage
+from playwright.async_api import Page as AsyncPage
+
+from scrapling.core._types import Optional, List, Literal
+
+PageState = Literal["ready", "busy", "error"]  # States that a page can be in
+
+
+@dataclass
+class PageInfo:
+    """Information about the page and its current state"""
+
+    __slots__ = ("page", "state", "url")
+    page: SyncPage | AsyncPage
+    state: PageState
+    url: Optional[str]
+
+    def mark_busy(self, url: str = ""):
+        """Mark the page as busy"""
+        self.state = "busy"
+        self.url = url
+
+    def mark_ready(self):
+        """Mark the page as ready for new requests"""
+        self.state = "ready"
+        self.url = ""
+
+    def mark_error(self):
+        """Mark the page as having an error"""
+        self.state = "error"
+
+    def __repr__(self):
+        return f'Page(URL="{self.url!r}", state={self.state!r})'
+
+    def __eq__(self, other_page):
+        """Comparing this page to another page object."""
+        if other_page.__class__ is not self.__class__:
+            return NotImplemented
+        return self.page == other_page.page
+
+
+class PagePool:
+    """Manages a pool of browser pages/tabs with state tracking"""
+
+    __slots__ = ("max_pages", "pages", "_lock")
+
+    def __init__(self, max_pages: int = 5):
+        self.max_pages = max_pages
+        self.pages: List[PageInfo] = []
+        self._lock = RLock()
+
+    def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:
+        """Add a new page to the pool"""
+        with self._lock:
+            if len(self.pages) >= self.max_pages:
+                raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")
+
+            page_info = PageInfo(page, "ready", "")
+            self.pages.append(page_info)
+            return page_info
+
+    def get_ready_page(self) -> Optional[PageInfo]:
+        """Get a page that's ready for use"""
+        with self._lock:
+            for page_info in self.pages:
+                if page_info.state == "ready":
+                    return page_info
+            return None
+
+    @property
+    def pages_count(self) -> int:
+        """Get the total number of pages"""
+        return len(self.pages)
+
+    @property
+    def ready_count(self) -> int:
+        """Get the number of ready pages"""
+        with self._lock:
+            return sum(1 for p in self.pages if p.state == "ready")
+
+    @property
+    def busy_count(self) -> int:
+        """Get the number of busy pages"""
+        with self._lock:
+            return sum(1 for p in self.pages if p.state == "busy")
+
+    def cleanup_error_pages(self):
+        """Remove pages in error state"""
+        with self._lock:
+            self.pages = [p for p in self.pages if p.state != "error"]

scrapling/engines/_browsers/_validators.py

@@ -0,0 +1,150 @@
+from msgspec import Struct, convert, ValidationError
+from urllib.parse import urlparse
+from pathlib import Path
+
+from scrapling.core._types import (
+    Optional,
+    Dict,
+    Callable,
+    List,
+    SelectorWaitStates,
+)
+from scrapling.engines.toolbelt import construct_proxy_dict
+
+
+class PlaywrightConfig(Struct, kw_only=True, frozen=False):
+    """Configuration struct for validation"""
+
+    max_pages: int = 1
+    cdp_url: Optional[str] = None
+    headless: bool = True
+    google_search: bool = True
+    hide_canvas: bool = False
+    disable_webgl: bool = False
+    real_chrome: bool = False
+    stealth: bool = False
+    wait: int | float = 0
+    page_action: Optional[Callable] = None
+    proxy: Optional[str | Dict[str, str]] = (
+        None  # The default value for proxy in Playwright's source is `None`
+    )
+    locale: str = "en-US"
+    extra_headers: Optional[Dict[str, str]] = None
+    useragent: Optional[str] = None
+    timeout: int | float = 30000
+    disable_resources: bool = False
+    wait_selector: Optional[str] = None
+    cookies: Optional[List[Dict]] = None
+    network_idle: bool = False
+    wait_selector_state: SelectorWaitStates = "attached"
+    selector_config: Optional[Dict] = None
+
+    def __post_init__(self):
+        """Custom validation after msgspec validation"""
+        if self.max_pages < 1 or self.max_pages > 50:
+            raise ValueError("max_pages must be between 1 and 50")
+        if self.timeout < 0:
+            raise ValueError("timeout must be >= 0")
+        if self.page_action is not None and not callable(self.page_action):
+            raise TypeError(
+                f"page_action must be callable, got {type(self.page_action).__name__}"
+            )
+        if self.proxy:
+            self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
+        if self.cdp_url:
+            self.__validate_cdp(self.cdp_url)
+        if not self.cookies:
+            self.cookies = []
+        if not self.selector_config:
+            self.selector_config = {}
+
+    @staticmethod
+    def __validate_cdp(cdp_url):
+        try:
+            # Check the scheme
+            if not cdp_url.startswith(("ws://", "wss://")):
+                raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
+
+            # Validate hostname and port
+            if not urlparse(cdp_url).netloc:
+                raise ValueError("Invalid hostname for the CDP URL")
+
+        except AttributeError as e:
+            raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
+
+        except Exception as e:
+            raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
+
+
+class CamoufoxConfig(Struct, kw_only=True, frozen=False):
+    """Configuration struct for validation"""
+
+    max_pages: int = 1
+    headless: bool = True  # noqa: F821
+    block_images: bool = False
+    disable_resources: bool = False
+    block_webrtc: bool = False
+    allow_webgl: bool = True
+    network_idle: bool = False
+    humanize: bool | float = True
+    solve_cloudflare: bool = False
+    wait: int | float = 0
+    timeout: int | float = 30000
+    page_action: Optional[Callable] = None
+    wait_selector: Optional[str] = None
+    addons: Optional[List[str]] = None
+    wait_selector_state: SelectorWaitStates = "attached"
+    cookies: Optional[List[Dict]] = None
+    google_search: bool = True
+    extra_headers: Optional[Dict[str, str]] = None
+    proxy: Optional[str | Dict[str, str]] = (
+        None  # The default value for proxy in Playwright's source is `None`
+    )
+    os_randomize: bool = False
+    disable_ads: bool = False
+    geoip: bool = False
+    selector_config: Optional[Dict] = None
+    additional_args: Optional[Dict] = None
+
+    def __post_init__(self):
+        """Custom validation after msgspec validation"""
+        if self.max_pages < 1 or self.max_pages > 50:
+            raise ValueError("max_pages must be between 1 and 50")
+        if self.timeout < 0:
+            raise ValueError("timeout must be >= 0")
+        if self.page_action is not None and not callable(self.page_action):
+            raise TypeError(
+                f"page_action must be callable, got {type(self.page_action).__name__}"
+            )
+        if self.proxy:
+            self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
+
+        if not self.addons:
+            self.addons = []
+        else:
+            for addon in self.addons:
+                addon_path = Path(addon)
+                if not addon_path.exists():
+                    raise FileNotFoundError(f"Addon's path not found: {addon}")
+                elif not addon_path.is_dir():
+                    raise ValueError(
+                        f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
+                    )
+
+        if not self.cookies:
+            self.cookies = []
+        if self.solve_cloudflare and self.timeout < 60_000:
+            self.timeout = 60_000
+        if not self.selector_config:
+            self.selector_config = {}
+        if not self.additional_args:
+            self.additional_args = {}
+
+
+def validate(params, model):
+    try:
+        config = convert(params, model)
+    except ValidationError as e:
+        raise TypeError(f"Invalid argument type: {e}")
+
+    return config

scrapling/engines/constants.py

@@ -1,92 +1,108 @@
 # Disable loading these resources for speed
 DEFAULT_DISABLED_RESOURCES = {
-    'font',
-    'image',
-    'media',
-    'beacon',
-    'object',
-    'imageset',
-    'texttrack',
-    'websocket',
-    'csp_report',
-    'stylesheet',
+    "font",
+    "image",
+    "media",
+    "beacon",
+    "object",
+    "imageset",
+    "texttrack",
+    "websocket",
+    "csp_report",
+    "stylesheet",
 }
 
+HARMFUL_DEFAULT_ARGS = (
+    # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
+    "--enable-automation",
+    "--disable-popup-blocking",
+    # '--disable-component-update',
+    # '--disable-default-apps',
+    # '--disable-extensions',
+)
+
+DEFAULT_FLAGS = (
+    # Speed up chromium browsers by default
+    "--no-pings",
+    "--no-first-run",
+    "--disable-infobars",
+    "--disable-breakpad",
+    "--no-service-autorun",
+    "--homepage=about:blank",
+    "--password-store=basic",
+    "--no-default-browser-check",
+    "--disable-session-crashed-bubble",
+    "--disable-search-engine-choice-screen",
+)
+
 DEFAULT_STEALTH_FLAGS = (
     # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
     # Generally this will make the browser faster and less detectable
-    '--no-pings',
-    '--incognito',
-    '--test-type',
-    '--lang=en-US',
-    '--mute-audio',
-    '--no-first-run',
-    '--disable-sync',
-    '--hide-scrollbars',
-    '--disable-logging',
-    '--start-maximized',  # For headless check bypass
-    '--enable-async-dns',
-    '--disable-breakpad',
-    '--disable-infobars',
-    '--accept-lang=en-US',
-    '--use-mock-keychain',
-    '--disable-translate',
-    '--disable-extensions',
-    '--disable-voice-input',
-    '--window-position=0,0',
-    '--disable-wake-on-wifi',
-    '--ignore-gpu-blocklist',
-    '--enable-tcp-fast-open',
-    '--enable-web-bluetooth',
-    '--disable-hang-monitor',
-    '--password-store=basic',
-    '--disable-cloud-import',
-    '--disable-default-apps',
-    '--disable-print-preview',
-    '--disable-dev-shm-usage',
+    "--incognito",
+    "--test-type",
+    "--lang=en-US",
+    "--mute-audio",
+    "--disable-sync",
+    "--hide-scrollbars",
+    "--disable-logging",
+    "--start-maximized",  # For headless check bypass
+    "--enable-async-dns",
+    "--accept-lang=en-US",
+    "--use-mock-keychain",
+    "--disable-translate",
+    "--disable-extensions",
+    "--disable-voice-input",
+    "--window-position=0,0",
+    "--disable-wake-on-wifi",
+    "--ignore-gpu-blocklist",
+    "--enable-tcp-fast-open",
+    "--enable-web-bluetooth",
+    "--disable-hang-monitor",
+    "--disable-cloud-import",
+    "--disable-default-apps",
+    "--disable-print-preview",
+    "--disable-dev-shm-usage",
     # '--disable-popup-blocking',
-    '--metrics-recording-only',
-    '--disable-crash-reporter',
-    '--disable-partial-raster',
-    '--disable-gesture-typing',
-    '--disable-checker-imaging',
-    '--disable-prompt-on-repost',
-    '--force-color-profile=srgb',
-    '--font-render-hinting=none',
-    '--no-default-browser-check',
-    '--aggressive-cache-discard',
-    '--disable-component-update',
-    '--disable-cookie-encryption',
-    '--disable-domain-reliability',
-    '--disable-threaded-animation',
-    '--disable-threaded-scrolling',
-    # '--disable-reading-from-canvas',  # For Firefox
-    '--enable-simple-cache-backend',
-    '--disable-background-networking',
-    '--disable-session-crashed-bubble',
-    '--enable-surface-synchronization',
-    '--disable-image-animation-resync',
-    '--disable-renderer-backgrounding',
-    '--disable-ipc-flooding-protection',
-    '--prerender-from-omnibox=disabled',
-    '--safebrowsing-disable-auto-update',
-    '--disable-offer-upload-credit-cards',
-    '--disable-features=site-per-process',
-    '--disable-background-timer-throttling',
-    '--disable-new-content-rendering-timeout',
-    '--run-all-compositor-stages-before-draw',
-    '--disable-client-side-phishing-detection',
-    '--disable-backgrounding-occluded-windows',
-    '--disable-layer-tree-host-memory-pressure',
-    '--autoplay-policy=no-user-gesture-required',
-    '--disable-offer-store-unmasked-wallet-cards',
-    '--disable-blink-features=AutomationControlled',
-    '--webrtc-ip-handling-policy=disable_non_proxied_udp',
-    '--disable-component-extensions-with-background-pages',
-    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
-    '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
-    '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
-    '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
+    "--metrics-recording-only",
+    "--disable-crash-reporter",
+    "--disable-partial-raster",
+    "--disable-gesture-typing",
+    "--disable-checker-imaging",
+    "--disable-prompt-on-repost",
+    "--force-color-profile=srgb",
+    "--font-render-hinting=none",
+    "--aggressive-cache-discard",
+    "--disable-component-update",
+    "--disable-cookie-encryption",
+    "--disable-domain-reliability",
+    "--disable-threaded-animation",
+    "--disable-threaded-scrolling",
+    # '--disable-reading-from-canvas', # For Firefox
+    "--enable-simple-cache-backend",
+    "--disable-background-networking",
+    "--enable-surface-synchronization",
+    "--disable-image-animation-resync",
+    "--disable-renderer-backgrounding",
+    "--disable-ipc-flooding-protection",
+    "--prerender-from-omnibox=disabled",
+    "--safebrowsing-disable-auto-update",
+    "--disable-offer-upload-credit-cards",
+    "--disable-features=site-per-process",
+    "--disable-background-timer-throttling",
+    "--disable-new-content-rendering-timeout",
+    "--run-all-compositor-stages-before-draw",
+    "--disable-client-side-phishing-detection",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-layer-tree-host-memory-pressure",
+    "--autoplay-policy=no-user-gesture-required",
+    "--disable-offer-store-unmasked-wallet-cards",
+    "--disable-blink-features=AutomationControlled",
+    "--webrtc-ip-handling-policy=disable_non_proxied_udp",
+    "--disable-component-extensions-with-background-pages",
+    "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
+    "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance",
+    "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
+    "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
 )
 
 # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
@@ -95,13 +111,10 @@ NSTBROWSER_DEFAULT_QUERY = {
     "headless": True,
     "autoClose": True,
     "fingerprint": {
-        "flags": {
-            "timezone": "BasedOnIp",
-            "screen": "Custom"
-        },
-        "platform": 'linux',  # support: windows, mac, linux
-        "kernel": 'chromium',  # only support: chromium
-        "kernelMilestone": '128',
+        "flags": {"timezone": "BasedOnIp", "screen": "Custom"},
+        "platform": "linux",  # support: windows, mac, linux
+        "kernel": "chromium",  # only support: chromium
+        "kernelMilestone": "128",
         "hardwareConcurrency": 8,
         "deviceMemory": 8,
     },