scrapling 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

scrapling/engines/_browsers/_base.py

@@ -7,22 +7,16 @@ from playwright.async_api import (
     BrowserContext as AsyncBrowserContext,
     Playwright as AsyncPlaywright,
 )
-from camoufox.utils import (
-    launch_options as generate_launch_options,
-    installed_verstr as camoufox_version,
-)
+from camoufox.pkgman import installed_verstr as camoufox_version
+from camoufox.utils import launch_options as generate_launch_options
 
-from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
-from scrapling.core._types import (
-    Any,
-    Dict,
-    Optional,
-)
 from ._page import PageInfo, PagePool
-from ._config_tools import _compiled_stealth_scripts
-from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.parser import Selector
+from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING
 from scrapling.engines.toolbelt.fingerprints import get_os_name
 from ._validators import validate, PlaywrightConfig, CamoufoxConfig
+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
 
 __ff_version_str__ = camoufox_version().split(".", 1)[0]
 
@@ -45,6 +39,7 @@ class SyncSession:
         """Get a new page to use"""
 
         # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc.
+        assert self.context is not None, "Browser context not initialized"
         page = self.context.new_page()
         page.set_default_navigation_timeout(timeout)
         page.set_default_timeout(timeout)
@@ -69,11 +64,14 @@ class SyncSession:
         }
 
 
-class AsyncSession(SyncSession):
+class AsyncSession:
     def __init__(self, max_pages: int = 1):
-        super().__init__(max_pages)
+        self.max_pages = max_pages
+        self.page_pool = PagePool(max_pages)
+        self._max_wait_for_page = 60
         self.playwright: Optional[AsyncPlaywright] = None
         self.context: Optional[AsyncBrowserContext] = None
+        self._closed = False
         self._lock = Lock()
 
     async def _get_page(
@@ -83,6 +81,9 @@ class AsyncSession(SyncSession):
         disable_resources: bool,
     ) -> PageInfo:  # pragma: no cover
         """Get a new page to use"""
+        if TYPE_CHECKING:
+            assert self.context is not None, "Browser context not initialized"
+
         async with self._lock:
             # If we're at max capacity after cleanup, wait for busy pages to finish
             if self.page_pool.pages_count >= self.max_pages:
@@ -96,6 +97,7 @@ class AsyncSession(SyncSession):
                         f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period"
                     )
 
+            assert self.context is not None, "Browser context not initialized"
             page = await self.context.new_page()
             page.set_default_navigation_timeout(timeout)
             page.set_default_timeout(timeout)
@@ -111,6 +113,14 @@ class AsyncSession(SyncSession):
 
             return self.page_pool.add_page(page)
 
+    def get_pool_stats(self) -> Dict[str, int]:
+        """Get statistics about the current page pool"""
+        return {
+            "total_pages": self.page_pool.pages_count,
+            "busy_pages": self.page_pool.busy_count,
+            "max_pages": self.max_pages,
+        }
+
 
 class DynamicSessionMixin:
     def __validate__(self, **params):
@@ -138,11 +148,16 @@ class DynamicSessionMixin:
         self.init_script = config.init_script
         self.wait_selector_state = config.wait_selector_state
         self.selector_config = config.selector_config
+        self.additional_args = config.additional_args
         self.page_action = config.page_action
-        self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
+        self.user_data_dir = config.user_data_dir
+        self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
         self.__initiate_browser_options__()
 
     def __initiate_browser_options__(self):
+        if TYPE_CHECKING:
+            assert isinstance(self.proxy, tuple)
+
         if not self.cdp_url:
             # `launch_options` is used with persistent context
             self.launch_options = dict(
@@ -160,6 +175,8 @@ class DynamicSessionMixin:
             )
             self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
             self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
+            self.launch_options["user_data_dir"] = self.user_data_dir
+            self.launch_options.update(cast(Dict, self.additional_args))
             self.context_options = dict()
         else:
             # while `context_options` is left to be used when cdp mode is enabled
@@ -175,11 +192,12 @@ class DynamicSessionMixin:
             )
             self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])
             self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
+            self.context_options.update(cast(Dict, self.additional_args))
 
 
 class StealthySessionMixin:
     def __validate__(self, **params):
-        config = validate(params, model=CamoufoxConfig)
+        config: CamoufoxConfig = validate(params, model=CamoufoxConfig)
 
         self.max_pages = config.max_pages
         self.headless = config.headless
@@ -208,15 +226,16 @@ class StealthySessionMixin:
         self.selector_config = config.selector_config
         self.additional_args = config.additional_args
         self.page_action = config.page_action
-        self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
+        self.user_data_dir = config.user_data_dir
+        self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
         self.__initiate_browser_options__()
 
     def __initiate_browser_options__(self):
         """Initiate browser options."""
-        self.launch_options = generate_launch_options(
+        self.launch_options: Dict[str, Any] = generate_launch_options(
             **{
                 "geoip": self.geoip,
-                "proxy": dict(self.proxy) if self.proxy else self.proxy,
+                "proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy,
                 "addons": self.addons,
                 "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
                 "headless": self.headless,
@@ -226,7 +245,7 @@ class StealthySessionMixin:
                 "block_webrtc": self.block_webrtc,
                 "block_images": self.block_images,  # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
                 "os": None if self.os_randomize else get_os_name(),
-                "user_data_dir": "",
+                "user_data_dir": self.user_data_dir,
                 "ff_version": __ff_version_str__,
                 "firefox_user_prefs": {
                     # This is what enabling `enable_cache` does internally, so we do it from here instead
@@ -236,7 +255,7 @@ class StealthySessionMixin:
                     "browser.cache.disk_cache_ssl": True,
                     "browser.cache.disk.smart_size.enabled": True,
                 },
-                **self.additional_args,
+                **cast(Dict, self.additional_args),
             }
         )
 
@@ -268,4 +287,9 @@ class StealthySessionMixin:
             if f"cType: '{ctype}'" in page_content:
                 return ctype
 
+        # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)
+        selector = Selector(content=page_content)
+        if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):
+            return "embedded"
+
         return None

scrapling/engines/_browsers/_camoufox.py

@@ -1,3 +1,4 @@
+from random import randint
 from re import compile as re_compile
 
 from playwright.sync_api import (
@@ -20,10 +21,12 @@ from ._validators import validate_fetch as _validate
 from ._base import SyncSession, AsyncSession, StealthySessionMixin
 from scrapling.core.utils import log
 from scrapling.core._types import (
+    Any,
     Dict,
     List,
     Optional,
     Callable,
+    TYPE_CHECKING,
     SelectorWaitStates,
 )
 from scrapling.engines.toolbelt.convertor import (
@@ -33,7 +36,7 @@ from scrapling.engines.toolbelt.convertor import (
 from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
 
 __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
-_UNSET = object()
+_UNSET: Any = object()
 
 
 class StealthySession(StealthySessionMixin, SyncSession):
@@ -101,6 +104,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         os_randomize: bool = False,
         disable_ads: bool = False,
         geoip: bool = False,
+        user_data_dir: str = "",
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -116,7 +120,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
@@ -133,6 +137,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -156,6 +161,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             block_images=block_images,
             block_webrtc=block_webrtc,
             os_randomize=os_randomize,
+            user_data_dir=user_data_dir,
             wait_selector=wait_selector,
             google_search=google_search,
             extra_headers=extra_headers,
@@ -170,9 +176,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
     def __create__(self):
         """Create a browser for this instance and context."""
         self.playwright = sync_playwright().start()
-        self.context = self.playwright.firefox.launch_persistent_context(  # pragma: no cover
-            **self.launch_options
-        )
+        self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
 
         if self.init_script:  # pragma: no cover
             self.context.add_init_script(path=self.init_script)
@@ -203,9 +207,9 @@ class StealthySession(StealthySessionMixin, SyncSession):
         self._closed = True
 
     @staticmethod
-    def _get_page_content(page: Page) -> str | None:
+    def _get_page_content(page: Page) -> str:
         """
-        A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
+        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
         :param page: The page to extract content from.
         :return:
         """
@@ -215,6 +219,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             except PlaywrightError:
                 page.wait_for_timeout(1000)
                 continue
+        return ""  # pyright: ignore
 
     def _solve_cloudflare(self, page: Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
@@ -222,6 +227,10 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param page: The targeted page
         :return:
         """
+        try:
+            page.wait_for_load_state("networkidle", timeout=5000)
+        except PlaywrightError:
+            pass
         challenge_type = self._detect_cloudflare(self._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
@@ -237,26 +246,42 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 return
 
             else:
-                while "Verifying you are human." in self._get_page_content(page):
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
-                    page.wait_for_timeout(500)
-
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+                if challenge_type != "embedded":
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in self._get_page_content(page):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        page.wait_for_timeout(500)
+
+                outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
-                if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
-                    return
+                if iframe is not None:
+                    iframe.wait_for_load_state(state="domcontentloaded")
+                    iframe.wait_for_load_state("networkidle")
+
+                    if challenge_type != "embedded":
+                        while not iframe.frame_element().is_visible():
+                            # Double-checking that the iframe is loaded
+                            page.wait_for_timeout(500)
+                    outer_box: Any = iframe.frame_element().bounding_box()
 
-                while not iframe.frame_element().is_visible():
-                    # Double-checking that the iframe is loaded
-                    page.wait_for_timeout(500)
+                if not iframe or not outer_box:
+                    outer_box: Any = page.locator(box_selector).last.bounding_box()
 
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = page.locator(".main-content p+div>div>div").bounding_box()
-                captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+                captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                page.locator(".zone-name-title").wait_for(state="hidden")
+                page.wait_for_load_state("networkidle")
+                if iframe is not None:
+                    # Wait for the frame to be removed from the page
+                    while iframe in page.frames:
+                        page.wait_for_timeout(100)
+                if challenge_type != "embedded":
+                    page.locator(box_selector).last.wait_for(state="detached")
+                    page.locator(".zone-name-title").wait_for(state="hidden")
+                page.wait_for_load_state(state="load")
                 page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
@@ -293,7 +318,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
@@ -328,6 +353,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             if (
                 finished_response.request.resource_type == "document"
                 and finished_response.request.is_navigation_request()
+                and finished_response.request.frame == page_info.page.main_frame
             ):
                 final_response = finished_response
 
@@ -380,7 +406,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 page_info.page, first_response, final_response, params.selector_config
             )
 
-            # Close the page, to free up resources
+            # Close the page to free up resources
             page_info.page.close()
             self.page_pool.pages.remove(page_info)
 
@@ -420,6 +446,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         os_randomize: bool = False,
         disable_ads: bool = False,
         geoip: bool = False,
+        user_data_dir: str = "",
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -435,7 +462,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
@@ -453,6 +480,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
+        :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -478,6 +506,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             wait_selector=wait_selector,
             google_search=google_search,
             extra_headers=extra_headers,
+            user_data_dir=user_data_dir,
             additional_args=additional_args,
             selector_config=selector_config,
             solve_cloudflare=solve_cloudflare,
@@ -497,7 +526,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             await self.context.add_init_script(path=self.init_script)
 
         if self.cookies:
-            await self.context.add_cookies(self.cookies)
+            await self.context.add_cookies(self.cookies)  # pyright: ignore [reportArgumentType]
 
     async def __aenter__(self):
         await self.__create__()
@@ -513,18 +542,18 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
 
         if self.context:
             await self.context.close()
-            self.context = None
+            self.context = None  # pyright: ignore
 
         if self.playwright:
             await self.playwright.stop()
-            self.playwright = None
+            self.playwright = None  # pyright: ignore
 
         self._closed = True
 
     @staticmethod
-    async def _get_page_content(page: async_Page) -> str | None:
+    async def _get_page_content(page: async_Page) -> str:
         """
-        A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
+        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
         :param page: The page to extract content from.
         :return:
         """
@@ -534,6 +563,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             except PlaywrightError:
                 await page.wait_for_timeout(1000)
                 continue
+        return ""  # pyright: ignore
 
     async def _solve_cloudflare(self, page: async_Page):
         """Solve the cloudflare challenge displayed on the playwright page passed. The async version
@@ -541,6 +571,10 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param page: The async targeted page
         :return:
         """
+        try:
+            await page.wait_for_load_state("networkidle", timeout=5000)
+        except PlaywrightError:
+            pass
         challenge_type = self._detect_cloudflare(await self._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
@@ -556,26 +590,42 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 return
 
             else:
-                while "Verifying you are human." in (await self._get_page_content(page)):
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
-                    await page.wait_for_timeout(500)
-
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+                if challenge_type != "embedded":
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in (await self._get_page_content(page)):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        await page.wait_for_timeout(500)
+
+                outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
-                if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
-                    return
+                if iframe is not None:
+                    await iframe.wait_for_load_state(state="domcontentloaded")
+                    await iframe.wait_for_load_state("networkidle")
 
-                while not await (await iframe.frame_element()).is_visible():
-                    # Double-checking that the iframe is loaded
-                    await page.wait_for_timeout(500)
+                    if challenge_type != "embedded":
+                        while not await (await iframe.frame_element()).is_visible():
+                            # Double-checking that the iframe is loaded
+                            await page.wait_for_timeout(500)
+                    outer_box: Any = await (await iframe.frame_element()).bounding_box()
+
+                if not iframe or not outer_box:
+                    outer_box: Any = await page.locator(box_selector).last.bounding_box()
 
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
-                captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+                captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                await page.locator(".zone-name-title").wait_for(state="hidden")
+                await page.wait_for_load_state("networkidle")
+                if iframe is not None:
+                    # Wait for the frame to be removed from the page
+                    while iframe in page.frames:
+                        await page.wait_for_timeout(100)
+                if challenge_type != "embedded":
+                    await page.locator(box_selector).wait_for(state="detached")
+                    await page.locator(".zone-name-title").wait_for(state="hidden")
+                await page.wait_for_load_state(state="load")
                 await page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
@@ -612,7 +662,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
@@ -647,12 +697,17 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             if (
                 finished_response.request.resource_type == "document"
                 and finished_response.request.is_navigation_request()
+                and finished_response.request.frame == page_info.page.main_frame
             ):
                 final_response = finished_response
 
         page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
         page_info.mark_busy(url=url)
 
+        if TYPE_CHECKING:
+            if not isinstance(page_info.page, async_Page):
+                raise TypeError
+
         try:
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
@@ -701,7 +756,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 page_info.page, first_response, final_response, params.selector_config
             )
 
-            # Close the page, to free up resources
+            # Close the page to free up resources
             await page_info.page.close()
             self.page_pool.pages.remove(page_info)
 

scrapling/engines/_browsers/_config_tools.py

@@ -62,7 +62,7 @@ def _set_flags(hide_canvas, disable_webgl):  # pragma: no cover
 @lru_cache(2, typed=True)
 def _launch_kwargs(
     headless,
-    proxy,
+    proxy: Tuple,
     locale,
     extra_headers,
     useragent,