scrapling 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

scrapling/engines/_browsers/_camoufox.py

@@ -1,29 +1,29 @@
+from random import randint
 from re import compile as re_compile
 
 from playwright.sync_api import (
-    Response as SyncPlaywrightResponse,
-    sync_playwright,
-    Locator,
     Page,
+    Locator,
+    sync_playwright,
 )
 from playwright.async_api import (
     async_playwright,
-    Response as AsyncPlaywrightResponse,
-    BrowserContext as AsyncBrowserContext,
-    Playwright as AsyncPlaywright,
-    Locator as AsyncLocator,
     Page as async_Page,
+    Locator as AsyncLocator,
+    Playwright as AsyncPlaywright,
+    BrowserContext as AsyncBrowserContext,
 )
-from playwright._impl._errors import Error as PlaywrightError
 
-from ._validators import validate_fetch as _validate
+from ._validators import validate_fetch as _validate, CamoufoxConfig
 from ._base import SyncSession, AsyncSession, StealthySessionMixin
 from scrapling.core.utils import log
 from scrapling.core._types import (
+    Any,
     Dict,
     List,
     Optional,
     Callable,
+    TYPE_CHECKING,
     SelectorWaitStates,
 )
 from scrapling.engines.toolbelt.convertor import (
@@ -33,7 +33,7 @@ from scrapling.engines.toolbelt.convertor import (
 from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
 
 __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
-_UNSET = object()
+_UNSET: Any = object()
 
 
 class StealthySession(StealthySessionMixin, SyncSession):
@@ -101,6 +101,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         os_randomize: bool = False,
         disable_ads: bool = False,
         geoip: bool = False,
+        user_data_dir: str = "",
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -133,6 +134,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -156,6 +158,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             block_images=block_images,
             block_webrtc=block_webrtc,
             os_randomize=os_randomize,
+            user_data_dir=user_data_dir,
             wait_selector=wait_selector,
             google_search=google_search,
             extra_headers=extra_headers,
@@ -170,9 +173,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
     def __create__(self):
         """Create a browser for this instance and context."""
         self.playwright = sync_playwright().start()
-        self.context = self.playwright.firefox.launch_persistent_context(  # pragma: no cover
-            **self.launch_options
-        )
+        self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
 
         if self.init_script:  # pragma: no cover
             self.context.add_init_script(path=self.init_script)
@@ -180,56 +181,21 @@ class StealthySession(StealthySessionMixin, SyncSession):
         if self.cookies:  # pragma: no cover
             self.context.add_cookies(self.cookies)
 
-    def __enter__(self):  # pragma: no cover
-        self.__create__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-    def close(self):  # pragma: no cover
-        """Close all resources"""
-        if self._closed:  # pragma: no cover
-            return
-
-        if self.context:
-            self.context.close()
-            self.context = None
-
-        if self.playwright:
-            self.playwright.stop()
-            self.playwright = None
-
-        self._closed = True
-
-    @staticmethod
-    def _get_page_content(page: Page) -> str | None:
-        """
-        A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
-        :param page: The page to extract content from.
-        :return:
-        """
-        while True:
-            try:
-                return page.content() or ""
-            except PlaywrightError:
-                page.wait_for_timeout(1000)
-                continue
-
     def _solve_cloudflare(self, page: Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
 
         :param page: The targeted page
         :return:
         """
-        challenge_type = self._detect_cloudflare(self._get_page_content(page))
+        self._wait_for_networkidle(page, timeout=5000)
+        challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
             return
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":
-                while "<title>Just a moment...</title>" in (self._get_page_content(page)):
+                while "<title>Just a moment...</title>" in (ResponseFactory._get_page_content(page)):
                     log.info("Waiting for Cloudflare wait page to disappear.")
                     page.wait_for_timeout(1000)
                     page.wait_for_load_state()
@@ -240,31 +206,43 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
                 if challenge_type != "embedded":
                     box_selector = ".main-content p+div>div>div"
-                    while "Verifying you are human." in self._get_page_content(page):
+                    while "Verifying you are human." in ResponseFactory._get_page_content(page):
                         # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
                         page.wait_for_timeout(500)
 
+                outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
-                if iframe is None:
-                    log.error("Didn't find Cloudflare iframe!")
-                    return
+                if iframe is not None:
+                    self._wait_for_page_stability(iframe, True, True)
 
-                if challenge_type != "embedded":
-                    while not iframe.frame_element().is_visible():
-                        # Double-checking that the iframe is loaded
-                        page.wait_for_timeout(500)
+                    if challenge_type != "embedded":
+                        while not iframe.frame_element().is_visible():
+                            # Double-checking that the iframe is loaded
+                            page.wait_for_timeout(500)
+                    outer_box: Any = iframe.frame_element().bounding_box()
+
+                if not iframe or not outer_box:
+                    outer_box: Any = page.locator(box_selector).last.bounding_box()
 
-                iframe.wait_for_load_state(state="domcontentloaded")
-                iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = page.locator(box_selector).last.bounding_box()
-                captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+                captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
+                self._wait_for_networkidle(page)
+                if iframe is not None:
+                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                    attempts = 0
+                    while iframe in page.frames:
+                        if attempts >= 300:
+                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                            break
+                        page.wait_for_timeout(100)
+                        attempts += 1
                 if challenge_type != "embedded":
+                    page.locator(box_selector).last.wait_for(state="detached")
                     page.locator(".zone-name-title").wait_for(state="hidden")
-                page.wait_for_load_state(state="domcontentloaded")
+                self._wait_for_page_stability(page, True, False)
 
                 log.info("Cloudflare captcha is solved")
                 return
@@ -319,37 +297,26 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            CamoufoxConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        def handle_response(finished_response: SyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-            ):
-                final_response = finished_response
-
         page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
 
         try:  # pragma: no cover
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = page_info.page.goto(url, referer=referer)
-            if params.load_dom:
-                page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                page_info.page.wait_for_load_state("networkidle")
+            self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -357,11 +324,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             if params.solve_cloudflare:
                 self._solve_cloudflare(page_info.page)
                 # Make sure the page is fully loaded after the captcha
-                page_info.page.wait_for_load_state(state="load")
-                if params.load_dom:
-                    page_info.page.wait_for_load_state(state="domcontentloaded")
-                if params.network_idle:
-                    page_info.page.wait_for_load_state("networkidle")
+                self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
             if params.page_action:
                 try:
@@ -374,20 +337,16 @@ class StealthySession(StealthySessionMixin, SyncSession):
                     waiter: Locator = page_info.page.locator(params.wait_selector)
                     waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    page_info.page.wait_for_load_state(state="load")
-                    if params.load_dom:
-                        page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        page_info.page.wait_for_load_state("networkidle")
+                    self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
                 except Exception as e:
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
             page_info.page.wait_for_timeout(params.wait)
             response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
-            # Close the page, to free up resources
+            # Close the page to free up resources
             page_info.page.close()
             self.page_pool.pages.remove(page_info)
 
@@ -427,6 +386,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         os_randomize: bool = False,
         disable_ads: bool = False,
         geoip: bool = False,
+        user_data_dir: str = "",
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -460,6 +420,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
+        :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -485,6 +446,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             wait_selector=wait_selector,
             google_search=google_search,
             extra_headers=extra_headers,
+            user_data_dir=user_data_dir,
             additional_args=additional_args,
             selector_config=selector_config,
             solve_cloudflare=solve_cloudflare,
@@ -504,58 +466,23 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             await self.context.add_init_script(path=self.init_script)
 
         if self.cookies:
-            await self.context.add_cookies(self.cookies)
-
-    async def __aenter__(self):
-        await self.__create__()
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-
-    async def close(self):
-        """Close all resources"""
-        if self._closed:  # pragma: no cover
-            return
-
-        if self.context:
-            await self.context.close()
-            self.context = None
+            await self.context.add_cookies(self.cookies)  # pyright: ignore [reportArgumentType]
 
-        if self.playwright:
-            await self.playwright.stop()
-            self.playwright = None
-
-        self._closed = True
-
-    @staticmethod
-    async def _get_page_content(page: async_Page) -> str | None:
-        """
-        A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
-        :param page: The page to extract content from.
-        :return:
-        """
-        while True:
-            try:
-                return (await page.content()) or ""
-            except PlaywrightError:
-                await page.wait_for_timeout(1000)
-                continue
-
-    async def _solve_cloudflare(self, page: async_Page):
+    async def _solve_cloudflare(self, page: async_Page):  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed. The async version
 
         :param page: The async targeted page
         :return:
         """
-        challenge_type = self._detect_cloudflare(await self._get_page_content(page))
+        await self._wait_for_networkidle(page, timeout=5000)
+        challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
             return
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":  # pragma: no cover
-                while "<title>Just a moment...</title>" in (await self._get_page_content(page)):
+                while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
                     log.info("Waiting for Cloudflare wait page to disappear.")
                     await page.wait_for_timeout(1000)
                     await page.wait_for_load_state()
@@ -566,31 +493,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
                 if challenge_type != "embedded":
                     box_selector = ".main-content p+div>div>div"
-                    while "Verifying you are human." in (await self._get_page_content(page)):
+                    while "Verifying you are human." in (await ResponseFactory._get_async_page_content(page)):
                         # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
                         await page.wait_for_timeout(500)
 
+                outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
-                if iframe is None:
-                    log.error("Didn't find Cloudflare iframe!")
-                    return
+                if iframe is not None:
+                    await self._wait_for_page_stability(iframe, True, True)
 
-                if challenge_type != "embedded":
-                    while not await (await iframe.frame_element()).is_visible():
-                        # Double-checking that the iframe is loaded
-                        await page.wait_for_timeout(500)
+                    if challenge_type != "embedded":
+                        while not await (await iframe.frame_element()).is_visible():
+                            # Double-checking that the iframe is loaded
+                            await page.wait_for_timeout(500)
+                    outer_box: Any = await (await iframe.frame_element()).bounding_box()
+
+                if not iframe or not outer_box:
+                    outer_box: Any = await page.locator(box_selector).last.bounding_box()
 
-                await iframe.wait_for_load_state(state="domcontentloaded")
-                await iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = await page.locator(box_selector).last.bounding_box()
-                captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+                captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
+                await self._wait_for_networkidle(page)
+                if iframe is not None:
+                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                    attempts = 0
+                    while iframe in page.frames:
+                        if attempts >= 300:
+                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                            break
+                        await page.wait_for_timeout(100)
+                        attempts += 1
                 if challenge_type != "embedded":
+                    await page.locator(box_selector).wait_for(state="detached")
                     await page.locator(".zone-name-title").wait_for(state="hidden")
-                await page.wait_for_load_state(state="domcontentloaded")
+                await self._wait_for_page_stability(page, True, False)
 
                 log.info("Cloudflare captcha is solved")
                 return
@@ -645,37 +584,30 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            CamoufoxConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        async def handle_response(finished_response: AsyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-            ):
-                final_response = finished_response
-
         page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
+
+        if TYPE_CHECKING:
+            if not isinstance(page_info.page, async_Page):
+                raise TypeError
 
         try:
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = await page_info.page.goto(url, referer=referer)
-            if params.load_dom:
-                await page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                await page_info.page.wait_for_load_state("networkidle")
+            await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -683,11 +615,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             if params.solve_cloudflare:
                 await self._solve_cloudflare(page_info.page)
                 # Make sure the page is fully loaded after the captcha
-                await page_info.page.wait_for_load_state(state="load")
-                if params.load_dom:
-                    await page_info.page.wait_for_load_state(state="domcontentloaded")
-                if params.network_idle:
-                    await page_info.page.wait_for_load_state("networkidle")
+                await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
 
             if params.page_action:
                 try:
@@ -700,11 +628,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                     waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
                     await waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    await page_info.page.wait_for_load_state(state="load")
-                    if params.load_dom:
-                        await page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        await page_info.page.wait_for_load_state("networkidle")
+                    await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
                 except Exception as e:
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
@@ -712,10 +636,10 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
 
             # Create response object
             response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
-            # Close the page, to free up resources
+            # Close the page to free up resources
             await page_info.page.close()
             self.page_pool.pages.remove(page_info)
 

scrapling/engines/_browsers/_config_tools.py

@@ -62,7 +62,7 @@ def _set_flags(hide_canvas, disable_webgl):  # pragma: no cover
 @lru_cache(2, typed=True)
 def _launch_kwargs(
     headless,
-    proxy,
+    proxy: Tuple,
     locale,
     extra_headers,
     useragent,
@@ -70,12 +70,17 @@ def _launch_kwargs(
     stealth,
     hide_canvas,
     disable_webgl,
+    extra_flags: Tuple,
 ) -> Tuple:
     """Creates the arguments we will use while launching playwright's browser"""
+    base_args = DEFAULT_FLAGS
+    if extra_flags:
+        base_args = base_args + extra_flags
+
     launch_kwargs = {
         "locale": locale,
         "headless": headless,
-        "args": DEFAULT_FLAGS,
+        "args": base_args,
         "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
         "proxy": proxy or tuple(),
         "device_scale_factor": 2,
@@ -85,9 +90,10 @@ def _launch_kwargs(
         "user_agent": useragent or __default_useragent__,
     }
     if stealth:
+        stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)
         launch_kwargs.update(
             {
-                "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl),
+                "args": stealth_args,
                 "chromium_sandbox": True,
                 "is_mobile": False,
                 "has_touch": False,