PlaywrightCapture 1.23.11__tar.gz → 1.23.13__tar.gz

{playwrightcapture-1.23.11 → playwrightcapture-1.23.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.23.11
+Version: 1.23.13
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause

{playwrightcapture-1.23.11 → playwrightcapture-1.23.13}/playwrightcapture/capture.py

@@ -122,7 +122,7 @@ class Capture():
                               'password': proxy.get('password', '')}
 
         self.should_retry: bool = False
-        self.__network_not_idle: int = 1
+        self.__network_not_idle: int = 2  # makes sure we do not wait for network idle the max amount of time the capture is allowed to take
         self._cookies: list[SetCookieParam] = []
         self._http_credentials: HttpCredentials = {}
         self._geolocation: Geolocation = {}
@@ -461,7 +461,7 @@ class Capture():
                 await page.locator("#onetrust-accept-btn-handler").click()
 
         await page.add_locator_handler(
-            page.locator('.ot-sdk-container'),
+            page.locator('#onetrust-banner-sdk'),
             handler
         )
         self.logger.info('OT handler added')
@@ -524,6 +524,7 @@ class Capture():
                            ) -> CaptureResponse:
 
         to_return: CaptureResponse = {}
+        got_favicons = False
 
         # We don't need to be super strict on the lock, as it simply triggers a wait for network idle before stoping the capture
         # but we still need it to be an integer in case we have more than one download triggered and one finished when the others haven't
@@ -551,8 +552,12 @@ class Capture():
 
         async def store_request(request: Request) -> None:
             # This method is called on each request, to store the body (if it is an image) in a dict indexed by URL
+            if got_favicons:
+                return
             try:
                 if response := await request.response():
+                    if got_favicons:
+                        return
                     if response.ok:
                         try:
                             if body := await response.body():
@@ -585,7 +590,7 @@ class Capture():
             await stealth_async(page)
             page.set_default_timeout(self._capture_timeout * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
-            page.on("request", store_request)
+            page.on("requestfinished", store_request)
 
         try:
             # Parse the URL. If there is a fragment, we need to scroll to it manually
@@ -645,8 +650,8 @@ class Capture():
                 # Same technique as: https://github.com/NikolaiT/uncaptcha3
                 if CAN_SOLVE_CAPTCHA:
                     try:
-                        if (await page.locator("//iframe[@title='reCAPTCHA']").first.is_visible(timeout=5000)
-                                and await page.locator("//iframe[@title='reCAPTCHA']").first.is_enabled(timeout=5000)):
+                        if (await page.locator("//iframe[@title='reCAPTCHA']").first.is_visible(timeout=3000)
+                                and await page.locator("//iframe[@title='reCAPTCHA']").first.is_enabled(timeout=2000)):
                             self.logger.info('Found a captcha')
                             await self._recaptcha_solver(page)
                     except PlaywrightTimeoutError as e:
@@ -675,7 +680,7 @@ class Capture():
                         # We got a fragment, make sure we go to it and scroll only a little bit.
                         fragment = unquote(parsed_url.fragment)
                         try:
-                            await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=5000)
+                            await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=3000)
                             await self._safe_wait(page)
                             await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
                             self.logger.debug('Jumped to fragment.')
@@ -717,10 +722,12 @@ class Capture():
 
                 to_return['last_redirected_url'] = page.url
 
-                to_return['png'] = await self._failsafe_get_screenshot(page)
-
                 if 'html' in to_return and to_return['html'] is not None and with_favicon:
                     to_return['potential_favicons'] = self.get_favicons(page.url, to_return['html'])
+                    got_favicons = True
+
+                await self._safe_wait(page)
+                to_return['png'] = await self._failsafe_get_screenshot(page)
 
                 if self.wait_for_download > 0:
                     self.logger.info('Waiting for download to finish...')
@@ -853,17 +860,17 @@ class Capture():
 
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
         try:
-            return await page.screenshot(full_page=True)
+            return await page.screenshot(full_page=True, timeout=5000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css")
+            return await page.screenshot(full_page=True, scale="css", timeout=10000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 
         try:
-            return await page.screenshot()
+            return await page.screenshot(scale="css", animations='disabled', timeout=10000)
         except Error as e:
             self.logger.warning(f"Unable to get any screenshot: {e}")
             raise e
@@ -871,7 +878,7 @@ class Capture():
     async def _safe_wait(self, page: Page) -> None:
         try:
             # If we don't have networkidle relatively quick, it's probably because we're playing a video.
-            await page.wait_for_load_state('networkidle', timeout=10000 / self.__network_not_idle)
+            await page.wait_for_load_state('networkidle', timeout=self._capture_timeout / self.__network_not_idle)
         except PlaywrightTimeoutError:
             # Network never idle, keep going
             self.__network_not_idle += 1

{playwrightcapture-1.23.11 → playwrightcapture-1.23.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.23.11"
+version = "1.23.13"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"