PlaywrightCapture 1.23.13__tar.gz → 1.24.0__tar.gz

{playwrightcapture-1.23.13 → playwrightcapture-1.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.23.13
+Version: 1.24.0
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
-Requires-Dist: SpeechRecognition (>=3.10.1,<4.0.0) ; extra == "recaptcha"
+Requires-Dist: SpeechRecognition (>=3.10.2,<4.0.0) ; extra == "recaptcha"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
 Requires-Dist: playwright (>=1.42.0,<2.0.0)
@@ -53,8 +53,8 @@ A very basic example:
 from playwrightcapture import Capture
 
 async with Capture() as capture:
-    await capture.prepare_context()
-    entries = await capture.capture_page(url)
+    await capture.initialize_context()
+    entries = await capture.capture_page(url, max_depth_capture_time=90)
 ```
 
 Entries is a dictionaries that contains (if all goes well) the HAR, the screenshot, all the cookies of the session, the URL as it is in the browser at the end of the capture, and the full HTML page as rendered.

{playwrightcapture-1.23.13 → playwrightcapture-1.24.0}/README.md

@@ -16,8 +16,8 @@ A very basic example:
 from playwrightcapture import Capture
 
 async with Capture() as capture:
-    await capture.prepare_context()
-    entries = await capture.capture_page(url)
+    await capture.initialize_context()
+    entries = await capture.capture_page(url, max_depth_capture_time=90)
 ```
 
 Entries is a dictionaries that contains (if all goes well) the HAR, the screenshot, all the cookies of the session, the URL as it is in the browser at the end of the capture, and the full HTML page as rendered.

{playwrightcapture-1.23.13 → playwrightcapture-1.24.0}/playwrightcapture/capture.py

@@ -426,10 +426,10 @@ class Capture():
             while max_tries > 0:
                 # cf_locator = page.frame_locator("iframe[title=\"Widget containing a Cloudflare security challenge\"]").get_by_label("Verify you are human")
                 cf_locator = page.frame_locator("iframe[title=\"Widget containing a Cloudflare security challenge\"]").get_by_role("checkbox")
-                await self._safe_wait(page)
+                await self._safe_wait(page, 5)
                 await cf_locator.click(force=True, position={"x": random.uniform(1, 32), "y": random.uniform(1, 32)})
                 self.logger.info('Cloudflare widget visible.')
-                await self._safe_wait(page)
+                await self._safe_wait(page, 5)
                 await self._wait_for_random_timeout(page, 2)
                 spinner = page.locator('#challenge-spinner')
                 while True:
@@ -449,7 +449,7 @@ class Capture():
         async def handler() -> None:
             self.logger.debug('Didomi dialog found, clicking through.')
             if await page.locator("#didomi-notice-agree-button").is_visible():
-                await page.locator("#didomi-notice-agree-button").click()
+                await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
         await page.add_locator_handler(page.locator(".didomi-popup-view"), handler)
         self.logger.info('Didomi handler added')
@@ -458,7 +458,7 @@ class Capture():
         async def handler() -> None:
             self.logger.info('######## OT Dialog found, clicking through.')
             if await page.locator("#onetrust-accept-btn-handler").is_visible():
-                await page.locator("#onetrust-accept-btn-handler").click()
+                await page.locator("#onetrust-accept-btn-handler").click(timeout=2000)
 
         await page.add_locator_handler(
             page.locator('#onetrust-banner-sdk'),
@@ -470,7 +470,7 @@ class Capture():
         async def handler() -> None:
             self.logger.info('######## HS Dialog found, clicking through.')
             if await page.locator("#hs-eu-confirmation-button").is_visible():
-                await page.locator("#hs-eu-confirmation-button").click()
+                await page.locator("#hs-eu-confirmation-button").click(timeout=2000)
 
         await page.add_locator_handler(
             page.locator('#hs-eu-cookie-confirmation'),
@@ -482,7 +482,7 @@ class Capture():
         async def handler() -> None:
             self.logger.info('######## Cookiebot Dialog found, clicking through.')
             if await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").is_visible():
-                await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").click()
+                await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").click(timeout=2000)
 
         await page.add_locator_handler(
             page.locator('#CybotCookiebotDialogBody'),
@@ -494,20 +494,42 @@ class Capture():
         async def handler() -> None:
             if await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").is_visible():
                 self.logger.info('Consent window found, clicking through.')
-                await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").click()
+                await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").click(timeout=2000)
+            elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
+                await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
                 self.logger.info('Consent window found, but no button to click through.')
+
         await page.add_locator_handler(
             page.get_by_role("alertdialog"),
             handler
         )
         self.logger.info('alert dialog handler added')
 
+    async def __dialog_clickthrough(self, page: Page) -> None:
+        async def handler() -> None:
+            if await page.locator(".qc-cmp2-summary-buttons").locator("button").first.is_visible():
+                self.logger.info('Consent window found, clicking through.')
+                await page.locator(".qc-cmp2-summary-buttons").locator("button").locator("nth=-1").click(timeout=2000)
+            elif await page.locator("#popin_tc_privacy").locator("#popin_tc_privacy_button_2").is_visible():
+                self.logger.info('Consent window found, clicking through.')
+                await page.locator("#popin_tc_privacy").locator("#popin_tc_privacy_button_2").click(timeout=2000)
+            elif await page.get_by_test_id("uc-accept-all-button").is_visible():
+                self.logger.info('Consent window found, clicking through.')
+                await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
+            else:
+                self.logger.info('Consent window found, but no button to click through.')
+        await page.add_locator_handler(
+            page.get_by_role("dialog").last,
+            handler
+        )
+        self.logger.info('dialog handler added')
+
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
             self.logger.info('######## Complianz found, clicking through.')
             if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
-                await page.locator('.cmplz-show').locator("button.cmplz-accept").click()
+                await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
 
         await page.add_locator_handler(
             page.locator('.cmplz-show'),
@@ -515,6 +537,30 @@ class Capture():
         )
         self.logger.info('Complianz handler added')
 
+    async def __dialog_yahoo_clickthrough(self, page: Page) -> None:
+        async def handler() -> None:
+            self.logger.info('######## Yahoo found, clicking through.')
+            if await page.locator('.con-wizard').locator("button.accept-all").is_visible():
+                await page.locator('.con-wizard').locator("button.accept-all").click(timeout=2000)
+
+        await page.add_locator_handler(
+            page.locator('.con-wizard'),
+            handler
+        )
+        self.logger.info('Yahoo handler added')
+
+    async def __dialog_ppms_clickthrough(self, page: Page) -> None:
+        async def handler() -> None:
+            self.logger.info('######## piwik found, clicking through.')
+            if await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").is_visible():
+                await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").click(timeout=2000)
+
+        await page.add_locator_handler(
+            page.locator('#ppms_cm_popup_overlay'),
+            handler
+        )
+        self.logger.info('Yahoo handler added')
+
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
                            page: Page | None=None, depth: int=0,
@@ -552,13 +598,13 @@ class Capture():
 
         async def store_request(request: Request) -> None:
             # This method is called on each request, to store the body (if it is an image) in a dict indexed by URL
-            if got_favicons:
+            if got_favicons or request.resource_type != 'image':
                 return
             try:
                 if response := await request.response():
                     if got_favicons:
                         return
-                    if response.ok:
+                    if request.resource_type == 'image' and response.ok:
                         try:
                             if body := await response.body():
                                 try:
@@ -584,8 +630,11 @@ class Capture():
                 await self.__dialog_onetrust_clickthrough(page)
                 await self.__dialog_hubspot_clickthrough(page)
                 await self.__dialog_cookiebot_clickthrough(page)
-                await self.__dialog_alert_dialog_clickthrough(page)
                 await self.__dialog_complianz_clickthrough(page)
+                await self.__dialog_yahoo_clickthrough(page)
+                await self.__dialog_ppms_clickthrough(page)
+                await self.__dialog_alert_dialog_clickthrough(page)
+                await self.__dialog_clickthrough(page)
 
             await stealth_async(page)
             page.set_default_timeout(self._capture_timeout * 1000)
@@ -642,10 +691,6 @@ class Capture():
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
                 self.logger.debug('Start instrumentation.')
 
-                if allow_tracking:
-                    # This event is required trigger the add_locator_handler
-                    await page.locator("body").click(button="right")
-
                 # ==== recaptcha
                 # Same technique as: https://github.com/NikolaiT/uncaptcha3
                 if CAN_SOLVE_CAPTCHA:
@@ -670,10 +715,16 @@ class Capture():
                 # check if we have anything on the page. If we don't, the page is not working properly.
                 if await self._failsafe_get_content(page):
                     self.logger.debug('Got rendered content')
+                    if allow_tracking:
+                        await self._wait_for_random_timeout(page, 2)
+                        # This event is required trigger the add_locator_handler
+                        if page.locator("body").is_visible():
+                            await page.locator("body").click(button="right", timeout=2000)
+
                     # move mouse
                     await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
                     self.logger.debug('Moved mouse.')
-                    await self._safe_wait(page)
+                    await self._wait_for_random_timeout(page, 2)
                     self.logger.debug('Keep going after moving mouse.')
 
                     if parsed_url.fragment:
@@ -681,7 +732,7 @@ class Capture():
                         fragment = unquote(parsed_url.fragment)
                         try:
                             await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=3000)
-                            await self._safe_wait(page)
+                            await self._wait_for_random_timeout(page, 2)
                             await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
                             self.logger.debug('Jumped to fragment.')
                         except PlaywrightTimeoutError as e:
@@ -699,39 +750,21 @@ class Capture():
                         except Error as e:
                             self.logger.debug(f'Unable to scroll: {e}')
 
-                    await self._safe_wait(page)
+                    await self._wait_for_random_timeout(page, 3)
                     self.logger.debug('Keep going after moving on page.')
 
                     try:
                         await page.keyboard.press('PageUp')
                         self.logger.debug('PageUp on keyboard')
-                        await self._safe_wait(page)
+                        await self._wait_for_random_timeout(page, 3)
                         await page.keyboard.press('PageDown')
                         self.logger.debug('PageDown on keyboard')
                     except Error as e:
                         self.logger.debug(f'Unable to use keyboard: {e}')
 
-                self.logger.debug('Done with instrumentation, waiting for network idle.')
-                await self._safe_wait(page)
-                await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after network idle
-                await self._safe_wait(page)
-                self.logger.debug('Done with instrumentation, done with waiting.')
-
-                if content := await self._failsafe_get_content(page):
-                    to_return['html'] = content
-
-                to_return['last_redirected_url'] = page.url
-
-                if 'html' in to_return and to_return['html'] is not None and with_favicon:
-                    to_return['potential_favicons'] = self.get_favicons(page.url, to_return['html'])
-                    got_favicons = True
-
-                await self._safe_wait(page)
-                to_return['png'] = await self._failsafe_get_screenshot(page)
-
                 if self.wait_for_download > 0:
                     self.logger.info('Waiting for download to finish...')
-                    await self._safe_wait(page)
+                    await self._safe_wait(page, 20)
 
                 if multiple_downloads:
                     if len(multiple_downloads) == 1:
@@ -746,6 +779,22 @@ class Capture():
                                 filename, file_content = f_details
                                 z.writestr(f'{i}_{filename}', file_content)
                         to_return["downloaded_file"] = mem_zip.getvalue()
+
+                self.logger.debug('Done with instrumentation, waiting for network idle.')
+                await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
+                await self._safe_wait(page)
+                self.logger.debug('Done with instrumentation, done with waiting.')
+
+                if content := await self._failsafe_get_content(page):
+                    to_return['html'] = content
+
+                if 'html' in to_return and to_return['html'] is not None and with_favicon:
+                    to_return['potential_favicons'] = self.get_favicons(page.url, to_return['html'])
+                    got_favicons = True
+
+                to_return['last_redirected_url'] = page.url
+                to_return['png'] = await self._failsafe_get_screenshot(page)
+
                 self._already_captured.add(url)
                 if depth > 0 and to_return.get('html') and to_return['html']:
                     if child_urls := self._get_links_from_rendered_page(page.url, to_return['html'], rendered_hostname_only):
@@ -859,29 +908,38 @@ class Capture():
         return to_return
 
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
+        self.logger.debug("Capturing a screenshot of the full page.")
         try:
             return await page.screenshot(full_page=True, timeout=5000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css", timeout=10000)
+            return await page.screenshot(full_page=True, scale="css", timeout=5000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 
         try:
-            return await page.screenshot(scale="css", animations='disabled', timeout=10000)
+            return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
         except Error as e:
             self.logger.warning(f"Unable to get any screenshot: {e}")
             raise e
 
-    async def _safe_wait(self, page: Page) -> None:
+    async def _safe_wait(self, page: Page, force_max_wait_in_sec: int | None=None) -> None:
+        max_wait: float
         try:
+            if force_max_wait_in_sec is not None:
+                max_wait = force_max_wait_in_sec
+            else:
+                max_wait = self._capture_timeout / self.__network_not_idle
+            max_wait *= 1000
+            self.logger.debug(f'Waiting for network idle, max wait: {max_wait}s')
             # If we don't have networkidle relatively quick, it's probably because we're playing a video.
-            await page.wait_for_load_state('networkidle', timeout=self._capture_timeout / self.__network_not_idle)
+            await page.wait_for_load_state('networkidle', timeout=max_wait)
         except PlaywrightTimeoutError:
             # Network never idle, keep going
             self.__network_not_idle += 1
+            self.logger.debug(f'Timed out - Waiting for network idle, max wait: {max_wait}s')
 
     async def _failsafe_get_content(self, page: Page) -> str | None:
         ''' The page might be changing for all kind of reason (generally a JS timeout).
@@ -894,7 +952,7 @@ class Capture():
                 self.logger.debug('Unable to get page content, trying again.')
                 tries -= 1
                 await self._wait_for_random_timeout(page, 1)
-                await self._safe_wait(page)
+                await self._safe_wait(page, 5)
             except Exception as e:
                 self.logger.warning(f'The Playwright Page is in a broken state: {e}.')
                 break
@@ -1004,7 +1062,7 @@ class Capture():
                 text = recognizer.recognize_google(audio)
             await main_frame.get_by_role("textbox", name="Enter what you hear").fill(text)
             await main_frame.get_by_role("button", name="Verify").click()
-            await self._safe_wait(page)
+            await self._safe_wait(page, 5)
             await self._wait_for_random_timeout(page, random.randint(3, 6))
             try:
                 if await recaptcha_init_frame.locator("//span[@id='recaptcha-anchor']").first.is_checked(timeout=5000):
@@ -1059,7 +1117,7 @@ class Capture():
         if timeout > 1000:
             self.logger.warning(f'The waiting time is too long {timeout}, we expect seconds, not miliseconds.')
             timeout = int(timeout / 1000)
-        _wait_time = random.randrange(timeout * 1000 - 500, timeout * 1000 + 500)
+        _wait_time = random.randrange(max(timeout * 1000 - 500, 500), max(timeout * 1000 + 500, 1000))
         await page.wait_for_timeout(_wait_time)
 
     def make_frame_tree(self, frame: Frame) -> dict[str, list[dict[str, Any]]]:

{playwrightcapture-1.23.13 → playwrightcapture-1.24.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.23.13"
+version = "1.24.0"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -25,7 +25,7 @@ beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.1.2"
 requests = {extras = ["socks"], version = "^2.31.0"}
 pydub = {version = "^0.25.1", optional = true}
-SpeechRecognition = {version = "^3.10.1", optional = true}
+SpeechRecognition = {version = "^3.10.2", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"