PlaywrightCapture 1.25.10__py3-none-any.whl → 1.25.12__py3-none-any.whl

playwrightcapture/capture.py

@@ -532,7 +532,7 @@ class Capture():
         async def handler() -> None:
             self.logger.debug('Didomi dialog found, clicking through.')
             if await page.locator("#didomi-notice-agree-button").is_visible():
-                await page.locator("#didomi-notice-agree-button").click(timeout=30000)
+                await page.locator("#didomi-notice-agree-button").click(timeout=3000)
 
         await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
@@ -575,10 +575,7 @@ class Capture():
 
     async def __dialog_alert_dialog_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            if await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").is_visible():
-                self.logger.info('Consent window found, clicking through.')
-                await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").click(timeout=2000)
-            elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
+            if await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
                 self.logger.info('Consent window found (alert dialog), but no button to click through.')
@@ -601,6 +598,11 @@ class Capture():
             elif await page.get_by_test_id("uc-accept-all-button").is_visible():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
+            elif await page.locator('#axeptio_btn_acceptAll').is_visible():
+                await page.locator('#axeptio_btn_acceptAll').click(timeout=2000)
+            elif await page.locator('.fc-cta-consent').is_visible():
+                # https://developers.google.com/funding-choices/fc-api-docs
+                await page.locator('.fc-cta-consent').click(timeout=2000)
             else:
                 self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
@@ -646,6 +648,67 @@ class Capture():
         )
         self.logger.info('Piwik handler added')
 
+    async def __frame_consent(self, frame: Frame) -> bool:
+        """Search & Click content in iframes. Cannot easily use the locator handler for this without having many many handlers.
+        And the iframes don't have a title or a role to easily identify them so we just try with generic locators that vary by language."""
+
+        labels_to_click: list[str] = [
+            # German
+            "Alle akzeptieren",
+            "Zustimmen & weiter",
+            # French
+            "Accepter et continuer",
+            "Tout accepter",
+            "Accepter",
+            "Accepter les cookies",
+            # English
+            "Accept & continue",
+            "Accept all",
+            # Dutch
+            "Accepteer",
+            # Spanish
+            "Aceptar todo",
+            # Italian
+            "Accetta tutto",
+            # Arabic
+            "قبول الكل",
+            # Portuguese
+            "Aceitar tudo",
+            # Polish
+            "Akceptuj wszystko",
+        ]
+
+        got_button: bool = False
+        try:
+            if await frame.locator("button.button__acceptAll").is_visible():
+                self.logger.info('Consent window found, clicking through.')
+                got_button = True
+                await frame.locator("button.button__acceptAll").click(timeout=2000)
+            for label in labels_to_click:
+                if await frame.get_by_label(label).is_visible():
+                    got_button = True
+                    self.logger.debug(f'Got button by label on frame: {label}')
+                    await frame.get_by_label(label).click(timeout=2000)
+                    break
+                if await frame.get_by_role("button", name=label).is_visible():
+                    got_button = True
+                    self.logger.debug(f'Got button by role on frame: {label}')
+                    await frame.get_by_role("button", name=label).click(timeout=2000)
+                    break
+        except Exception as e:
+            self.logger.info(f'Issue with frame consent: {e}')
+        return got_button
+
+    async def _move_time_forward(self, page: Page, time: int) -> None:
+        time = max(time, 7)
+        try:
+            async with timeout(3):
+                await page.clock.run_for(random.randint((time - 5) * 1000,
+                                                        (time + 5) * 1000))
+                self.logger.debug(f'Moved time forward by ~{time}s.')
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.warning('Unable to move time forward.')
+
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
                            page: Page | None=None, depth: int=0,
@@ -714,7 +777,7 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
-                # await page.clock.install()
+                await page.clock.install()
             except Error as e:
                 self.logger.warning(f'The context is in a broken state: {e}')
                 self.should_retry = True
@@ -839,6 +902,21 @@ class Capture():
                         except Exception as e:
                             self.logger.warning(f'Could not find body: {e}')
 
+                        await self._wait_for_random_timeout(page, 5)
+                        # triggering clicks on very generic frames is sometimes impossible, using button and common language.
+                        self.logger.debug('Check other frames for button')
+                        for frame in page.frames:
+                            if await self.__frame_consent(frame):
+                                await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
+                        self.logger.debug('Done with frames.')
+
+                        self.logger.debug('Check main frame for button')
+                        if await self.__frame_consent(page.main_frame):
+                            self.logger.debug('Got button on main frame')
+                            await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
+
+                    await self._move_time_forward(page, 10)
+
                     if parsed_url.fragment:
                         # We got a fragment, make sure we go to it and scroll only a little bit.
                         fragment = unquote(parsed_url.fragment)
@@ -902,17 +980,13 @@ class Capture():
                                 z.writestr(f'{i}_{filename}', file_content)
                         to_return["downloaded_file"] = mem_zip.getvalue()
 
-                # fast forward 30s
-                # try:
-                #    async with timeout(3):
-                #        await page.clock.run_for("47")
-                #        self.logger.debug('Moved time forward.')
-                # except (TimeoutError, asyncio.TimeoutError):
-                #    self.logger.warning('Unable to move time forward.')
+                # fast forward ~30s
+                await self._move_time_forward(page, 30)
 
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
                 await self._safe_wait(page)
+
                 self.logger.debug('Done with instrumentation, done with waiting.')
 
                 if content := await self._failsafe_get_content(page):

{playwrightcapture-1.25.10.dist-info → playwrightcapture-1.25.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.10
+Version: 1.25.12
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -22,16 +22,16 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.4,<4.0.0) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.9,<0.10)
-Requires-Dist: aiohttp[speedups] (>=3.10.1,<4.0.0)
+Requires-Dist: aiohttp[speedups] (>=3.10.3,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.45.1,<2.0.0)
+Requires-Dist: playwright (>=1.46.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.26,<2.0)
+Requires-Dist: puremagic (>=1.27,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=72.1.0,<73.0.0)
+Requires-Dist: setuptools (>=72.2.0,<73.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.10.dist-info → playwrightcapture-1.25.12.dist-info}/RECORD

@@ -1,9 +1,9 @@
 playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=AayQPmsjzcN2TIBi4uuwDmV0kqIVLDdAfEaNPUg8TXY,72606
+playwrightcapture/capture.py,sha256=R8ayCKpD1Q78ewaQ8-4tuMP8XVE8YGOwhwVYZDs4a8g,75867
 playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
 playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
 playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.25.10.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.25.10.dist-info/METADATA,sha256=3nVVIzM7kcR62_XYA1mRUxfoflPZwfLS5B8KDPy3Hks,3172
-playwrightcapture-1.25.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.25.10.dist-info/RECORD,,
+playwrightcapture-1.25.12.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.12.dist-info/METADATA,sha256=Kw9CVpRSN9vLL1XSf_ZGIKAzX11VY4MJuXrO0OVCmHw,3172
+playwrightcapture-1.25.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.12.dist-info/RECORD,,