PlaywrightCapture 1.25.10__tar.gz → 1.25.12__tar.gz

{playwrightcapture-1.25.10 → playwrightcapture-1.25.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.10
+Version: 1.25.12
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -22,16 +22,16 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.4,<4.0.0) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.9,<0.10)
-Requires-Dist: aiohttp[speedups] (>=3.10.1,<4.0.0)
+Requires-Dist: aiohttp[speedups] (>=3.10.3,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.45.1,<2.0.0)
+Requires-Dist: playwright (>=1.46.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.26,<2.0)
+Requires-Dist: puremagic (>=1.27,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=72.1.0,<73.0.0)
+Requires-Dist: setuptools (>=72.2.0,<73.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.10 → playwrightcapture-1.25.12}/playwrightcapture/capture.py

@@ -532,7 +532,7 @@ class Capture():
         async def handler() -> None:
             self.logger.debug('Didomi dialog found, clicking through.')
             if await page.locator("#didomi-notice-agree-button").is_visible():
-                await page.locator("#didomi-notice-agree-button").click(timeout=30000)
+                await page.locator("#didomi-notice-agree-button").click(timeout=3000)
 
         await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
@@ -575,10 +575,7 @@ class Capture():
 
     async def __dialog_alert_dialog_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            if await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").is_visible():
-                self.logger.info('Consent window found, clicking through.')
-                await page.frame_locator("iframe[title=\"Consent window\"]").locator("button.button__acceptAll").click(timeout=2000)
-            elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
+            if await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
                 self.logger.info('Consent window found (alert dialog), but no button to click through.')
@@ -601,6 +598,11 @@ class Capture():
             elif await page.get_by_test_id("uc-accept-all-button").is_visible():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
+            elif await page.locator('#axeptio_btn_acceptAll').is_visible():
+                await page.locator('#axeptio_btn_acceptAll').click(timeout=2000)
+            elif await page.locator('.fc-cta-consent').is_visible():
+                # https://developers.google.com/funding-choices/fc-api-docs
+                await page.locator('.fc-cta-consent').click(timeout=2000)
             else:
                 self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
@@ -646,6 +648,67 @@ class Capture():
         )
         self.logger.info('Piwik handler added')
 
+    async def __frame_consent(self, frame: Frame) -> bool:
+        """Search & Click content in iframes. Cannot easily use the locator handler for this without having many many handlers.
+        And the iframes don't have a title or a role to easily identify them so we just try with generic locators that vary by language."""
+
+        labels_to_click: list[str] = [
+            # German
+            "Alle akzeptieren",
+            "Zustimmen & weiter",
+            # French
+            "Accepter et continuer",
+            "Tout accepter",
+            "Accepter",
+            "Accepter les cookies",
+            # English
+            "Accept & continue",
+            "Accept all",
+            # Dutch
+            "Accepteer",
+            # Spanish
+            "Aceptar todo",
+            # Italian
+            "Accetta tutto",
+            # Arabic
+            "قبول الكل",
+            # Portuguese
+            "Aceitar tudo",
+            # Polish
+            "Akceptuj wszystko",
+        ]
+
+        got_button: bool = False
+        try:
+            if await frame.locator("button.button__acceptAll").is_visible():
+                self.logger.info('Consent window found, clicking through.')
+                got_button = True
+                await frame.locator("button.button__acceptAll").click(timeout=2000)
+            for label in labels_to_click:
+                if await frame.get_by_label(label).is_visible():
+                    got_button = True
+                    self.logger.debug(f'Got button by label on frame: {label}')
+                    await frame.get_by_label(label).click(timeout=2000)
+                    break
+                if await frame.get_by_role("button", name=label).is_visible():
+                    got_button = True
+                    self.logger.debug(f'Got button by role on frame: {label}')
+                    await frame.get_by_role("button", name=label).click(timeout=2000)
+                    break
+        except Exception as e:
+            self.logger.info(f'Issue with frame consent: {e}')
+        return got_button
+
+    async def _move_time_forward(self, page: Page, time: int) -> None:
+        time = max(time, 7)
+        try:
+            async with timeout(3):
+                await page.clock.run_for(random.randint((time - 5) * 1000,
+                                                        (time + 5) * 1000))
+                self.logger.debug(f'Moved time forward by ~{time}s.')
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.warning('Unable to move time forward.')
+
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
                            page: Page | None=None, depth: int=0,
@@ -714,7 +777,7 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
-                # await page.clock.install()
+                await page.clock.install()
             except Error as e:
                 self.logger.warning(f'The context is in a broken state: {e}')
                 self.should_retry = True
@@ -839,6 +902,21 @@ class Capture():
                         except Exception as e:
                             self.logger.warning(f'Could not find body: {e}')
 
+                        await self._wait_for_random_timeout(page, 5)
+                        # triggering clicks on very generic frames is sometimes impossible, using button and common language.
+                        self.logger.debug('Check other frames for button')
+                        for frame in page.frames:
+                            if await self.__frame_consent(frame):
+                                await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
+                        self.logger.debug('Done with frames.')
+
+                        self.logger.debug('Check main frame for button')
+                        if await self.__frame_consent(page.main_frame):
+                            self.logger.debug('Got button on main frame')
+                            await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
+
+                    await self._move_time_forward(page, 10)
+
                     if parsed_url.fragment:
                         # We got a fragment, make sure we go to it and scroll only a little bit.
                         fragment = unquote(parsed_url.fragment)
@@ -902,17 +980,13 @@ class Capture():
                                 z.writestr(f'{i}_{filename}', file_content)
                         to_return["downloaded_file"] = mem_zip.getvalue()
 
-                # fast forward 30s
-                # try:
-                #    async with timeout(3):
-                #        await page.clock.run_for("47")
-                #        self.logger.debug('Moved time forward.')
-                # except (TimeoutError, asyncio.TimeoutError):
-                #    self.logger.warning('Unable to move time forward.')
+                # fast forward ~30s
+                await self._move_time_forward(page, 30)
 
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
                 await self._safe_wait(page)
+
                 self.logger.debug('Done with instrumentation, done with waiting.')
 
                 if content := await self._failsafe_get_content(page):

{playwrightcapture-1.25.10 → playwrightcapture-1.25.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.25.10"
+version = "1.25.12"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -19,7 +19,7 @@ classifiers=[
 
 [tool.poetry.dependencies]
 python = "^3.8"
-playwright = "^1.45.1"
+playwright = "^1.46.0"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.2.1"
@@ -28,10 +28,10 @@ SpeechRecognition = {version = "^3.10.4", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^72.1.0"
-puremagic = "^1.26"
+setuptools = "^72.2.0"
+puremagic = "^1.27"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
-aiohttp = {extras = ["speedups"], version = "^3.10.1"}
+aiohttp = {extras = ["speedups"], version = "^3.10.3"}
 aiohttp-socks = "^0.9"
 
 [tool.poetry.extras]