PlaywrightCapture 1.25.9__py3-none-any.whl → 1.25.11__py3-none-any.whl

playwrightcapture/capture.py

@@ -96,28 +96,40 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
         return msg, kwargs
 
 
+# good test pages:
+# https://bot.incolumitas.com/
+# https://fingerprintjs.github.io/BotD/main/
+
 @dataclass
 class PCStealthConfig(StealthConfig):  # type: ignore[misc]
 
     @property
     def enabled_scripts(self) -> Generator[str, None, None]:
-        self.webdriver = True
-        self.webgl_vendor = True
         self.chrome_app = True
         self.chrome_csi = True
-        self.chrome_load_times = True
         self.chrome_runtime = True
+        self.chrome_load_times = True
+        self.navigator_plugins = True
+        self.hairline = True
         self.iframe_content_window = True
         self.media_codecs = True
+
+        # permissions are handled directly in playwright
+        self.navigator_permissions = False
+        # Platform is correct now
+        self.navigator_platform = False
+        # probably useless, but it will fallback to 4 regardless
         self.navigator_hardware_concurrency = 4
+        # Webgl vendor is correct now
+        self.webgl_vendor = False
+        # Set by the viewport
+        self.outerdimensions = False
+
+        # Not working with Playwright 1.45+
         self.navigator_languages = False  # Causes issue
-        self.navigator_permissions = True
-        self.navigator_platform = True
-        self.navigator_plugins = True
         self.navigator_user_agent = False  # Causes issues
         self.navigator_vendor = False  # Causes issues
-        self.outerdimensions = True
-        self.hairline = True
+
         yield from super().enabled_scripts
 
 
@@ -184,7 +196,7 @@ class Capture():
         self._viewport: ViewportSize | None = None
         self._user_agent: str = ''
         self._timezone_id: str = ''
-        self._locale: str = ''
+        self._locale: str = 'en-US'
         self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = None
 
     def __prepare_proxy_playwright(self, proxy: str) -> ProxySettings:
@@ -462,21 +474,22 @@ class Capture():
         # NOTE: Which perms are supported by which browsers varies
         # See https://github.com/microsoft/playwright/issues/16577
         chromium_permissions = [
-            'geolocation',
-            'midi',
-            'midi-sysex',
-            'notifications',
-            'camera',
-            'microphone',
-            'background-sync',
-            'ambient-light-sensor',
             'accelerometer',
-            'gyroscope',
-            'magnetometer',
             'accessibility-events',
+            'ambient-light-sensor',
+            'background-sync',
+            'camera',
             'clipboard-read',
             'clipboard-write',
-            'payment-handler'
+            'geolocation',
+            'gyroscope',
+            'magnetometer',
+            'microphone',
+            'midi-sysex',
+            'midi',
+            'notifications',
+            'payment-handler',
+            'storage-access'
         ]
 
         firefox_permissions = ['geolocation', 'notifications']
@@ -588,6 +601,8 @@ class Capture():
             elif await page.get_by_test_id("uc-accept-all-button").is_visible():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
+            elif await page.locator('#axeptio_btn_acceptAll').is_visible():
+                await page.locator('#axeptio_btn_acceptAll').click(timeout=2000)
             else:
                 self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
@@ -633,6 +648,24 @@ class Capture():
         )
         self.logger.info('Piwik handler added')
 
+    async def __frame_consent(self, frame: Frame) -> bool:
+        """Search & Click content in iframes. Cannot easily use the locator handler for this without having many many handlers.
+        And the iframes don't have a title or a role to easily identify them so we just try with generic locators that vary by language."""
+        got_button: bool = False
+        if await frame.get_by_label("Alle akzeptieren").is_visible():
+            got_button = True
+            await frame.get_by_label("Alle akzeptieren").click(timeout=2000)
+        elif await frame.get_by_label("Accept & continue").is_visible():
+            got_button = True
+            await frame.get_by_label("Accept & continue").click(timeout=2000)
+        elif await frame.get_by_label("Accepter et continuer").is_visible():
+            got_button = True
+            await frame.get_by_label("Accepter et continuer").click(timeout=2000)
+        elif await frame.get_by_label("Accepteer").is_visible():
+            got_button = True
+            await frame.get_by_label("Accepteer").click(timeout=2000)
+        return got_button
+
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
                            page: Page | None=None, depth: int=0,
@@ -720,7 +753,6 @@ class Capture():
                 await self.__dialog_clickthrough(page)
 
             await stealth_async(page, PCStealthConfig())
-            # await stealth_async(page)
 
             page.set_default_timeout((self._capture_timeout - 2) * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
@@ -799,6 +831,8 @@ class Capture():
                             self.logger.warning(f'Target closed while resolving captcha on {url}: {e}')
                         except Error as e:
                             self.logger.warning(f'Error while resolving captcha on {url}: {e}')
+                        except (TimeoutError, asyncio.TimeoutError) as e:
+                            self.logger.warning(f'[Timeout] Error while resolving captcha on {url}: {e}')
                         except Exception as e:
                             self.logger.exception(f'General error with captcha solving on {url}: {e}')
                     # ======
@@ -897,16 +931,32 @@ class Capture():
                 #    self.logger.warning('Unable to move time forward.')
 
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
+                if allow_tracking:
+                    self.logger.debug('Check iFrames for button')
+                    for frame in page.frames:
+                        frame_title = await frame.title()
+                        self.logger.debug(f'Check button on {frame_title}')
+                        if await self.__frame_consent(frame):
+                            self.logger.debug(f'Got button on {frame_title}')
+                            await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
+                    self.logger.debug('Done with iFrames.')
+
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
                 await self._safe_wait(page)
+
                 self.logger.debug('Done with instrumentation, done with waiting.')
 
                 if content := await self._failsafe_get_content(page):
                     to_return['html'] = content
 
                 if 'html' in to_return and to_return['html'] is not None and with_favicon:
-                    to_return['potential_favicons'] = await self.get_favicons(page.url, to_return['html'])
-                    got_favicons = True
+                    try:
+                        to_return['potential_favicons'] = await self.get_favicons(page.url, to_return['html'])
+                        got_favicons = True
+                    except (TimeoutError, asyncio.TimeoutError) as e:
+                        self.logger.warning(f'[Timeout] Unable to get favicons: {e}')
+                    except Exception as e:
+                        self.logger.warning(f'Unable to get favicons: {e}')
 
                 to_return['last_redirected_url'] = page.url
                 to_return['png'] = await self._failsafe_get_screenshot(page)
@@ -1209,7 +1259,8 @@ class Capture():
         if self.proxy and self.proxy.get('server'):
             connector = ProxyConnector.from_url(self.proxy['server'])
 
-        async with aiohttp.ClientSession(connector=connector) as session:
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             while True:
                 try:
                     href = await main_frame.get_by_role("link", name="Alternatively, download audio as MP3").get_attribute("href")
@@ -1219,7 +1270,7 @@ class Capture():
                 if not href:
                     self.logger.warning('Unable to find download link for captcha.')
                     return False
-                async with session.get(href, timeout=10, ssl=False) as response:
+                async with session.get(href, ssl=False) as response:
                     response.raise_for_status()
                     mp3_content = await response.read()
                 with NamedTemporaryFile() as mp3_file, NamedTemporaryFile() as wav_file:
@@ -1417,7 +1468,8 @@ class Capture():
             return set()
         to_fetch, to_return = extracted_favicons
         to_fetch.add('/favicon.ico')
-        async with aiohttp.ClientSession(connector=connector) as session:
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             session.headers['user-agent'] = self.user_agent
             for u in to_fetch:
                 try:
@@ -1427,7 +1479,7 @@ class Capture():
                     if url_to_fetch in self._requests:
                         favicon = self._requests[url_to_fetch]
                     if not favicon:
-                        async with session.get(url_to_fetch, timeout=5, ssl=False) as favicon_response:
+                        async with session.get(url_to_fetch, ssl=False) as favicon_response:
                             favicon_response.raise_for_status()
                             favicon = await favicon_response.read()
                     if favicon:

{playwrightcapture-1.25.9.dist-info → playwrightcapture-1.25.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.9
+Version: 1.25.11
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -21,17 +21,17 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.4,<4.0.0) ; extra == "recaptcha"
-Requires-Dist: aiohttp-socks (>=0.8.4,<0.9.0)
-Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
+Requires-Dist: aiohttp-socks (>=0.9,<0.10)
+Requires-Dist: aiohttp[speedups] (>=3.10.3,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.45.1,<2.0.0)
+Requires-Dist: playwright (>=1.46.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.26,<2.0)
+Requires-Dist: puremagic (>=1.27,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=71.1.0,<72.0.0)
+Requires-Dist: setuptools (>=72.1.0,<73.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

playwrightcapture-1.25.11.dist-info/RECORD

@@ -0,0 +1,9 @@
+playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
+playwrightcapture/capture.py,sha256=Rmo_EVRlR9btsgE2H99OtGPRZwIe8RVq-JCc2GzUWiI,74446
+playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
+playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
+playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+playwrightcapture-1.25.11.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.11.dist-info/METADATA,sha256=nGuO6TAlz2lKM15HiIgZJ4iERLBO_AXNBBpgqo8nfhM,3172
+playwrightcapture-1.25.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.11.dist-info/RECORD,,

playwrightcapture-1.25.9.dist-info/RECORD

@@ -1,9 +0,0 @@
-playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=uS8e87-7jl8F7TgfzhKhlV4pGf8n6twu9rVzzlqIhXM,71671
-playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
-playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
-playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.25.9.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.25.9.dist-info/METADATA,sha256=7ds0ymzTNfkYLajoJTc-t1G4wOhPHRbxmg6CCZkMtUE,3173
-playwrightcapture-1.25.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.25.9.dist-info/RECORD,,