PlaywrightCapture 1.24.11__py3-none-any.whl → 1.25.1__py3-none-any.whl

playwrightcapture/capture.py

@@ -13,10 +13,11 @@ import sys
 import time
 
 from base64 import b64decode
+from dataclasses import dataclass
 from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
-from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping
+from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping, Generator
 from urllib.parse import urlparse, unquote, urljoin
 from zipfile import ZipFile
 
@@ -30,7 +31,7 @@ from charset_normalizer import from_bytes
 from playwright._impl._errors import TargetClosedError
 from playwright.async_api import async_playwright, Frame, Error, Page, Download, Request
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-from playwright_stealth import stealth_async  # type: ignore[import-untyped]
+from playwright_stealth import stealth_async, StealthConfig  # type: ignore[import-untyped]
 from puremagic import PureError, from_string  # type: ignore[import-untyped]
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
@@ -95,6 +96,31 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
         return msg, kwargs
 
 
+@dataclass
+class PCStealthConfig(StealthConfig):  # type: ignore[misc]
+
+    @property
+    def enabled_scripts(self) -> Generator[str, None, None]:
+        self.webdriver = True
+        self.webgl_vendor = True
+        self.chrome_app = True
+        self.chrome_csi = True
+        self.chrome_load_times = True
+        self.chrome_runtime = True
+        self.iframe_content_window = True
+        self.media_codecs = True
+        self.navigator_hardware_concurrency = 4
+        self.navigator_languages = False  # Causes issue
+        self.navigator_permissions = True
+        self.navigator_platform = True
+        self.navigator_plugins = True
+        self.navigator_user_agent = False  # Causes issues
+        self.navigator_vendor = False  # Causes issues
+        self.outerdimensions = True
+        self.hairline = True
+        yield from super().enabled_scripts
+
+
 class Capture():
 
     _browsers: list[BROWSER] = ['chromium', 'firefox', 'webkit']
@@ -171,7 +197,8 @@ class Capture():
             raise UnknownPlaywrightBrowser(f'Incorrect browser name {self.browser_name}, must be in {", ".join(self._browsers)}')
 
         self.browser = await self.playwright[self.browser_name].launch(
-            proxy=self.proxy if self.proxy else None
+            proxy=self.proxy if self.proxy else None,
+            # headless=False
         )
 
         # Set of URLs that were captured in that context
@@ -385,6 +412,7 @@ class Capture():
         self.context = await self.browser.new_context(
             record_har_path=self._temp_harfile.name,
             ignore_https_errors=True,
+            bypass_csp=True,
             http_credentials=self.http_credentials if self.http_credentials else None,
             user_agent=self.user_agent if self.user_agent else device_context_settings.pop('user_agent', None),
             locale=self.locale if self.locale else None,
@@ -472,7 +500,7 @@ class Capture():
         async def handler() -> None:
             self.logger.debug('Didomi dialog found, clicking through.')
             if await page.locator("#didomi-notice-agree-button").is_visible():
-                await page.locator("#didomi-notice-agree-button").click(timeout=2000)
+                await page.locator("#didomi-notice-agree-button").click(timeout=30000)
 
         await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
@@ -521,7 +549,7 @@ class Capture():
             elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (alert dialog), but no button to click through.')
 
         await page.add_locator_handler(
             page.get_by_role("alertdialog").last,
@@ -542,7 +570,7 @@ class Capture():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
             page.get_by_role("dialog").last,
             handler,
@@ -584,7 +612,7 @@ class Capture():
             handler,
             times=1, no_wait_after=True
         )
-        self.logger.info('Yahoo handler added')
+        self.logger.info('Piwik handler added')
 
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
@@ -654,6 +682,7 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
+                await page.clock.install()
             except Error as e:
                 self.logger.warning(f'The context is in a broken state: {e}')
                 self.should_retry = True
@@ -671,7 +700,9 @@ class Capture():
                 await self.__dialog_alert_dialog_clickthrough(page)
                 await self.__dialog_clickthrough(page)
 
-            await stealth_async(page)
+            await stealth_async(page, PCStealthConfig())
+            # await stealth_async(page)
+
             page.set_default_timeout((self._capture_timeout - 2) * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
             page.on("requestfinished", store_request)
@@ -718,11 +749,11 @@ class Capture():
                 else:
                     raise initial_error
             else:
+                await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
                 try:
                     await page.bring_to_front()
                     self.logger.debug('Page moved to front.')
                 except Error as e:
-                    self.should_retry = True
                     self.logger.warning('Page in a broken state.')
                     raise e
 
@@ -754,25 +785,40 @@ class Capture():
                 # check if we have anything on the page. If we don't, the page is not working properly.
                 if await self._failsafe_get_content(page):
                     self.logger.debug('Got rendered content')
-                    if allow_tracking:
-                        await self._wait_for_random_timeout(page, 2)
-                        # This event is required trigger the add_locator_handler
-                        if await page.locator("body").first.is_visible():
-                            await page.locator("body").first.click(button="right", timeout=2000)
 
                     # move mouse
                     await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
                     self.logger.debug('Moved mouse.')
-                    await self._wait_for_random_timeout(page, 2)
+                    await self._wait_for_random_timeout(page, 5)
                     self.logger.debug('Keep going after moving mouse.')
 
+                    if allow_tracking:
+                        await self._wait_for_random_timeout(page, 5)
+                        # This event is required trigger the add_locator_handler
+                        try:
+                            if await page.locator("body").first.is_visible():
+                                self.logger.debug('Got body.')
+                                await page.locator("body").first.click(button="right",
+                                                                       timeout=5000,
+                                                                       delay=50)
+                                self.logger.debug('Clicked on body.')
+                        except Exception as e:
+                            self.logger.warning(f'Could not find body: {e}')
+
+                    # fast forward 30s
+                    await page.clock.run_for(10000)
+                    await page.clock.resume()
+                    await self._wait_for_random_timeout(page, 5)  # Wait 5 sec
+                    self.logger.warning('Moved time forward.')
+
                     if parsed_url.fragment:
                         # We got a fragment, make sure we go to it and scroll only a little bit.
                         fragment = unquote(parsed_url.fragment)
                         try:
                             await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=3000)
                             await self._wait_for_random_timeout(page, 2)
-                            await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
+                            async with timeout(3):
+                                await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
                             self.logger.debug('Jumped to fragment.')
                         except PlaywrightTimeoutError as e:
                             self.logger.info(f'Unable to go to fragment "{fragment}" (timeout): {e}')
@@ -780,14 +826,20 @@ class Capture():
                             self.logger.warning(f'Target closed, unable to go to fragment "{fragment}": {e}')
                         except Error as e:
                             self.logger.exception(f'Unable to go to fragment "{fragment}": {e}')
+                        except TimeoutError:
+                            self.logger.debug('Unable to scroll due to timeout')
                     else:
                         # scroll more
                         try:
-                            # NOTE using page.mouse.wheel causes the instrumentation to fail, sometimes
-                            await page.mouse.wheel(delta_y=random.uniform(1500, 3000), delta_x=0)
+                            # NOTE using page.mouse.wheel causes the instrumentation to fail, sometimes.
+                            #   2024-07-08: Also, it sometimes get stuck.
+                            async with timeout(3):
+                                await page.mouse.wheel(delta_y=random.uniform(1500, 3000), delta_x=0)
                             self.logger.debug('Scrolled down.')
                         except Error as e:
                             self.logger.debug(f'Unable to scroll: {e}')
+                        except TimeoutError:
+                            self.logger.debug('Unable to scroll due to timeout')
 
                     await self._wait_for_random_timeout(page, 3)
                     self.logger.debug('Keep going after moving on page.')
@@ -800,7 +852,6 @@ class Capture():
                         self.logger.debug('PageDown on keyboard')
                     except Error as e:
                         self.logger.debug(f'Unable to use keyboard: {e}')
-
                 if self.wait_for_download > 0:
                     self.logger.info('Waiting for download to finish...')
                     await self._safe_wait(page, 20)
@@ -920,19 +971,20 @@ class Capture():
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
                             'Target page, context or browser has been closed',
-                            'Protocol error (Page.bringToFront): Not attached to an active page',
                             'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
                             'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
-                            'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Handshake failed',
                             'Peer sent fatal TLS alert: Internal error',
+                            'Peer sent fatal TLS alert: The server name sent was not recognized',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
                             'Error receiving data: Connection reset by peer',
                             'Internal SOCKSv5 proxy server error.',
                             'Host unreachable through SOCKSv5 server.',
-                            'HTTP/2 Error: NO_ERROR']:
+                            'HTTP/2 Error: NO_ERROR',
+                            'HTTP/2 Error: PROTOCOL_ERROR']:
                 # Other errors, let's give it another shot
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
@@ -942,16 +994,18 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
                             'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
-                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
-                # No need to retry, the credentials are wrong/missing.
+                            'net::ERR_CERT_DATE_INVALID',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH',
+                            'net::ERR_UNSAFE_PORT']:
+                # No need to retry, the credentials/certs are wrong/missing.
                 pass
-            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to', 'Page.bringToFront']]):
                 self.should_retry = True
             elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
                 pass
             else:
                 # Unexpected ones
-                self.logger.exception(f'Something went poorly with {url}: {e.message}')
+                self.logger.exception(f'Something went poorly with {url}: "{e.name}" - {e.message}')
         except Exception as e:
             # we may get a non-playwright exception to.
             # The ones we try to handle here should be treated as if they were.
@@ -988,12 +1042,12 @@ class Capture():
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
         self.logger.debug("Capturing a screenshot of the full page.")
         try:
-            return await page.screenshot(full_page=True, timeout=5000)
+            return await page.screenshot(full_page=True, timeout=10000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css", timeout=5000)
+            return await page.screenshot(full_page=True, scale="css", timeout=30000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 
@@ -1194,8 +1248,13 @@ class Capture():
                 'net::ERR_EMPTY_RESPONSE',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
+                'net::ERR_INVALID_REDIRECT',
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
+                'net::ERR_NETWORK_ACCESS_DENIED',
+                'net::ERR_QUIC_PROTOCOL_ERROR',
+                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
+                'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
                 'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
                 'net::ERR_SSL_PROTOCOL_ERROR',

{playwrightcapture-1.24.11.dist-info → playwrightcapture-1.25.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.11
+Version: 1.25.1
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,14 +26,14 @@ Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.44.0,<2.0.0)
+Requires-Dist: playwright (>=1.45.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.23,<2.0)
+Requires-Dist: puremagic (>=1.25,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=70.0.0,<71.0.0)
+Requires-Dist: setuptools (>=70.3.0,<71.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
-Requires-Dist: w3lib (>=2.1.2,<3.0.0)
+Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture
 Description-Content-Type: text/markdown
 

playwrightcapture-1.25.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
+playwrightcapture/capture.py,sha256=dIsqmpBiAfkjv400Pj_WtC_69se19ARcW3eDx0LTiuI,69525
+playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
+playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
+playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+playwrightcapture-1.25.1.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.1.dist-info/METADATA,sha256=PAPj-xddph1jI5cah7404Cqu2_6xvwHq4BKotq-Gyhk,3173
+playwrightcapture-1.25.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.1.dist-info/RECORD,,

playwrightcapture-1.24.11.dist-info/RECORD

@@ -1,9 +0,0 @@
-playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=bEbKQKnUT4mPXzgQF8NI6hKK1LmVcBnt2MXB9BXNwgQ,66759
-playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
-playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
-playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.24.11.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.24.11.dist-info/METADATA,sha256=c5hVyRttyjdcwwG-CVfDqmqm_a33VWePttyxKtaBJyw,3174
-playwrightcapture-1.24.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.24.11.dist-info/RECORD,,