PlaywrightCapture 1.25.0__tar.gz → 1.25.2__tar.gz

{playwrightcapture-1.25.0 → playwrightcapture-1.25.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.0
+Version: 1.25.2
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -31,7 +31,7 @@ Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.25,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=70.2.0,<71.0.0)
+Requires-Dist: setuptools (>=70.3.0,<71.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.0 → playwrightcapture-1.25.2}/playwrightcapture/capture.py

@@ -13,10 +13,11 @@ import sys
 import time
 
 from base64 import b64decode
+from dataclasses import dataclass
 from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
-from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping
+from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping, Generator
 from urllib.parse import urlparse, unquote, urljoin
 from zipfile import ZipFile
 
@@ -30,7 +31,7 @@ from charset_normalizer import from_bytes
 from playwright._impl._errors import TargetClosedError
 from playwright.async_api import async_playwright, Frame, Error, Page, Download, Request
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-from playwright_stealth import stealth_async  # type: ignore[import-untyped]
+from playwright_stealth import stealth_async, StealthConfig  # type: ignore[import-untyped]
 from puremagic import PureError, from_string  # type: ignore[import-untyped]
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
@@ -95,6 +96,31 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
         return msg, kwargs
 
 
+@dataclass
+class PCStealthConfig(StealthConfig):  # type: ignore[misc]
+
+    @property
+    def enabled_scripts(self) -> Generator[str, None, None]:
+        self.webdriver = True
+        self.webgl_vendor = True
+        self.chrome_app = True
+        self.chrome_csi = True
+        self.chrome_load_times = True
+        self.chrome_runtime = True
+        self.iframe_content_window = True
+        self.media_codecs = True
+        self.navigator_hardware_concurrency = 4
+        self.navigator_languages = False  # Causes issue
+        self.navigator_permissions = True
+        self.navigator_platform = True
+        self.navigator_plugins = True
+        self.navigator_user_agent = False  # Causes issues
+        self.navigator_vendor = False  # Causes issues
+        self.outerdimensions = True
+        self.hairline = True
+        yield from super().enabled_scripts
+
+
 class Capture():
 
     _browsers: list[BROWSER] = ['chromium', 'firefox', 'webkit']
@@ -171,7 +197,8 @@ class Capture():
             raise UnknownPlaywrightBrowser(f'Incorrect browser name {self.browser_name}, must be in {", ".join(self._browsers)}')
 
         self.browser = await self.playwright[self.browser_name].launch(
-            proxy=self.proxy if self.proxy else None
+            proxy=self.proxy if self.proxy else None,
+            # headless=False
         )
 
         # Set of URLs that were captured in that context
@@ -385,6 +412,7 @@ class Capture():
         self.context = await self.browser.new_context(
             record_har_path=self._temp_harfile.name,
             ignore_https_errors=True,
+            bypass_csp=True,
             http_credentials=self.http_credentials if self.http_credentials else None,
             user_agent=self.user_agent if self.user_agent else device_context_settings.pop('user_agent', None),
             locale=self.locale if self.locale else None,
@@ -472,7 +500,7 @@ class Capture():
         async def handler() -> None:
             self.logger.debug('Didomi dialog found, clicking through.')
             if await page.locator("#didomi-notice-agree-button").is_visible():
-                await page.locator("#didomi-notice-agree-button").click(timeout=2000)
+                await page.locator("#didomi-notice-agree-button").click(timeout=30000)
 
         await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
@@ -654,6 +682,7 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
+                await page.clock.install()
             except Error as e:
                 self.logger.warning(f'The context is in a broken state: {e}')
                 self.should_retry = True
@@ -671,7 +700,9 @@ class Capture():
                 await self.__dialog_alert_dialog_clickthrough(page)
                 await self.__dialog_clickthrough(page)
 
-            await stealth_async(page)
+            await stealth_async(page, PCStealthConfig())
+            # await stealth_async(page)
+
             page.set_default_timeout((self._capture_timeout - 2) * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
             page.on("requestfinished", store_request)
@@ -718,6 +749,7 @@ class Capture():
                 else:
                     raise initial_error
             else:
+                await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
                 try:
                     await page.bring_to_front()
                     self.logger.debug('Page moved to front.')
@@ -753,28 +785,34 @@ class Capture():
                 # check if we have anything on the page. If we don't, the page is not working properly.
                 if await self._failsafe_get_content(page):
                     self.logger.debug('Got rendered content')
+
+                    # move mouse
+                    await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
+                    self.logger.debug('Moved mouse.')
+                    await self._wait_for_random_timeout(page, 5)
+                    self.logger.debug('Keep going after moving mouse.')
+
                     if allow_tracking:
-                        await self._wait_for_random_timeout(page, 2)
+                        await self._wait_for_random_timeout(page, 5)
                         # This event is required trigger the add_locator_handler
                         try:
                             if await page.locator("body").first.is_visible():
-                                await page.locator("body").first.click(button="right", timeout=5000)
+                                self.logger.debug('Got body.')
+                                await page.locator("body").first.click(button="right",
+                                                                       timeout=5000,
+                                                                       delay=50)
+                                self.logger.debug('Clicked on body.')
                         except Exception as e:
                             self.logger.warning(f'Could not find body: {e}')
 
-                    # move mouse
-                    await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
-                    self.logger.debug('Moved mouse.')
-                    await self._wait_for_random_timeout(page, 2)
-                    self.logger.debug('Keep going after moving mouse.')
-
                     if parsed_url.fragment:
                         # We got a fragment, make sure we go to it and scroll only a little bit.
                         fragment = unquote(parsed_url.fragment)
                         try:
                             await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=3000)
                             await self._wait_for_random_timeout(page, 2)
-                            await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
+                            async with timeout(3):
+                                await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
                             self.logger.debug('Jumped to fragment.')
                         except PlaywrightTimeoutError as e:
                             self.logger.info(f'Unable to go to fragment "{fragment}" (timeout): {e}')
@@ -782,14 +820,20 @@ class Capture():
                             self.logger.warning(f'Target closed, unable to go to fragment "{fragment}": {e}')
                         except Error as e:
                             self.logger.exception(f'Unable to go to fragment "{fragment}": {e}')
+                        except TimeoutError:
+                            self.logger.debug('Unable to scroll due to timeout')
                     else:
                         # scroll more
                         try:
-                            # NOTE using page.mouse.wheel causes the instrumentation to fail, sometimes
-                            await page.mouse.wheel(delta_y=random.uniform(1500, 3000), delta_x=0)
+                            # NOTE using page.mouse.wheel causes the instrumentation to fail, sometimes.
+                            #   2024-07-08: Also, it sometimes get stuck.
+                            async with timeout(3):
+                                await page.mouse.wheel(delta_y=random.uniform(1500, 3000), delta_x=0)
                             self.logger.debug('Scrolled down.')
                         except Error as e:
                             self.logger.debug(f'Unable to scroll: {e}')
+                        except TimeoutError:
+                            self.logger.debug('Unable to scroll due to timeout')
 
                     await self._wait_for_random_timeout(page, 3)
                     self.logger.debug('Keep going after moving on page.')
@@ -802,7 +846,6 @@ class Capture():
                         self.logger.debug('PageDown on keyboard')
                     except Error as e:
                         self.logger.debug(f'Unable to use keyboard: {e}')
-
                 if self.wait_for_download > 0:
                     self.logger.info('Waiting for download to finish...')
                     await self._safe_wait(page, 20)
@@ -821,6 +864,10 @@ class Capture():
                                 z.writestr(f'{i}_{filename}', file_content)
                         to_return["downloaded_file"] = mem_zip.getvalue()
 
+                # fast forward 30s
+                await page.clock.run_for("30")
+                self.logger.debug('Moved time forward.')
+
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
                 await self._safe_wait(page)
@@ -993,12 +1040,12 @@ class Capture():
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
         self.logger.debug("Capturing a screenshot of the full page.")
         try:
-            return await page.screenshot(full_page=True, timeout=5000)
+            return await page.screenshot(full_page=True, timeout=10000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css", timeout=5000)
+            return await page.screenshot(full_page=True, scale="css", timeout=30000)
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 

{playwrightcapture-1.25.0 → playwrightcapture-1.25.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.25.0"
+version = "1.25.2"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -28,7 +28,7 @@ SpeechRecognition = {version = "^3.10.4", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^70.2.0"
+setuptools = "^70.3.0"
 puremagic = "^1.25"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 aiohttp = {extras = ["speedups"], version = "^3.9.5"}