PlaywrightCapture 1.31.3__tar.gz → 1.31.5__tar.gz

{playwrightcapture-1.31.3 → playwrightcapture-1.31.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: PlaywrightCapture
-Version: 1.31.3
+Version: 1.31.5
 Summary: A simple library to capture websites using playwright
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -20,14 +20,14 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.14.3) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.10.1)
-Requires-Dist: aiohttp[speedups] (>=3.12.13)
+Requires-Dist: aiohttp[speedups] (>=3.12.14)
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
 Requires-Dist: dateparser (>=1.2.2)
 Requires-Dist: dnspython (>=2.7.0,<3.0.0)
-Requires-Dist: playwright (>=1.53.0)
+Requires-Dist: playwright (>=1.54.0)
 Requires-Dist: playwright-stealth (>=2)
-Requires-Dist: puremagic (>=1.29)
+Requires-Dist: puremagic (>=1.30)
 Requires-Dist: pydub (>=0.25.1) ; (python_version < "3.10") and (extra == "recaptcha")
 Requires-Dist: pydub-ng (>=0.2.0) ; (python_version >= "3.10") and (extra == "recaptcha")
 Requires-Dist: python-socks (>=2.7.1,<3.0.0)

{playwrightcapture-1.31.3 → playwrightcapture-1.31.5}/playwrightcapture/capture.py

@@ -118,7 +118,8 @@ class Capture():
                  proxy: str | dict[str, str] | None=None,
                  socks5_dns_resolver: str | list[str] | None=None,
                  general_timeout_in_sec: int | None=None, loglevel: str | int='INFO',
-                 uuid: str | None=None, headless: bool=True):
+                 uuid: str | None=None, headless: bool=True,
+                 *, init_script: str | None=None):
         """Captures a page with Playwright.
 
         :param browser: The browser to use for the capture.
@@ -129,6 +130,7 @@ class Capture():
         :param loglevel: Python loglevel
         :param uuid: The UUID of the capture.
         :param headless: Whether to run the browser in headless mode. WARNING: requires to run in a graphical environment.
+        :param init_script: An optional JavaScript that will be executed on each page - See https://playwright.dev/python/docs/api/class-browsercontext#browser-context-add-init-script
         """
         master_logger = logging.getLogger('playwrightcapture')
         master_logger.setLevel(loglevel)
@@ -179,6 +181,8 @@ class Capture():
         self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = None
         self._java_script_enabled = True
 
+        self._init_script = init_script
+
     def __prepare_proxy_playwright(self, proxy: str) -> ProxySettings:
         splitted = urlsplit(proxy)
         if splitted.username and splitted.password:
@@ -460,6 +464,9 @@ class Capture():
         )
         self.context.set_default_timeout(self._capture_timeout * 1000)
 
+        if self._init_script:
+            await self.context.add_init_script(script=self._init_script)
+
         # very quick and dirty get a platform from the UA so it's not always Win32
         # This this is deprecated and not very important.
         # Ref: https://developer.mozilla.org/en-US/docs/Web/API/Navigator/platform
@@ -491,7 +498,7 @@ class Capture():
                # 'navigator_vendor': False,  # It's set correctly by playwright
                'navigator_webdriver': True,
                # 'sec_ch_ua': True,
-               # 'webgl_vendor': False,  # It's set correctly by playwright
+               'webgl_vendor': True,  # It's not net correctly by playwright in headless mode.
 
                # ## Overwrite the default values
                'navigator_languages_override': None,
@@ -501,10 +508,10 @@ class Capture():
                # 'sec_ch_ua_override': Stealth._get_greased_chrome_sec_ua_ch(ua),
                # 'webgl_renderer_override': None,
                # 'webgl_vendor_override': None,
-               })
 
-        # stealth.hook_playwright_context(self.playwright)
-        await stealth.apply_stealth_async(self.context)
+               # For testing
+               # 'script_logging': True,
+               })
 
         if self.cookies:
             try:
@@ -552,6 +559,9 @@ class Capture():
         elif self.browser_name == 'chromium':
             await self.context.grant_permissions(chromium_permissions)
 
+        # Apply stealth
+        await stealth.apply_stealth_async(self.context)
+
     async def __cloudflare_bypass_attempt(self, page: Page) -> None:
         # This method aims to bypass cloudflare checks, but it mostly doesn't work.
         max_tries = 5
@@ -1354,7 +1364,11 @@ class Capture():
             return href
 
         urls: set[str] = set()
-        soup = BeautifulSoup(rendered_html, "lxml")
+        try:
+            soup = BeautifulSoup(rendered_html, "lxml")
+        except Exception as e:
+            self.logger.info(f'Unable to parse HTML: {e}')
+            soup = BeautifulSoup(rendered_html, "html.parser")
 
         rendered_hostname = urlparse(rendered_url).hostname
         # The simple ones: the links.

{playwrightcapture-1.31.3 → playwrightcapture-1.31.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "PlaywrightCapture"
-version = "1.31.3"
+version = "1.31.5"
 description = "A simple library to capture websites using playwright"
 authors = [
     {name="Raphaël Vinot", email= "raphael.vinot@circl.lu"}
@@ -12,16 +12,16 @@ requires-python = ">=3.9,<4.0"
 dynamic = [ "classifiers" ]
 
 dependencies = [
-    "playwright (>=1.53.0)",
+    "playwright (>=1.54.0)",
     "dateparser (>=1.2.2)",
     "beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)",
     "w3lib (>=2.3.1)",
     "tzdata (>=2025.2)",
     "playwright-stealth (>=2)",
     "setuptools (>=80.9.0)",
-    "puremagic (>=1.29)",
+    "puremagic (>=1.30)",
     "async-timeout (>=5.0.1) ; python_version < \"3.11\"",
-    "aiohttp[speedups] (>=3.12.13)",
+    "aiohttp[speedups] (>=3.12.14)",
     "aiohttp-socks (>=0.10.1)",
     "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.12\"",
     "dnspython (>=2.7.0,<3.0.0)",
@@ -51,8 +51,8 @@ recaptcha = [
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20250516"
 pytest = "^8.4.1"
-mypy = "^1.16.1"
-types-dateparser = "^1.2.0.20250601"
+mypy = "^1.17.0"
+types-dateparser = "^1.2.2.20250627"
 types-pytz = "^2025.2.0.20250516"