PlaywrightCapture 1.40.4__tar.gz → 1.40.6__tar.gz

{playwrightcapture-1.40.4 → playwrightcapture-1.40.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PlaywrightCapture
-Version: 1.40.4
+Version: 1.40.6
 Summary: A simple library to capture websites using playwright
 License-Expression: BSD-3-Clause
 License-File: LICENSE
@@ -25,7 +25,7 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.15.0)
 Requires-Dist: charset-normalizer (>=3.4.6,<4.0.0)
 Requires-Dist: dnspython (>=2.7.0,<3.0.0)
-Requires-Dist: lookyloo-models (>=0.3.0)
+Requires-Dist: lookyloo-models (>=0.3.1)
 Requires-Dist: orjson (>=3.11.4,<4.0.0)
 Requires-Dist: playwright (>=1.62.0)
 Requires-Dist: playwright-stealth (>=2.0.3)

{playwrightcapture-1.40.4 → playwrightcapture-1.40.6}/playwrightcapture/capture.py

@@ -238,7 +238,10 @@ class Capture():
 
     async def __aenter__(self) -> Capture:
         """Launch Playwright and the configured browser for this capture."""
-        self.playwright = await async_playwright().start()
+        try:
+            self.playwright = await async_playwright().start()
+        except Exception as e:
+            raise PlaywrightCaptureException('Unable to launch Playwright.') from e
 
         if self.device_name:
             if self.device_name in self.playwright.devices:
@@ -1299,6 +1302,49 @@ class Capture():
             except Exception as e:
                 self.logger.warning(f'Unable to get trusted timestamps: {e}')
 
+    def __check_local_url(self, url: str) -> tuple[bool, str]:
+        try:
+            _url = Url(url)
+        except ValueError as e:
+            self.logger.warning(f"Unable to parse URL: {url}: {e}")
+            return False, f"Unable to parse URL '{url}', blocked."
+
+        if not _url.host:
+            self.logger.warning(f"Missing Host: {url}")
+            return False, f"Missing host in URL '{url}', blocked."
+        try:
+            ip = ipaddress.ip_address(_url.host.try_into_ip())
+            if ip.is_global:
+                return True, "Global IP"
+            # Non-global IP
+            self.logger.warning(f"Attempt to open a non-public IP: {url}")
+            return False, f"Attempted to open {url}, blocked."
+        except ValueError:
+            # not an IP, continue to hostname
+            pass
+
+        try:
+            hostname = _url.host.try_into_hostname()
+            if not hostname:
+                self.logger.warning(f"Missing hostname: {url}")
+                return False, f"Missing hostname in URL '{url}', blocked."
+            elif str(hostname) == 'localhost':
+                self.logger.warning(f"Attempt to open localhost: {url}")
+                return False, "Attempted to open localhost, blocked."
+            elif suffix := hostname.suffix:
+                if str(suffix) == 'local':
+                    # Got a local domain
+                    self.logger.warning(f"Attempt to open a local domain: {url}")
+                    return False, f"Attempted to open {url}, blocked."
+            # NOTE: may want to add other things there, but do not want to trigger a domain resolution.
+            return True, "Public hostname"
+        except ValueError:
+            # not an hostname
+            pass
+        # should not happen (should have an IP or a hostname
+        self.logger.warning(f"Opening a weird URL: {url}")
+        return False, f"Attempted to open a weird URL '{url}', blocked."
+
     async def open_page(self, page: Page, url: str, errors: list[str], referer: str | None=None) -> None:
 
         async def catch_file_route(route: Route, request: Request) -> None:
@@ -1316,51 +1362,10 @@ class Capture():
                 await route.continue_()
             else:
                 # other URLs
-                try:
-                    _url = Url(request.url)
-                except ValueError as e:
-                    # some URLs will fail here when they have a specific scheme
-                    allowed_schemes = ('chrome-extension:', )
-                    if request.url.startswith(allowed_schemes):
-                        return await route.continue_()
-                    self.logger.warning(f"Unable to parse URL: {request.url}: {e}")
-                    return await route.fulfill(status=404, content_type="text/plain", body=f"Unable to parse URL '{request.url}', blocked.")
-
-                if not _url.host:
-                    self.logger.warning(f"Missing Host: {request.url}")
-                    return await route.fulfill(status=404, content_type="text/plain", body=f"Missing host in URL '{request.url}', blocked.")
-                try:
-                    ip = ipaddress.ip_address(_url.host.try_into_ip())
-                    if ip.is_global:
-                        return await route.continue_()
-                    # Non-global IP
-                    self.logger.warning(f"Attempt to open a non-public IP: {request.url}")
-                    return await route.fulfill(status=404, content_type="text/plain", body=f"Attempted to open {request.url}, blocked.")
-                except ValueError:
-                    # not an IP, continue to hostname
-                    pass
-
-                try:
-                    hostname = _url.host.try_into_hostname()
-                    if not hostname:
-                        self.logger.warning(f"Missing hostname: {request.url}")
-                        return await route.fulfill(status=404, content_type="text/plain", body=f"Missing hostname in URL '{request.url}', blocked.")
-                    elif str(hostname) == 'localhost':
-                        self.logger.warning(f"Attempt to open localhost: {request.url}")
-                        return await route.fulfill(status=404, content_type="text/plain", body="Attempted to open localhost, blocked.")
-                    elif suffix := hostname.suffix:
-                        if str(suffix) == 'local':
-                            # Got a local domain
-                            self.logger.warning(f"Attempt to open a local domain: {request.url}")
-                            return await route.fulfill(status=404, content_type="text/plain", body=f"Attempted to open {request.url}, blocked.")
-                    # NOTE: may want to add other things there, but do not want to trigger a domain resolution.
-                    return await route.continue_()
-                except ValueError:
-                    # not an hostname
-                    pass
-                # should not happen (should have an IP or a hostname
-                self.logger.warning(f"Opening a weird URL: {request.url}")
-                return await route.fulfill(status=404, content_type="text/plain", body=f"Attempted to open a weird URL '{request.url}', blocked.")
+                not_local, message = self.__check_local_url(request.url)
+                if not_local:
+                    await route.continue_()
+                await route.fulfill(status=404, content_type="text/plain", body=message)
 
         if self.only_global_lookup:
             # routes are processed in reverse order, the latest one in this list is processed first
@@ -1368,12 +1373,20 @@ class Capture():
             await page.route("**/*", handler=catch_local_route)  # last one to be checked
             # unless they're the actual URL we're rendering, block.
             await page.route("file:**/*", handler=catch_file_route)
-            # blob URLs, they're always fine as their content is created in memory by the browser
-            # https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/blob
-            await page.route("blob:**/*", lambda route: route.continue_())
-            # filesystem URLs, they point to cache or a virtual file system that is browser managed
-            # https://developer.mozilla.org/en-US/docs/Web/API/File_System_API
-            await page.route("filesystem:**/*", lambda route: route.continue_())
+
+            # these schemes are allowed to continue by default, they do not hit anything private
+            allowed_schemes = [
+                # blob URLs, they're always fine as their content is created in memory by the browser
+                # https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/blob
+                "blob:**/*",
+                # filesystem URLs, they point to cache or a virtual file system that is browser managed
+                # https://developer.mozilla.org/en-US/docs/Web/API/File_System_API
+                "filesystem:**/*",
+                # also purely internal to the browser
+                "chrome-extension:**/*"
+            ]
+            for scheme in allowed_schemes:
+                await page.route(scheme, lambda route: route.continue_())
 
         try:
             await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else '')
@@ -1808,7 +1821,7 @@ class Capture():
                     # The frame is loaded, if we get no content there, only retry one more time.
                     tries -= 1
                 except Exception as e:
-                    self.logger.warning(f'The Playwright Page is in a broken state: {e}.')
+                    self.logger.warning(f'The Playwright Page is in a broken state (unable to get content): {e}.')
                     break
             tries -= 1
             if tries > 0:
@@ -1816,7 +1829,7 @@ class Capture():
                     await self._wait_for_random_timeout(page, 2)
                     await self._safe_wait(page, 2)
                 except Exception as e:
-                    self.logger.warning(f'The Playwright Page is in a broken state: {e}.')
+                    self.logger.warning(f'The Playwright Page is in a broken state (unable to wait): {e}.')
                     break
 
         # got no content
@@ -2125,17 +2138,25 @@ class Capture():
         else:
             f_name = frame.name
         frame_id = f'{f_name}@{frame.url}'
+        to_return: FramesResponse = {'name': frame.name, 'url': frame.url, 'content': ''}
         if frame.is_detached():
             self.logger.debug(f'{frame_id} is detached.')
-        to_return: FramesResponse = {'name': frame.name, 'url': frame.url, 'content': await self._failsafe_get_content(frame)}
+        else:
+            to_return['content'] = await self._failsafe_get_content(frame)
+            if frame.child_frames:
+                all_children = await asyncio.gather(*[self.make_frame_tree(child) for child in frame.child_frames], return_exceptions=True)
+                # separate exceptions vs. actual responses
+                for e in [be for be in all_children if isinstance(be, BaseException)]:
+                    self.logger.warning(f'Exception in {f_name}: {e}')
+
+                to_return['children'] = [fc for fc in all_children if not isinstance(fc, BaseException)]
+
         if not to_return.get('content'):
             if frame.url in ['about:blank', 'about:srcdoc', '', None, 'chrome-error://chromewebdata/']:
                 # too noisy in the warnings
                 self.logger.debug(f'Got no content for {frame_id}.')
             else:
                 self.logger.warning(f'Got no content for {frame_id}.')
-        if frame.child_frames:
-            to_return['children'] = await asyncio.gather(*[self.make_frame_tree(child) for child in frame.child_frames])
         return to_return
 
     # Method copied from HAR2Tree
@@ -2253,6 +2274,12 @@ class Capture():
                 try:
                     self.logger.debug(f'Attempting to fetch favicon from {u}.')
                     url_to_fetch = urljoin(rendered_url, u)
+                    if self.only_global_lookup:
+                        not_local, message = self.__check_local_url(url_to_fetch)
+                        if not_local is False:
+                            # got a local URL
+                            self.logger.warning(f'Attempted to get a local favicon: {message}')
+                            continue
                     favicon = b''
                     if url_to_fetch in self._requests:
                         favicon = self._requests[url_to_fetch]
@@ -2300,7 +2327,7 @@ class Capture():
         # use the same technique as in lookyloo to resolve many domains in parallel
         semaphore = asyncio.Semaphore(30)
         all_requests = [resolver.resolve(hostname, semaphore) for hostname in hostnames]
-        await asyncio.gather(*all_requests)
+        await asyncio.gather(*all_requests, return_exceptions=True)
         self.logger.debug('Resolved all domains through the proxy.')
         for entry in harfile['log']['entries']:
             if entry['request']['url']:

{playwrightcapture-1.40.4 → playwrightcapture-1.40.6}/playwrightcapture/helpers.py

@@ -2,11 +2,13 @@
 
 from __future__ import annotations
 
+import asyncio
+from functools import cache
 import sys
 
 from collections import defaultdict
 
-from playwright.sync_api import sync_playwright
+from playwright.async_api import async_playwright
 
 from .exceptions import UnknownPlaywrightDeviceType
 
@@ -26,10 +28,15 @@ class PlaywrightDevice(TypedDict):
     default_browser_type: str
 
 
+@cache
 def get_devices(in_testsuite: bool=False) -> dict[str, dict[str, dict[str, PlaywrightDevice]]]:
+
+    async def get_devices() -> dict[str, PlaywrightDevice]:
+        async with async_playwright() as playwright:
+            return playwright.devices
+
     to_return: dict[str, dict[str, dict[str, PlaywrightDevice]]] = {'desktop': defaultdict(dict), 'mobile': defaultdict(dict)}
-    with sync_playwright() as playwright:
-        devices: dict[str, PlaywrightDevice] = playwright.devices
+    devices: dict[str, PlaywrightDevice] = asyncio.run(get_devices())
     for device_name, settings in devices.items():
         splitted_name = device_name.split(' ')
         if splitted_name[0] == 'Desktop':

{playwrightcapture-1.40.4 → playwrightcapture-1.40.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "PlaywrightCapture"
-version = "1.40.4"
+version = "1.40.6"
 description = "A simple library to capture websites using playwright"
 authors = [
     {name="Raphaël Vinot", email= "raphael.vinot@circl.lu"}
@@ -25,7 +25,7 @@ dependencies = [
     "rfc3161-client (>=1.0.4,<2.0.0)",
     "orjson (>=3.11.4,<4.0.0)",
     "pure-magic-rs (>=0.4.3)",
-    "lookyloo-models (>=0.3.0)",
+    "lookyloo-models (>=0.3.1)",
     "charset-normalizer (>=3.4.6,<4.0.0)",
     "pyfaup-rs (>=0.4.6,<0.5.0)"
 ]
@@ -53,7 +53,7 @@ recaptcha = [
 types-beautifulsoup4 = "^4.12.0.20250516"
 pytest = "^9.1.1"
 mypy = "^2.3.0"
-types-dateparser = "^1.4.1.20260617"
+types-dateparser = "^1.4.1.20260806"
 types-pytz = "^2026.3.1.20260727"