PlaywrightCapture 1.41.3__tar.gz → 1.41.5__tar.gz

{playwrightcapture-1.41.3 → playwrightcapture-1.41.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PlaywrightCapture
-Version: 1.41.3
+Version: 1.41.5
 Summary: A simple library to capture websites using playwright
 License-Expression: BSD-3-Clause
 License-File: LICENSE
@@ -25,11 +25,11 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.15.0)
 Requires-Dist: charset-normalizer (>=3.4.6,<4.0.0)
 Requires-Dist: dnspython (>=2.7.0,<3.0.0)
-Requires-Dist: lookyloo-models (>=0.3.1)
-Requires-Dist: orjson (>=3.11.4,<4.0.0)
-Requires-Dist: playwright (>=1.62.0)
+Requires-Dist: lookyloo-models (>=0.4.3)
+Requires-Dist: orjson (>=3.12,<4.0.0)
+Requires-Dist: playwright (>=1.63.0)
 Requires-Dist: playwright-stealth (>=2.0.3)
-Requires-Dist: pure-magic-rs (>=0.4.3)
+Requires-Dist: pure-magic-rs (>=0.5)
 Requires-Dist: pydub-ng (>=0.2.0) ; extra == "recaptcha"
 Requires-Dist: pyfaup-rs (>=0.4.6,<0.5.0)
 Requires-Dist: python-socks (>=3.0.0,<4.0.0)

{playwrightcapture-1.41.3 → playwrightcapture-1.41.5}/playwrightcapture/capture.py

@@ -18,7 +18,7 @@ from base64 import b64decode, b64encode
 from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
-from typing import Any, Literal, TYPE_CHECKING, overload
+from typing import Any, Literal, TYPE_CHECKING
 from collections.abc import Awaitable, Callable, MutableMapping
 from urllib.parse import urlparse, unquote, urljoin, urlsplit, urlunsplit, parse_qs, unquote_plus
 from zipfile import ZipFile
@@ -29,7 +29,7 @@ import orjson
 from aiohttp_socks import ProxyConnector
 from bs4 import BeautifulSoup
 from charset_normalizer import from_bytes
-from lookyloo_models import Cookie, CaptureSettings
+from lookyloo_models import (Cookie, CaptureSettings, ViewportSettings, ProxySettings)
 from playwright._impl._errors import TargetClosedError
 from playwright.async_api import async_playwright, Frame, Error, Page, Download, Request, Route, ConsoleMessage
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
@@ -43,9 +43,6 @@ from w3lib.url import canonicalize_url, safe_url_string
 from .exceptions import UnknownPlaywrightBrowser, UnknownPlaywrightDevice, InvalidPlaywrightParameter, PlaywrightCaptureException
 from .socks5dnslookup import Socks5Resolver
 
-from zoneinfo import available_timezones
-all_timezones_set = available_timezones()
-
 if sys.version_info < (3, 11):
     from async_timeout import timeout
 else:
@@ -67,10 +64,6 @@ else:
 
 
 if TYPE_CHECKING:
-    from playwright._impl._api_structures import (Geolocation,
-                                                  HttpCredentials, Headers,
-                                                  ViewportSize,
-                                                  ProxySettings, StorageState)
     BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -87,7 +80,7 @@ class CaptureResponse(TypedDict, total=False):
     last_redirected_url: str
     har: dict[str, Any] | None
     cookies: list[dict[str, Any]] | None
-    storage: StorageState | None
+    storage: dict[str, Any] | None
     error: str | None
     error_name: str | None
     html: str | None
@@ -137,7 +130,7 @@ class TrustedTimestampSettings(TypedDict, total=False):
 class Capture():
 
     _browsers: list[BROWSER] = ['chromium', 'firefox', 'webkit']
-    _default_viewport: ViewportSize = {'width': 1920, 'height': 1080}
+    _default_viewport: dict[str, int] = ViewportSettings.model_validate({'width': 1920, 'height': 1080}).model_dump(exclude_none=True)
     _default_timeout: int = 90  # set to 90s by default
     _minimal_timeout: int = 15  # set to 15s - It makes little sense to attempt a capture below that limit.
 
@@ -166,19 +159,8 @@ class Capture():
         self._requests: dict[str, bytes] = {}
 
         # Initialize the values with getter/setter
-        self._headers: Headers = {}
-        self._cookies: list[Cookie] = []
-        self._storage: StorageState = {}
-        self._viewport: ViewportSize | None = None
-        self._user_agent: str = ''
-        self._http_credentials: HttpCredentials = {}
-        self._geolocation: Geolocation = {}
-        self._timezone_id: str = ''
-        self._locale: str = 'en-US'
-        self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = None
-        self._java_script_enabled: bool = True
+        self._headers: dict[str, str] = {}
         self._capture_timeout: int = self._default_timeout
-        self._proxy: ProxySettings = {}
 
         # ###
 
@@ -199,19 +181,44 @@ class Capture():
         self.remote_headfull = capture_settings.remote_headfull
         self._init_script = capture_settings.init_script
 
+        self._proxy: dict[str, str] | None = None
+        if capture_settings.proxy:
+            if isinstance(capture_settings.proxy, ProxySettings):
+                self._proxy = capture_settings.proxy.model_dump(exclude_none=True)
+            else:
+                # The legacy option to pass 'force_tor', should have been transformed before we get here
+                self.logger.error(f'Incorrect value for the proxy: {capture_settings.proxy}')
+
         self.headers = capture_settings.headers
-        self.cookies = [c.model_dump(exclude_none=True) for c in capture_settings.cookies] if capture_settings.cookies else None
-        self.storage = capture_settings.storage
-        self.viewport = capture_settings.viewport.model_dump(exclude_none=True) if capture_settings.viewport else None
-        self.user_agent = capture_settings.user_agent
-        self.http_credentials = capture_settings.http_credentials.model_dump(exclude_none=True) if capture_settings.http_credentials else None
-        self.geolocation = capture_settings.geolocation.model_dump(exclude_none=True) if capture_settings.geolocation else None
-        self.timezone_id = capture_settings.timezone_id
-        self.locale = capture_settings.locale
-        self.color_scheme = capture_settings.color_scheme
-        self.java_script_enabled = capture_settings.java_script_enabled
+        self._cookies: list[dict[str, Any]] = [cookie.model_dump(exclude_none=True) for cookie in capture_settings.cookies] if capture_settings.cookies else []
+        self._storage: dict[str, Any] | None = capture_settings.storage.model_dump(exclude_none=True) if capture_settings.storage else None
+        self._viewport: dict[str, int] | None = capture_settings.viewport.model_dump(exclude_none=True) if capture_settings.viewport else None
+        self._user_agent: str = capture_settings.user_agent if capture_settings.user_agent else ''
+        self._http_credentials: dict[str, str] | None = capture_settings.http_credentials.model_dump(exclude_none=True) if capture_settings.http_credentials else None
+        self._geolocation: dict[str, float] | None = capture_settings.geolocation.model_dump(exclude_none=True) if capture_settings.geolocation else None
+        self._timezone_id = str(capture_settings.timezone_id) if capture_settings.timezone_id else None
+        self._locale: str = capture_settings.locale if capture_settings.locale else 'en-US'
+        self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = capture_settings.color_scheme if capture_settings.color_scheme else None
+        self._java_script_enabled: bool = capture_settings.java_script_enabled
         self.capture_timeout = capture_settings.general_timeout_in_sec
-        self.proxy = capture_settings.proxy
+        self.allow_tracking = capture_settings.allow_tracking
+        self.rendered_hostname_only = capture_settings.rendered_hostname_only
+        self.with_screenshot = capture_settings.with_screenshot
+        self.with_favicon = capture_settings.with_favicon
+        self.with_trusted_timestamps = capture_settings.with_trusted_timestamps
+        self.capture_depth = capture_settings.depth
+        self.final_wait = capture_settings.final_wait
+
+        self.initial_url: str
+        if capture_settings.url:
+            # This url could be None in transit when the thing to capture is a file (so the models allows it)
+            # But at this stage, the value must have been set to the local path of the file,
+            # if it is none, the capture will for sure fail.
+            self.initial_url = capture_settings.url
+        else:
+            raise InvalidPlaywrightParameter('No URL provided, cannot capture.')
+
+        self.initial_referer = capture_settings.referer
 
         self.should_retry: bool = False
         self.__network_not_idle: int = 2  # makes sure we do not wait for network idle the max amount of time the capture is allowed to take
@@ -236,8 +243,8 @@ class Capture():
         if env:
             self._env.update(env)
 
-    def __prepare_proxy_aiohttp(self, proxy: ProxySettings) -> str:
-        if 'username' in proxy and 'password' in proxy:
+    def __prepare_proxy_aiohttp(self, proxy: dict[str, str]) -> str:
+        if proxy.get('username') and proxy.get('password'):
             splitted = urlsplit(proxy['server'])
             return urlunsplit((splitted.scheme, f'{proxy["username"]}:{proxy["password"]}@{splitted.netloc}', splitted.path, splitted.query, splitted.fragment))
         return proxy['server']
@@ -283,7 +290,7 @@ class Capture():
             launch_env = {**os.environ, **self._env}
 
         self.browser = await self.playwright[self.browser_name].launch(
-            proxy=self.proxy if self.proxy else None,
+            proxy=self._proxy,  # type: ignore[arg-type]
             channel="chromium" if self.browser_name == "chromium" else None,
             args=args,
             headless=self.headless,
@@ -296,6 +303,9 @@ class Capture():
         # Create the temporary file to store the HAR content.
         self._temp_harfile = NamedTemporaryFile(delete=False, prefix="playwright_capture_har", suffix=".json")
 
+        # all the errors gathered during the capture
+        self.errors: list[str] = []
+
         return self
 
     async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> bool:
@@ -326,7 +336,7 @@ class Capture():
             return False
         return True
 
-    async def setup_page_capture(self, *, allow_tracking: bool=False) -> Page:
+    async def setup_page_capture(self) -> Page:
         """Prepare a page for a single-page capture without changing capture semantics.
 
         This method preserves the existing per-page setup used by capture_page:
@@ -375,7 +385,7 @@ class Capture():
                     if request.resource_type == 'image' and response.ok:
                         try:
                             if body := await response.body():
-                                m = self.magicdb.best_magic_buffer(body)
+                                m = self.magicdb.best_magic_buffer(body, None)
                                 if m.mime_type.startswith('image'):
                                     self._requests[request.url] = body
                         except Exception:
@@ -432,7 +442,7 @@ class Capture():
             except Error as e:
                 self.logger.warning(f'Failed at fetching PDF in headless chromium: {e}')
 
-        if allow_tracking:
+        if self.allow_tracking:
             # Add authorization clickthroughs
             await self.__dialog_didomi_clickthrough(page)
             await self.__dialog_onetrust_clickthrough(page)
@@ -457,36 +467,13 @@ class Capture():
         page.on("console", handle_console_msg)
         return page
 
-    def __prepare_proxy_playwright(self, proxy: str) -> ProxySettings:
-        splitted = urlsplit(proxy)
-        if splitted.username and splitted.password:
-            return {'username': splitted.username, 'password': splitted.password,
-                    'server': urlunsplit((splitted.scheme, f'{splitted.hostname}:{splitted.port}', splitted.path, splitted.query, splitted.fragment))}
-        return {'server': proxy}
-
-    @property
-    def proxy(self) -> ProxySettings:
-        return self._proxy
-
-    @proxy.setter
-    def proxy(self, proxy: str | dict[str, str] | None) -> None:
-        if proxy:
-            if isinstance(proxy, str):
-                self._proxy = self.__prepare_proxy_playwright(proxy)
-            elif isinstance(proxy, dict):
-                self._proxy = {'server': proxy['server'],
-                               'bypass': proxy.get('bypass', ''),
-                               'username': proxy.get('username', ''),
-                               'password': proxy.get('password', '')}
-            else:
-                raise InvalidPlaywrightParameter(f'Invalid proxy parameter: "{proxy}" ({type(proxy)})')
-
     @property
     def capture_timeout(self) -> int:
         return self._capture_timeout
 
     @capture_timeout.setter
     def capture_timeout(self, timeout: int | None) -> None:
+        # NOTE: This check is relevant, and needs to happen there
         if not timeout:
             self._capture_timeout = self._default_timeout
         else:
@@ -497,101 +484,12 @@ class Capture():
                 self._capture_timeout = timeout
 
     @property
-    def locale(self) -> str:
-        return self._locale
-
-    @locale.setter
-    def locale(self, locale: str | None) -> None:
-        if locale:
-            self._locale = locale
-
-    @property
-    def timezone_id(self) -> str:
-        return self._timezone_id
-
-    @timezone_id.setter
-    def timezone_id(self, timezone_id: str | None) -> None:
-        if not timezone_id:
-            return
-        if timezone_id in all_timezones_set:
-            self._timezone_id = timezone_id
-        else:
-            raise InvalidPlaywrightParameter(f'The Timezone ID provided ({timezone_id}) is invalid.')
-
-    @property
-    def http_credentials(self) -> HttpCredentials:
-        return self._http_credentials
-
-    @http_credentials.setter
-    def http_credentials(self, credentials: dict[str, str] | None) -> None:
-        if not credentials:
-            return
-        if 'username' in credentials and 'password' in credentials:
-            self._http_credentials = {'username': credentials['username'],
-                                      'password': credentials['password']}
-            if 'origin' in credentials:
-                self._http_credentials['origin'] = credentials['origin']
-        else:
-            raise InvalidPlaywrightParameter(f'At least a username and a password are required in the credentials: {credentials}')
-
-    def set_http_credentials(self, username: str, password: str, origin: str | None=None) -> None:
-        self._http_credentials = {'username': username, 'password': password, 'origin': origin}
-
-    @property
-    def geolocation(self) -> Geolocation:
-        return self._geolocation
-
-    @geolocation.setter
-    def geolocation(self, geolocation: dict[str, str | int | float] | None) -> None:
-        if not geolocation:
-            return
-        if 'latitude' in geolocation and 'longitude' in geolocation:
-            self._geolocation = {'latitude': float(geolocation['latitude']),
-                                 'longitude': float(geolocation['longitude'])}
-            if 'accuracy' in geolocation:
-                self._geolocation['accuracy'] = float(geolocation['accuracy'])
-        else:
-            raise InvalidPlaywrightParameter(f'At least a latitude and a longitude are required in the geolocation: {geolocation}')
-
-    @property
-    def cookies(self) -> list[Cookie]:
-        return self._cookies
-
-    @cookies.setter
-    def cookies(self, cookies: list[Cookie | dict[str, Any]] | None) -> None:
-        '''Cookies to send along to the initial request.
-
-        :param cookies: The cookies, in this format: https://playwright.dev/python/docs/api/class-browsercontext#browser-context-add-cookies
-        '''
-        if not cookies:
-            return
-        for raw_cookie in cookies:
-            if not raw_cookie:
-                continue
-            if isinstance(raw_cookie, Cookie):
-                self._cookies.append(raw_cookie)
-                continue
-            try:
-                self._cookies.append(Cookie.model_validate(raw_cookie))
-            except Exception as e:
-                self.logger.warning(f'Invalid cookie: {e}')
-
-    @property
-    def storage(self) -> StorageState:
-        return self._storage
-
-    @storage.setter
-    def storage(self, storage: dict[str, Any] | None) -> None:
-        if storage and 'cookies' in storage and 'origins' in storage:
-            self._storage['cookies'] = storage['cookies']
-            self._storage['origins'] = storage['origins']
-
-    @property
-    def headers(self) -> Headers:
+    def headers(self) -> dict[str, str]:
         return self._headers
 
     @headers.setter
     def headers(self, headers: dict[str, str] | None) -> None:
+        # NOTE: this is probably superseeded by the models, need to check that
         if not headers:
             return
         if isinstance(headers, dict):
@@ -616,64 +514,18 @@ class Capture():
                 continue
             self._headers[name] = value
 
-    @property
-    def viewport(self) -> ViewportSize | None:
-        return self._viewport
-
-    @viewport.setter
-    def viewport(self, viewport: dict[str, str | int] | None) -> None:
-        if not viewport:
-            return
-        if 'width' in viewport and 'height' in viewport:
-            self._viewport = {'width': int(viewport['width']), 'height': int(viewport['height'])}
-        else:
-            raise InvalidPlaywrightParameter(f'A viewport must have a height and a width - {viewport}')
-
-    @property
-    def user_agent(self) -> str:
-        return self._user_agent
-
-    @user_agent.setter
-    def user_agent(self, user_agent: str | None) -> None:
-        if user_agent is not None:
-            self._user_agent = user_agent
-
-    @property
-    def color_scheme(self) -> Literal['dark', 'light', 'no-preference', 'null'] | None:
-        return self._color_scheme
-
-    @color_scheme.setter
-    def color_scheme(self, color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None) -> None:
-        if not color_scheme:
-            return
-        schemes = ['light', 'dark', 'no-preference', 'null']
-        if color_scheme in schemes:
-            self._color_scheme = color_scheme
-        else:
-            raise InvalidPlaywrightParameter(f'Invalid color scheme ({color_scheme}), must be in {", ".join(schemes)}.')
-
-    @property
-    def java_script_enabled(self) -> bool:
-        return self._java_script_enabled
-
-    @java_script_enabled.setter
-    def java_script_enabled(self, enabled: bool) -> None:
-        self._java_script_enabled = enabled
-
     async def initialize_context(self) -> None:
         device_context_settings = {}
+        vp: dict[str, int] | None = None
         if self.device_name:
             device_context_settings = self.playwright.devices[self.device_name]
             # We need to make sure the device_context_settings dict doesn't contains
             # keys that are set by default in the context creation
-            if context_ua := device_context_settings.pop('user_agent', None):
-                ua = self.user_agent if self.user_agent else context_ua
-            if context_vp := device_context_settings.pop('viewport', self._default_viewport):
-                # Always true, but we also always want to pop it.
-                vp = self.viewport if self.viewport else context_vp
+            ua = device_context_settings.pop('user_agent', self._user_agent)
+            vp = device_context_settings.pop('viewport', self._viewport)
         else:
-            ua = self.user_agent
-            vp = self.viewport
+            ua = self._user_agent
+            vp = self._viewport
 
         # NOTE 2026-04-10: Very specific edge case:
         # * capture in remote headfull mode with xpra
@@ -690,19 +542,20 @@ class Capture():
             record_har_path=self._temp_harfile.name,
             ignore_https_errors=True,
             bypass_csp=True,
-            java_script_enabled=self.java_script_enabled,
-            http_credentials=self.http_credentials if self.http_credentials else None,
+            java_script_enabled=self._java_script_enabled,
+            http_credentials=self._http_credentials,  # type: ignore[arg-type]
             user_agent=ua,
-            locale=self.locale if self.locale else None,
-            timezone_id=self.timezone_id if self.timezone_id else None,
-            color_scheme=self.color_scheme if self.color_scheme else None,
-            viewport=vp,
-            storage_state=self.storage if self.storage else None,
+            locale=self._locale,
+            timezone_id=self._timezone_id,
+            color_scheme=self._color_scheme,
+            viewport=vp if vp else self._default_viewport,  # type: ignore[arg-type]
+            storage_state=self._storage,  # type: ignore[arg-type]
             # For debug only
             # record_video_dir='./videos/',
             **device_context_settings
         )
         self.context.set_default_timeout(self._capture_timeout * 1000)
+        await self.context.credentials.install()
 
         if self._init_script:
             await self.context.add_init_script(script=self._init_script)
@@ -753,11 +606,11 @@ class Capture():
                # 'script_logging': True,
                })
 
-        if self.cookies:
+        if self._cookies:
             try:
-                await self.context.add_cookies([c.model_dump(exclude_none=True) for c in self.cookies])  # type: ignore[misc]
+                await self.context.add_cookies(self._cookies)  # type: ignore[arg-type]
             except Exception:
-                self.logger.exception(f'Unable to set cookies: {self.cookies}')
+                self.logger.exception(f'Unable to set cookies: {self._cookies}')
 
         if self.headers:
             try:
@@ -765,8 +618,8 @@ class Capture():
             except Exception:
                 self.logger.exception(f'Unable to set HTTP Headers: {self.headers}')
 
-        if self.geolocation:
-            await self.context.set_geolocation(self.geolocation)
+        if self._geolocation:
+            await self.context.set_geolocation(self._geolocation)  # type: ignore[arg-type]
 
         # NOTE: Which perms are supported by which browsers varies
         # See https://github.com/microsoft/playwright/issues/16577
@@ -1052,7 +905,7 @@ class Capture():
         except Exception as e:
             self.logger.info(f'Error while moving time forward: {e}')
 
-    async def __instrumentation(self, page: Page, url: str, allow_tracking: bool, final_wait: int) -> None:
+    async def __instrumentation(self, page: Page, url: str) -> None:
         try:
             # NOTE: the clock must be installed after the page is loaded, otherwise it sometimes cause the complete capture to hang.
             await page.clock.install()
@@ -1103,7 +956,7 @@ class Capture():
             await self._wait_for_random_timeout(page, 5)
             self.logger.debug('Keep going after moving mouse.')
 
-            if allow_tracking:
+            if self.allow_tracking:
                 await self._wait_for_random_timeout(page, 5)
                 # This event is required trigger the add_locator_handler
                 try:
@@ -1182,18 +1035,57 @@ class Capture():
 
         self.logger.debug('Done with instrumentation.')
         # Wait at least 5 sec after instrumentation
-        self.logger.debug(f'Waiting another {max(final_wait, 5)}s.')
-        await self._wait_for_random_timeout(page, max(final_wait, 5))
+        self.logger.debug(f'Waiting another {max(self.final_wait, 5)}s.')
+        await self._wait_for_random_timeout(page, max(self.final_wait, 5))
         await self._safe_wait(page)
         self.logger.debug('Done with waiting.')
 
+    async def _safe_get_storage_state(self) -> dict[str, Any]:
+        # Collect storage state, including IndexedDB, to capture the full browser state.
+        # 2026-09-08: add WebAuth credentials
+        # 2026-09-17: Add opfs
+        # Quite a few captures fail either on opfs or on indexed db, so we hav a few fallbacks
+        to_store = {'indexed_db': True, 'opfs': True, 'credentials': True}
+        while True:
+            try:
+                async with timeout(15):
+                    return await self.context.storage_state(**to_store)  # type: ignore[return-value,arg-type]
+            except (TimeoutError, asyncio.TimeoutError):
+                self.logger.warning("Unable to get storage (timeout).")
+                self.errors.append("Unable to get the storage (timeout).")
+                self.should_retry = True
+                break
+            except Error as e:
+                if to_store['indexed_db'] and 'IndexedDB' in str(e):
+                    to_store['indexed_db'] = False
+                    self.errors.append('Unable to get the IndexedDB')
+                    self.logger.warning(f"Unable to get the IndexedDB: {e}")
+                    continue
+                if to_store['opfs'] and 'OPFS' in str(e):
+                    to_store['opfs'] = False
+                    self.errors.append('Unable to get the OPFS')
+                    self.logger.warning(f"Unable to get the OPFS: {e}")
+                    continue
+
+                if not to_store['indexed_db'] and not to_store['opfs']:
+                    # we disabled both options, quit
+                    self.should_retry = True
+                    self.errors.append(f'Unable to get the storage at all: {e}')
+                    self.logger.warning(f"Unable to get the storage at all: {e}")
+                    break
+            except Exception as e:
+                # When the driver explodes for no clear reason.
+                self.logger.warning(f"[Generic Exception] Unable to get the storage: {e}")
+                self.errors.append(f'[Generic Exception] Unable to get the storage: {e}')
+                self.should_retry = True
+                break
+        return {}
+
     async def _finalize_capture(
         self,
         *,
         page: Page,
         to_return: CaptureResponse,
-        errors: list[str],
-        with_trusted_timestamps: bool,
     ) -> None:
         """Common finalization logic for captures (downloads, cookies, storage, HAR, socks5, timestamps)."""
 
@@ -1224,36 +1116,19 @@ class Capture():
                 to_return['cookies'] = [Cookie.model_validate(c).model_dump(exclude_none=True) for c in await self.context.cookies()]
         except (TimeoutError, asyncio.TimeoutError):
             self.logger.warning("Unable to get cookies (timeout).")
-            errors.append("Unable to get the cookies (timeout).")
+            self.errors.append("Unable to get the cookies (timeout).")
             self.should_retry = True
         except Error as e:
             self.logger.warning(f"Unable to get cookies: {e}")
-            errors.append(f'Unable to get the cookies: {e}')
+            self.errors.append(f'Unable to get the cookies: {e}')
             self.should_retry = True
         except Exception as e:
             # When the driver explodes for no clear reason.
             self.logger.warning(f"[Generic Exception] Unable to get cookies: {e}")
-            errors.append(f'[Generic Exception] Unable to get the cookies: {e}')
+            self.errors.append(f'[Generic Exception] Unable to get the cookies: {e}')
             self.should_retry = True
 
-        # Collect storage state, including IndexedDB, to capture the full browser state.
-        # 2026-09-08: add WebAuth credentials
-        try:
-            async with timeout(15):
-                to_return['storage'] = await self.context.storage_state(indexed_db=True, credentials=True)
-        except (TimeoutError, asyncio.TimeoutError):
-            self.logger.warning("Unable to get storage (timeout).")
-            errors.append("Unable to get the storage (timeout).")
-            self.should_retry = True
-        except Error as e:
-            self.logger.warning(f"Unable to get the storage: {e}")
-            errors.append(f'Unable to get the storage: {e}')
-            self.should_retry = True
-        except Exception as e:
-            # When the driver explodes for no clear reason.
-            self.logger.warning(f"[Generic Exception] Unable to get the storage: {e}")
-            errors.append(f'[Generic Exception] Unable to get the storage: {e}')
-            self.should_retry = True
+        to_return['storage'] = await self._safe_get_storage_state()
 
         try:
             if page.is_closed():
@@ -1303,22 +1178,22 @@ class Capture():
             # When using a socks5 proxy, post-process the HAR to resolve IPs via
             # the proxy so the stored HAR contains addresses consistent with what
             # the proxy saw.
-            if (to_return.get('har') and self.proxy and self.proxy.get('server')
-                    and self.proxy['server'].startswith('socks5')):
+            if (to_return.get('har') and self._proxy and self._proxy.get('server')
+                    and self._proxy['server'].startswith('socks5')):
                 if har := to_return['har']:  # Could be None
                     try:
                         async with timeout(120):
                             await self.socks5_resolver(har)
                     except (TimeoutError, asyncio.TimeoutError):
                         self.logger.warning("Unable to resolve all the IPs via the socks5 proxy.")
-                        errors.append("Unable to resolve all the IPs via the socks5 proxy.")
+                        self.errors.append("Unable to resolve all the IPs via the socks5 proxy.")
                         self.should_retry = True
 
         except (TimeoutError, asyncio.TimeoutError):
             # If closing the context or generating the HAR takes too long, the
             # capture is considered incomplete but we still return what we have.
             self.logger.warning("[Timeout] Unable to close context at the end of the capture.")
-            errors.append("[Timeout] Unable to close context at the end of the capture.")
+            self.errors.append("[Timeout] Unable to close context at the end of the capture.")
             self.should_retry = True
             # In case of timeout, let the exception reach the async calls
             await asyncio.sleep(1)
@@ -1326,11 +1201,11 @@ class Capture():
             # Any other unexpected failure while finalizing the capture is logged
             # and surfaced as a generic HAR-generation error.
             self.logger.warning(f"Other exception while finishing up the capture: {e}.")
-            errors.append(f'Unable to generate HAR file: {e}')
+            self.errors.append(f'Unable to generate HAR file: {e}')
 
-        if errors:
-            to_return['error'] = '\n'.join(errors)
-        if with_trusted_timestamps:
+        if self.errors:
+            to_return['error'] = '\n'.join(self.errors)
+        if self.with_trusted_timestamps:
             try:
                 await self._get_trusted_timestamps(to_return)
             except Exception as e:
@@ -1346,7 +1221,7 @@ class Capture():
             return False, f"Unable to parse URL '{url}', blocked."
 
         if not _url.host:
-            self.logger.warning(f"Missing Host: {url}")
+            self.logger.info(f"Missing Host: {url}")
             return False, f"Missing host in URL '{url}', blocked."
         try:
             ip = ipaddress.ip_address(_url.host.try_into_ip())
@@ -1381,7 +1256,11 @@ class Capture():
         self.logger.warning(f"Opening a weird URL: {url}")
         return False, f"Attempted to open a weird URL '{url}', blocked."
 
-    async def open_page(self, page: Page, url: str, errors: list[str], referer: str | None=None) -> None:
+    async def open_page(self, page: Page, url: str | None = None, referer: str | None=None) -> None:
+        """This method opens the page but does nothing with it. Use it only if you need a custom instrumentation.
+        The usecase in lookyloo's context is to have a headfull capture in Xpra.
+        Prefer using `capture_page` instead.
+        """
 
         async def catch_file_route(route: Route, request: Request) -> None:
             if unquote(request.url) == url:
@@ -1424,9 +1303,10 @@ class Capture():
             ]
             for scheme in allowed_schemes:
                 await page.route(scheme, lambda route: route.continue_())
-
+        if not url:
+            url = self.initial_url
         try:
-            await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else '')
+            await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else self.initial_referer)
             try:
                 await page.bring_to_front()
                 self.logger.debug('Page moved to front.')
@@ -1442,7 +1322,7 @@ class Capture():
                 try:
                     async with page.expect_download() as download_info:
                         try:
-                            await page.goto(url, referer=referer if referer else '')
+                            await page.goto(url, referer=referer if referer else self.initial_referer)
                         except Exception:
                             pass
                         with NamedTemporaryFile() as tmp_f:
@@ -1460,7 +1340,7 @@ class Capture():
                         error_msg = download.failure()
                         if not error_msg:
                             raise e
-                        errors.append(f"Error while downloading: {error_msg}")
+                        self.errors.append(f"Error while downloading: {error_msg}")
                         self.logger.info(f'Error while downloading: {error_msg}')
                         self.should_retry = True
                     except Exception:
@@ -1470,44 +1350,10 @@ class Capture():
         else:
             await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
 
-    @overload
-    async def capture_page(self, url: str, *, max_depth_capture_time: int,
-                           referer: str | None=None,
-                           page: Page | None=None, depth: int=0,
-                           rendered_hostname_only: bool=True,
-                           with_screenshot: bool=True,
-                           with_favicon: bool=False,
-                           allow_tracking: bool=False,
-                           with_trusted_timestamps: bool=False,
-                           current_page_only: bool=False,
-                           final_wait: int=5
-                           ) -> CaptureResponse:
-        ...
-
-    @overload
-    async def capture_page(self, url: None=None, *, max_depth_capture_time: int,
-                           referer: str | None=None,
-                           page: Page, depth: int=0,
-                           rendered_hostname_only: bool=True,
-                           with_screenshot: bool=True,
-                           with_favicon: bool=False,
-                           allow_tracking: bool=False,
-                           with_trusted_timestamps: bool=False,
-                           current_page_only: bool=False,
-                           final_wait: int=5
-                           ) -> CaptureResponse:
-        ...
-
     async def capture_page(self, url: str | None=None, *, max_depth_capture_time: int,
                            referer: str | None=None,
-                           page: Page | None=None, depth: int=0,
-                           rendered_hostname_only: bool=True,
-                           with_screenshot: bool=True,
-                           with_favicon: bool=False,
-                           allow_tracking: bool=False,
-                           with_trusted_timestamps: bool=False,
+                           page: Page | None=None, depth: int | None = None,
                            current_page_only: bool=False,
-                           final_wait: int=5,
                            ) -> CaptureResponse:
         """Capture a URL and optionally recurse into child links.
 
@@ -1520,10 +1366,11 @@ class Capture():
         (no navigation, no recursion) and then finalizes.  This is the path
         used by remote headfull captures after setup_page_capture has already been
         called by the caller.
+
+        The `url` field is only needed when the URL to capture isn't the initial one (depth>0)
         """
 
         to_return: CaptureResponse = {}
-        errors: list[str] = []
         capturing_sub = False
 
         if current_page_only:
@@ -1532,7 +1379,7 @@ class Capture():
                 raise InvalidPlaywrightParameter('current_page_only requires an initialized page')
         else:
             if page is None:
-                page = await self.setup_page_capture(allow_tracking=allow_tracking)
+                page = await self.setup_page_capture()
             else:
                 # Automated capture with depth > 0
                 capturing_sub = True
@@ -1541,12 +1388,13 @@ class Capture():
             if not current_page_only:
                 # Standard navigation + capture path.
                 if not url:
-                    raise InvalidPlaywrightParameter('The URL to capture is missing.')
-                await self.open_page(page, url, errors, referer)
+                    url = self.initial_url
+                await self.open_page(page, url=url if url else self.initial_url,
+                                     referer=referer if referer else self.initial_referer)
 
                 try:
                     if self.headless:
-                        await self.__instrumentation(page, url, allow_tracking, final_wait)
+                        await self.__instrumentation(page, url)
                     else:
                         self.logger.debug('Headed mode, skipping instrumentation.')
                         await self._wait_for_random_timeout(page, self._capture_timeout - 5)
@@ -1580,7 +1428,7 @@ class Capture():
                         u = '/!\\ Unknown /!\\'
                     to_return['last_redirected_url'] = u
 
-            if 'html' in to_return and to_return['html'] is not None and with_favicon:
+            if 'html' in to_return and to_return['html'] is not None and self.with_favicon:
                 # We're probably (?) safe only looking for favicons in the main frame.
                 # TODO: check that?
                 try:
@@ -1591,7 +1439,7 @@ class Capture():
                 except Exception as e:
                     self.logger.warning(f'Unable to get favicons: {e}')
 
-            if with_screenshot:
+            if self.with_screenshot:
                 to_return['png'] = await self._failsafe_get_screenshot(page)
 
             # Keep that all the way down there in case the capture failed.
@@ -1600,11 +1448,15 @@ class Capture():
             else:
                 self._already_captured.add(page.url)
 
-            if depth > 0 and to_return.get('html') and to_return['html']:
+            if depth is None:
+                # fallback for the first call
+                depth = self.capture_depth
+
+            if depth is not None and depth > 0 and to_return.get('html') and to_return['html']:
                 # TODO with children frames:
                 # 1. if the frame has a URL, use that as base URL/referer for the subsequent captures
                 # 2. if it doesn't, the base URL is the url of the parent (which may or may not be the main frame)
-                if child_urls := self._get_links_from_rendered_page(page.url, to_return['html'], rendered_hostname_only):
+                if child_urls := self._get_links_from_rendered_page(page.url, to_return['html']):
                     to_return['children'] = []
                     depth -= 1
                     total_urls = len(child_urls)
@@ -1630,11 +1482,8 @@ class Capture():
                                 child_capture = await self.capture_page(
                                     url=url, referer=page.url,
                                     page=page, depth=depth,
-                                    rendered_hostname_only=rendered_hostname_only,
-                                    max_depth_capture_time=max_capture_time,
-                                    with_screenshot=with_screenshot,
-                                    final_wait=final_wait)
-                                if with_trusted_timestamps:
+                                    max_depth_capture_time=max_capture_time)
+                                if self.with_trusted_timestamps:
                                     try:
                                         await self._get_trusted_timestamps(child_capture)
                                     except Exception as e:
@@ -1654,7 +1503,7 @@ class Capture():
                         if consecutive_errors >= 5:
                             # if we have more than 5 consecutive errors, the capture is most probably broken, breaking.
                             self.logger.warning('Got more than 5 consecutive errors while capturing children, breaking.')
-                            errors.append("Got more than 5 consecutive errors while capturing children")
+                            self.errors.append("Got more than 5 consecutive errors while capturing children")
                             self.should_retry = True
                             break
 
@@ -1666,19 +1515,19 @@ class Capture():
                             self.logger.info(f'Unable to go back: {e}.')
 
         except PlaywrightTimeoutError as e:
-            errors.append(f"The capture took too long - {e.message}")
+            self.errors.append(f"The capture took too long - {e.message}")
             self.should_retry = True
         except (asyncio.TimeoutError, TimeoutError):
-            errors.append("Something in the capture took too long")
+            self.errors.append("Something in the capture took too long")
             self.should_retry = True
         except TargetClosedError as e:
-            errors.append(f"The target was closed - {e}")
+            self.errors.append(f"The target was closed - {e}")
             self.should_retry = True
         except Error as e:
             # NOTE: there are a lot of errors that look like duplicates and they are triggered at different times in the process.
             # it is tricky to figure our which one should (and should not) trigger a retry. Below is our best guess and it will change over time.
             self._update_exceptions(e)
-            errors.append(e.message)
+            self.errors.append(e.message)
             to_return['error_name'] = e.name
             # NOTE: e.name is generally (always?) "Error"
             if self._fatal_network_error(e) or self._fatal_auth_error(e) or self.fatal_browser_error(e):
@@ -1686,7 +1535,7 @@ class Capture():
             elif self._retry_network_error(e) or self._retry_browser_error(e):
                 # this one sounds like something we can retry...
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
-                errors.append(f'Issue with {url}: {e.message}')
+                self.errors.append(f'Issue with {url}: {e.message}')
                 self.should_retry = True
             else:
                 # Unexpected ones
@@ -1694,15 +1543,15 @@ class Capture():
         except PlaywrightCaptureException as e:
             # unrecoverable exeptions
             self.logger.warning(f'Unable to run capture: {e}')
-            errors.append(f'Unable to run capture: {e}')
+            self.errors.append(f'Unable to run capture: {e}')
             raise e
         except Exception as e:
             # we may get a non-playwright exception to.
             # The ones we try to handle here should be treated as if they were.
-            errors.append(str(e))
+            self.errors.append(str(e))
             if str(e) in ['Connection closed while reading from the driver']:
                 self.logger.info(f'Issue with {url} (retrying): {e}')
-                errors.append(f'Issue with {url}: {e}')
+                self.errors.append(f'Issue with {url}: {e}')
                 self.should_retry = True
             else:
                 raise e
@@ -1713,8 +1562,6 @@ class Capture():
                 await self._finalize_capture(
                     page=page,
                     to_return=to_return,
-                    errors=errors,
-                    with_trusted_timestamps=with_trusted_timestamps,
                 )
         self.logger.debug('Capture done')
         return to_return
@@ -1753,8 +1600,8 @@ class Capture():
         trusted_timestamps: dict[str, bytes] = {}
 
         connector = None
-        if self.proxy and self.proxy.get('server'):
-            connector = ProxyConnector.from_url(self.proxy['server'])
+        if self._proxy and self._proxy.get('server'):
+            connector = ProxyConnector.from_url(self._proxy['server'])
 
         timeout = aiohttp.ClientTimeout(total=10)
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
@@ -1840,6 +1687,21 @@ class Capture():
             self.__network_not_idle += 1
             self.logger.debug(f'Timed out waiting for network idle, max wait: {max_wait}s')
 
+    def __decode_datauri(self, parsed: tuple[str, str, bytes]) -> str | None:
+        mime, mime_params, content = parsed
+        charset = 'utf-8'
+        if mime_params:
+            # try to get charset
+            if qs := parse_qs(mime_params):
+                if charsets := qs.get('charset'):
+                    try:
+                        charset = codecs.lookup(charsets[0]).name
+                    except LookupError:
+                        charset = 'utf-8'
+        if content:
+            return unquote(content, encoding=charset)
+        return None
+
     async def _failsafe_get_content(self, page: Frame) -> str | None:
         ''' The page might be changing for all kind of reason (generally a JS timeout).
         In that case, we try a few times to get the HTML.'''
@@ -1874,18 +1736,8 @@ class Capture():
             self.logger.debug(f'Data URL in frame: {page.url}')
             # 2026-02-10: if the URL starts with data, we have a data URI, and possibly some content
             if parsed := self.__parse_data_uri(page.url):
-                mime, mime_params, content = parsed
-                charset = 'utf-8'
-                if mime_params:
-                    # try to get charset
-                    if qs := parse_qs(mime_params):
-                        if charsets := qs.get('charset'):
-                            try:
-                                charset = codecs.lookup(charsets[0]).name
-                            except LookupError:
-                                charset = 'utf-8'
-                if content:
-                    return unquote(content, encoding=charset)
+                if content := self.__decode_datauri(parsed):
+                    return content
                 else:
                     self.logger.warning('No content: {page.url}')
             else:
@@ -1895,7 +1747,7 @@ class Capture():
             return unquote(page.name)
         return None
 
-    def _get_links_from_rendered_page(self, rendered_url: str, rendered_html: str, rendered_hostname_only: bool) -> list[str]:
+    def _get_links_from_rendered_page(self, rendered_url: str, rendered_html: str) -> list[str]:
         def _sanitize(maybe_url: str) -> str | None:
             href = strip_html5_whitespace(maybe_url)
             href = safe_url_string(href)
@@ -1923,7 +1775,7 @@ class Capture():
                 continue
             try:
                 if href := _sanitize(href):
-                    if not rendered_hostname_only:
+                    if not self.rendered_hostname_only:
                         urls.add(href)
                     elif rendered_hostname and urlparse(href).hostname == rendered_hostname:
                         urls.add(href)
@@ -1984,8 +1836,8 @@ class Capture():
         await main_frame.get_by_role("button", name="Get an audio challenge").click()
 
         connector = None
-        if self.proxy and self.proxy.get('server'):
-            connector = ProxyConnector.from_url(self.proxy['server'])
+        if self._proxy and self._proxy.get('server'):
+            connector = ProxyConnector.from_url(self._proxy['server'])
 
         timeout = aiohttp.ClientTimeout(total=10)
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
@@ -2179,6 +2031,11 @@ class Capture():
         to_return: FramesResponse = {'name': frame.name, 'url': frame.url, 'content': ''}
         if frame.is_detached():
             self.logger.debug(f'{frame_id} is detached.')
+            # can be a data url
+            if frame.url and frame.url.strip() and frame.url.strip().startswith('data'):
+                if parsed := self.__parse_data_uri(frame.url):
+                    if content := self.__decode_datauri(parsed):
+                        to_return['content'] = content
         else:
             to_return['content'] = await self._failsafe_get_content(frame)
             if frame.child_frames:
@@ -2305,9 +2162,9 @@ class Capture():
             return await handler(req)
 
         connector = None
-        if self.proxy:
+        if self._proxy:
             # NOTE 2024-05-17: switch to async to fetch, the lib uses socks5h by default
-            connector = ProxyConnector.from_url(self.__prepare_proxy_aiohttp(self.proxy))
+            connector = ProxyConnector.from_url(self.__prepare_proxy_aiohttp(self._proxy))
 
         extracted_favicons = self.__extract_favicons(rendered_content)
         if not extracted_favicons:
@@ -2316,7 +2173,7 @@ class Capture():
         to_fetch.add('/favicon.ico')
         timeout = aiohttp.ClientTimeout(total=10)
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
-            session.headers['user-agent'] = self.user_agent
+            session.headers['user-agent'] = self._user_agent
             for u in to_fetch:
                 try:
                     self.logger.debug(f'Attempting to fetch favicon from {u}.')
@@ -2329,6 +2186,9 @@ class Capture():
                         continue
 
                     if self.only_global_lookup:
+                        if url_to_fetch == "file:///favicon.ico":
+                            # skip, no need to log that.
+                            continue
                         public, message = self.__check_local_url(url_to_fetch)
                         if public is False:
                             # got a local URL
@@ -2342,7 +2202,7 @@ class Capture():
                             favicon_response.raise_for_status()
                             favicon = await favicon_response.read()
                     if favicon:
-                        m = self.magicdb.best_magic_buffer(favicon)
+                        m = self.magicdb.best_magic_buffer(favicon, None)
                         if not m.mime_type:
                             # empty, ignore
                             pass
@@ -2367,7 +2227,11 @@ class Capture():
     # We get the HAR file, iterate over the entries an update the IPs
 
     async def socks5_resolver(self, harfile: dict[str, Any]) -> None:
-        resolver = Socks5Resolver(logger=self.logger, socks5_proxy=self.proxy['server'],
+        if not self._proxy:
+            raise InvalidPlaywrightParameter('Proxy required for socks5_resolver.')
+
+        resolver = Socks5Resolver(logger=self.logger,
+                                  socks5_proxy=self._proxy['server'],
                                   dns_resolver=self.socks5_dns_resolver)
         # get all the hostnames from the HAR file
         hostnames = set()

{playwrightcapture-1.41.3 → playwrightcapture-1.41.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "PlaywrightCapture"
-version = "1.41.3"
+version = "1.41.5"
 description = "A simple library to capture websites using playwright"
 authors = [
     {name="Raphaël Vinot", email= "raphael.vinot@circl.lu"}
@@ -12,7 +12,7 @@ requires-python = ">=3.10,<3.15"
 dynamic = [ "classifiers" ]
 
 dependencies = [
-    "playwright (>=1.62.0)",
+    "playwright (>=1.63.0)",
     "beautifulsoup4[charset-normalizer,lxml] (>=4.15.0)",
     "w3lib (>=2.4.1)",
     "playwright-stealth (>=2.0.3)",
@@ -23,9 +23,9 @@ dependencies = [
     "dnspython (>=2.7.0,<3.0.0)",
     "python-socks (>=3.0.0,<4.0.0)",
     "rfc3161-client (>=1.0.4,<2.0.0)",
-    "orjson (>=3.11.4,<4.0.0)",
-    "pure-magic-rs (>=0.4.3)",
-    "lookyloo-models (>=0.3.1)",
+    "orjson (>=3.12,<4.0.0)",
+    "pure-magic-rs (>=0.5)",
+    "lookyloo-models (>=0.4.3)",
     "charset-normalizer (>=3.4.6,<4.0.0)",
     "pyfaup-rs (>=0.4.6,<0.5.0)"
 ]