PlaywrightCapture 1.41.0__tar.gz → 1.41.2__tar.gz

{playwrightcapture-1.41.0 → playwrightcapture-1.41.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PlaywrightCapture
-Version: 1.41.0
+Version: 1.41.2
 Summary: A simple library to capture websites using playwright
 License-Expression: BSD-3-Clause
 License-File: LICENSE

{playwrightcapture-1.41.0 → playwrightcapture-1.41.2}/playwrightcapture/capture.py

@@ -1230,6 +1230,11 @@ class Capture():
             self.logger.warning(f"Unable to get cookies: {e}")
             errors.append(f'Unable to get the cookies: {e}')
             self.should_retry = True
+        except Exception as e:
+            # When the driver explodes for no clear reason.
+            self.logger.warning(f"[Generic Exception] Unable to get cookies: {e}")
+            errors.append(f'[Generic Exception] Unable to get the cookies: {e}')
+            self.should_retry = True
 
         # Collect storage state, including IndexedDB, to capture the full browser state.
         try:
@@ -1243,6 +1248,11 @@ class Capture():
             self.logger.warning(f"Unable to get the storage: {e}")
             errors.append(f'Unable to get the storage: {e}')
             self.should_retry = True
+        except Exception as e:
+            # When the driver explodes for no clear reason.
+            self.logger.warning(f"[Generic Exception] Unable to get the storage: {e}")
+            errors.append(f'[Generic Exception] Unable to get the storage: {e}')
+            self.should_retry = True
 
         try:
             if page.is_closed():
@@ -1387,8 +1397,8 @@ class Capture():
                 await route.continue_()
             else:
                 # other URLs
-                not_local, message = self.__check_local_url(request.url)
-                if not_local:
+                public, message = self.__check_local_url(request.url)
+                if public:
                     await route.continue_()
                 else:
                     await route.fulfill(status=404, content_type="text/plain", body=message)
@@ -2284,6 +2294,15 @@ class Capture():
         In the meantime, we try to get all the potential ones in this method.
         Method inspired by https://github.com/ail-project/ail-framework/blob/master/bin/lib/crawlers.py
         """
+        async def __check_local_middleware(req: aiohttp.ClientRequest, handler: aiohttp.ClientHandlerType) -> aiohttp.ClientResponse:
+            if self.only_global_lookup:
+                public, message = self.__check_local_url(str(req.url))
+                if public is False:
+                    # got a local URL
+                    self.logger.warning(f'Attempted to get a local favicon (in redirect): {message}')
+                    raise aiohttp.ClientError(f'Attempted to get a local favicon (in redirect): {message}')
+            return await handler(req)
+
         connector = None
         if self.proxy:
             # NOTE 2024-05-17: switch to async to fetch, the lib uses socks5h by default
@@ -2304,10 +2323,13 @@ class Capture():
                     if url_to_fetch.startswith('/'):
                         self.logger.info(f'Landing page not valid, cannot get favicon: {rendered_url}')
                         continue
+                    if url_to_fetch.startswith('blob'):
+                        self.logger.warning(f'Cannot get a favicon from a blob URL: {url_to_fetch}')
+                        continue
 
                     if self.only_global_lookup:
-                        not_local, message = self.__check_local_url(url_to_fetch)
-                        if not_local is False:
+                        public, message = self.__check_local_url(url_to_fetch)
+                        if public is False:
                             # got a local URL
                             self.logger.warning(f'Attempted to get a local favicon: {message}')
                             continue
@@ -2315,7 +2337,7 @@ class Capture():
                     if url_to_fetch in self._requests:
                         favicon = self._requests[url_to_fetch]
                     if not favicon:
-                        async with session.get(url_to_fetch, ssl=False) as favicon_response:
+                        async with session.get(url_to_fetch, ssl=False, middlewares=[__check_local_middleware]) as favicon_response:
                             favicon_response.raise_for_status()
                             favicon = await favicon_response.read()
                     if favicon:

{playwrightcapture-1.41.0 → playwrightcapture-1.41.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "PlaywrightCapture"
-version = "1.41.0"
+version = "1.41.2"
 description = "A simple library to capture websites using playwright"
 authors = [
     {name="Raphaël Vinot", email= "raphael.vinot@circl.lu"}