PlaywrightCapture 1.25.13__tar.gz → 1.25.15__tar.gz

{playwrightcapture-1.25.13 → playwrightcapture-1.25.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.13
+Version: 1.25.15
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -22,16 +22,16 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.4,<4.0.0) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.9,<0.10)
-Requires-Dist: aiohttp[speedups] (>=3.10.3,<4.0.0)
+Requires-Dist: aiohttp[speedups] (>=3.10.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.46.0,<2.0.0)
+Requires-Dist: playwright (>=1.47.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.27,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
-Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=72.2.0,<73.0.0)
+Requires-Dist: pytz (>=2024.2,<2025.0) ; python_version < "3.9"
+Requires-Dist: setuptools (>=74.1.2,<75.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.13 → playwrightcapture-1.25.15}/playwrightcapture/capture.py

@@ -236,12 +236,12 @@ class Capture():
 
         return self
 
-    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
         if hasattr(self, '_temp_harfile'):
             os.unlink(self._temp_harfile.name)
 
         try:
-            await self.browser.close()
+            await self.browser.close(reason="Closing browser at the end of the capture.")
         except Exception as e:
             # We may land in a situation where the capture was forcefully closed and the browser is already closed
             self.logger.info(f'Unable to close browser: {e}')
@@ -250,6 +250,7 @@ class Capture():
         except Exception as e:
             # this should't happen, but just in case it does...
             self.logger.info(f'Unable to stop playwright: {e}')
+        return True
 
     @property
     def locale(self) -> str:
@@ -746,7 +747,7 @@ class Capture():
                                                         (time + 5) * 1000))
                 self.logger.debug(f'Moved time forward by ~{time}s.')
         except (TimeoutError, asyncio.TimeoutError):
-            self.logger.warning('Unable to move time forward.')
+            self.logger.info('Unable to move time forward.')
 
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
@@ -1118,61 +1119,19 @@ class Capture():
             to_return['error'] = f"The target was closed - {e}"
             self.should_retry = True
         except Error as e:
+            # NOTE: there are a lot of errors that look like duplicates and they are trggered at different times in the process.
+            # it is tricky to figure our which one whouls (and should not) trigger a retry. Below is our best guess and it will change over time.
             self._update_exceptions(e)
             to_return['error'] = e.message
             to_return['error_name'] = e.name
             # TODO: check e.message and figure out if it is worth retrying or not.
             # NOTE: e.name is generally (always?) "Error"
-            if self._exception_is_network_error(e):
-                # Expected errors
+            if self._fatal_network_error(e) or self._fatal_auth_error(e) or self.fatal_browser_error(e):
                 self.logger.info(f'Unable to process {url}: {e.name}')
-                if e.name == 'net::ERR_CONNECTION_RESET':
-                    self.should_retry = True
-            elif e.name in ['NS_BINDING_CANCELLED_OLD_LOAD',
-                            'NS_BINDING_ABORTED',
-                            'NS_ERROR_PARSED_DATA_CACHED',
-                            'NS_ERROR_DOCUMENT_NOT_CACHED']:
+            elif self._retry_network_error(e) or self._retry_browser_error(e):
                 # this one sounds like something we can retry...
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
-            elif e.name in ['Download is starting',
-                            'Connection closed',
-                            'Connection terminated unexpectedly',
-                            'Navigation interrupted by another one',
-                            'Navigation failed because page was closed!',
-                            'Target page, context or browser has been closed',
-                            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
-                            'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
-                            'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
-                            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
-                            'Peer sent fatal TLS alert: Handshake failed',
-                            'Peer sent fatal TLS alert: Internal error',
-                            'Peer sent fatal TLS alert: The server name sent was not recognized',
-                            'Load cannot follow more than 20 redirections',
-                            'Page crashed',
-                            'Error receiving data: Connection reset by peer',
-                            'Internal SOCKSv5 proxy server error.',
-                            'Host unreachable through SOCKSv5 server.',
-                            'HTTP/2 Error: NO_ERROR',
-                            'HTTP/2 Error: PROTOCOL_ERROR']:
-                # Other errors, let's give it another shot
-                self.logger.info(f'Issue with {url} (retrying): {e.message}')
-                self.should_retry = True
-            elif e.name in ['Target page, context or browser has been closed']:
-                # The browser barfed, let's try again
-                self.logger.info(f'Browser barfed on {url} (retrying): {e.message}')
-                self.should_retry = True
-            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
-                            'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
-                            'net::ERR_CERT_DATE_INVALID',
-                            'net::ERR_UNEXPECTED_PROXY_AUTH',
-                            'net::ERR_UNSAFE_PORT']:
-                # No need to retry, the credentials/certs are wrong/missing.
-                pass
-            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to', 'Page.bringToFront']]):
-                self.should_retry = True
-            elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
-                pass
             else:
                 # Unexpected ones
                 self.logger.exception(f'Something went poorly with {url}: "{e.name}" - {e.message}')
@@ -1196,14 +1155,20 @@ class Capture():
                         to_return['error'] = f'Unable to get the cookies: {e}'
                 # frames_tree = self.make_frame_tree(page.main_frame)
                 try:
-                    page.remove_listener("requestfinished", store_request)
-                    await page.close()
-                    await self.context.close()  # context needs to be closed to generate the HAR
-                    self.logger.debug('Context closed.')
-                    with open(self._temp_harfile.name) as _har:
-                        to_return['har'] = json.load(_har)
-                    self.logger.debug('Got HAR.')
+                    async with timeout(60):
+                        page.remove_listener("requestfinished", store_request)
+                        await page.close(reason="Closing the page because the capture finished.")
+                        self.logger.debug('Page closed.')
+                        await self.context.close(reason="Closing the context because the capture finished.")  # context needs to be closed to generate the HAR
+                        self.logger.debug('Context closed.')
+                        with open(self._temp_harfile.name) as _har:
+                            to_return['har'] = json.load(_har)
+                        self.logger.debug('Got HAR.')
+                except (TimeoutError, asyncio.TimeoutError):
+                    self.logger.warning("Unable to close page and context at the end of the capture.")
+                    self.should_retry = True
                 except Exception as e:
+                    self.logger.warning("Other exception while finishingup the capture: {e}.")
                     if 'error' not in to_return:
                         to_return['error'] = f'Unable to generate HAR file: {e}'
         self.logger.debug('Capture done')
@@ -1212,17 +1177,27 @@ class Capture():
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
         self.logger.debug("Capturing a screenshot of the full page.")
         try:
-            return await page.screenshot(full_page=True, timeout=10000)
+            async with timeout(15):
+                return await page.screenshot(full_page=True, timeout=10000)
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.info("Screenshot of the full page got stuck, trying to scale it down.")
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css", timeout=30000)
+            async with timeout(35):
+                return await page.screenshot(full_page=True, scale="css", timeout=30000)
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.info("Screenshot of the full page got stuck, trying to get the current viewport only.")
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 
         try:
-            return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
+            async with timeout(10):
+                return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
+        except (TimeoutError, asyncio.TimeoutError) as e:
+            self.logger.info("Screenshot of the full page got stuck, unable to get any screenshot.")
+            raise e
         except Error as e:
             self.logger.info(f"Unable to get any screenshot: {e}")
             raise e
@@ -1404,12 +1379,66 @@ class Capture():
                 _, name = exception.message.split(': ', maxsplit=1)
             exception._name = name.strip()
 
-    def _exception_is_network_error(self, exception: Error) -> bool:
+    def _retry_browser_error(self, exception: Error) -> bool:
+        if exception.name in [
+            'Download is starting',
+            'Connection closed',
+            'Connection terminated unexpectedly',
+            'Navigation interrupted by another one',
+            'Navigation failed because page was closed!',
+            'Target page, context or browser has been closed',
+            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
+            'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
+            'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
+            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
+            'Peer sent fatal TLS alert: Handshake failed',
+            'Peer sent fatal TLS alert: Internal error',
+            'Peer sent fatal TLS alert: The server name sent was not recognized',
+            'Load cannot follow more than 20 redirections',
+            'Page crashed',
+            'Error receiving data: Connection reset by peer',
+            'Internal SOCKSv5 proxy server error.',
+            'Host unreachable through SOCKSv5 server.',
+            # The browser barfed
+            'Target page, context or browser has been closed',
+        ]:
+            # Other errors, let's give it another shot
+            return True
+        elif exception.name and any(msg in exception.name for msg in ['is interrupted by another navigation to',
+                                                                      'Page.bringToFront',
+                                                                      'TypeError']):
+            # Match on partial string with variable content
+            return True
+        return False
+
+    def _retry_network_error(self, exception: Error) -> bool:
+        if exception.name in [
+                'HTTP/2 Error: NO_ERROR',
+                'HTTP/2 Error: PROTOCOL_ERROR',
+                'NS_BINDING_ABORTED',
+                'NS_BINDING_CANCELLED_OLD_LOAD',
+                'NS_ERROR_DOCUMENT_NOT_CACHED',
+                'NS_ERROR_NET_PARTIAL_TRANSFER',
+                'NS_ERROR_PARSED_DATA_CACHED',
+                'net::ERR_CONNECTION_RESET',
+                'net::ERR_EMPTY_RESPONSE',
+                'net::ERR_INVALID_RESPONSE',
+                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
+                'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
+        ]:
+            return True
+        return False
+
+    def fatal_browser_error(self, exception: Error) -> bool:
+        if exception.name and any(msg in exception.name for msg in ['Error resolving', 'Could not connect to']):
+            return True
+        return False
+
+    def _fatal_network_error(self, exception: Error) -> bool:
         if exception.name in [
                 'NS_ERROR_ABORT',
                 'NS_ERROR_CONNECTION_REFUSED',
                 'NS_ERROR_NET_INTERRUPT',
-                'NS_ERROR_NET_PARTIAL_TRANSFER',
                 'NS_ERROR_NET_RESET',
                 'NS_ERROR_NET_TIMEOUT',
                 'NS_ERROR_REDIRECT_LOOP',
@@ -1420,30 +1449,37 @@ class Capture():
                 'net::ERR_ADDRESS_UNREACHABLE',
                 'net::ERR_CONNECTION_CLOSED',
                 'net::ERR_CONNECTION_REFUSED',
-                'net::ERR_CONNECTION_RESET',
                 'net::ERR_CONNECTION_TIMED_OUT',
-                'net::ERR_EMPTY_RESPONSE',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
                 'net::ERR_INVALID_REDIRECT',
-                'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_NETWORK_ACCESS_DENIED',
                 'net::ERR_QUIC_PROTOCOL_ERROR',
-                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
                 'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
                 'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
                 'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_SSL_UNRECOGNIZED_NAME_ALERT',
-                'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
                 'net::ERR_TIMED_OUT',
                 'net::ERR_TOO_MANY_REDIRECTS',
+                'net::ERR_UNSAFE_PORT',
                 'SSL_ERROR_UNKNOWN',
         ]:
             return True
         return False
 
+    def _fatal_auth_error(self, exception: Error) -> bool:
+        if exception.name in [
+                'net::ERR_INVALID_AUTH_CREDENTIALS',
+                'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
+                'net::ERR_CERT_DATE_INVALID',
+                'net::ERR_UNEXPECTED_PROXY_AUTH',
+        ]:
+            # No need to retry, the credentials/certs are wrong/missing.
+            return True
+        return False
+
     async def _wait_for_random_timeout(self, page: Page, timeout: int) -> None:
         '''Instead of waiting for the exact same time, we wait +-500ms around the given time. The time is fiven in seconds for simplicity's sake.'''
         if timeout > 1000:

{playwrightcapture-1.25.13 → playwrightcapture-1.25.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.25.13"
+version = "1.25.15"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -19,19 +19,19 @@ classifiers=[
 
 [tool.poetry.dependencies]
 python = "^3.8"
-playwright = "^1.46.0"
+playwright = "^1.47.0"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.2.1"
 pydub = {version = "^0.25.1", optional = true}
 SpeechRecognition = {version = "^3.10.4", optional = true}
-pytz = {"version" = "^2024.1", python = "<3.9"}
+pytz = {"version" = "^2024.2", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^72.2.0"
+setuptools = "^74.1.2"
 puremagic = "^1.27"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
-aiohttp = {extras = ["speedups"], version = "^3.10.3"}
+aiohttp = {extras = ["speedups"], version = "^3.10.5"}
 aiohttp-socks = "^0.9"
 
 [tool.poetry.extras]
@@ -41,11 +41,11 @@ recaptcha = ["pydub", "SpeechRecognition"]
 optional = true
 
 [tool.poetry.group.dev.dependencies]
-types-beautifulsoup4 = "^4.12.0.20240511"
-pytest = "^8.3.2"
-mypy = "^1.11.1"
+types-beautifulsoup4 = "^4.12.0.20240907"
+pytest = "^8.3.3"
+mypy = "^1.11.2"
 types-dateparser = "^1.2.0.20240420"
-types-pytz = "^2024.1.0.20240417"
+types-pytz = "^2024.2.0.20240913"
 
 
 [build-system]