PlaywrightCapture 1.25.14__py3-none-any.whl → 1.25.15__py3-none-any.whl

playwrightcapture/capture.py

@@ -236,12 +236,12 @@ class Capture():
 
         return self
 
-    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
         if hasattr(self, '_temp_harfile'):
             os.unlink(self._temp_harfile.name)
 
         try:
-            await self.browser.close()
+            await self.browser.close(reason="Closing browser at the end of the capture.")
         except Exception as e:
             # We may land in a situation where the capture was forcefully closed and the browser is already closed
             self.logger.info(f'Unable to close browser: {e}')
@@ -250,6 +250,7 @@ class Capture():
         except Exception as e:
             # this should't happen, but just in case it does...
             self.logger.info(f'Unable to stop playwright: {e}')
+        return True
 
     @property
     def locale(self) -> str:
@@ -746,7 +747,7 @@ class Capture():
                                                         (time + 5) * 1000))
                 self.logger.debug(f'Moved time forward by ~{time}s.')
         except (TimeoutError, asyncio.TimeoutError):
-            self.logger.warning('Unable to move time forward.')
+            self.logger.info('Unable to move time forward.')
 
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
@@ -1118,61 +1119,19 @@ class Capture():
             to_return['error'] = f"The target was closed - {e}"
             self.should_retry = True
         except Error as e:
+            # NOTE: there are a lot of errors that look like duplicates and they are trggered at different times in the process.
+            # it is tricky to figure our which one whouls (and should not) trigger a retry. Below is our best guess and it will change over time.
             self._update_exceptions(e)
             to_return['error'] = e.message
             to_return['error_name'] = e.name
             # TODO: check e.message and figure out if it is worth retrying or not.
             # NOTE: e.name is generally (always?) "Error"
-            if self._exception_is_network_error(e):
-                # Expected errors
+            if self._fatal_network_error(e) or self._fatal_auth_error(e) or self.fatal_browser_error(e):
                 self.logger.info(f'Unable to process {url}: {e.name}')
-                if e.name == 'net::ERR_CONNECTION_RESET':
-                    self.should_retry = True
-            elif e.name in ['NS_BINDING_CANCELLED_OLD_LOAD',
-                            'NS_BINDING_ABORTED',
-                            'NS_ERROR_PARSED_DATA_CACHED',
-                            'NS_ERROR_DOCUMENT_NOT_CACHED']:
+            elif self._retry_network_error(e) or self._retry_browser_error(e):
                 # this one sounds like something we can retry...
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
-            elif e.name in ['Download is starting',
-                            'Connection closed',
-                            'Connection terminated unexpectedly',
-                            'Navigation interrupted by another one',
-                            'Navigation failed because page was closed!',
-                            'Target page, context or browser has been closed',
-                            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
-                            'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
-                            'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
-                            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
-                            'Peer sent fatal TLS alert: Handshake failed',
-                            'Peer sent fatal TLS alert: Internal error',
-                            'Peer sent fatal TLS alert: The server name sent was not recognized',
-                            'Load cannot follow more than 20 redirections',
-                            'Page crashed',
-                            'Error receiving data: Connection reset by peer',
-                            'Internal SOCKSv5 proxy server error.',
-                            'Host unreachable through SOCKSv5 server.',
-                            'HTTP/2 Error: NO_ERROR',
-                            'HTTP/2 Error: PROTOCOL_ERROR']:
-                # Other errors, let's give it another shot
-                self.logger.info(f'Issue with {url} (retrying): {e.message}')
-                self.should_retry = True
-            elif e.name in ['Target page, context or browser has been closed']:
-                # The browser barfed, let's try again
-                self.logger.info(f'Browser barfed on {url} (retrying): {e.message}')
-                self.should_retry = True
-            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
-                            'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
-                            'net::ERR_CERT_DATE_INVALID',
-                            'net::ERR_UNEXPECTED_PROXY_AUTH',
-                            'net::ERR_UNSAFE_PORT']:
-                # No need to retry, the credentials/certs are wrong/missing.
-                pass
-            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to', 'Page.bringToFront']]):
-                self.should_retry = True
-            elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
-                pass
             else:
                 # Unexpected ones
                 self.logger.exception(f'Something went poorly with {url}: "{e.name}" - {e.message}')
@@ -1196,14 +1155,20 @@ class Capture():
                         to_return['error'] = f'Unable to get the cookies: {e}'
                 # frames_tree = self.make_frame_tree(page.main_frame)
                 try:
-                    page.remove_listener("requestfinished", store_request)
-                    await page.close()
-                    await self.context.close()  # context needs to be closed to generate the HAR
-                    self.logger.debug('Context closed.')
-                    with open(self._temp_harfile.name) as _har:
-                        to_return['har'] = json.load(_har)
-                    self.logger.debug('Got HAR.')
+                    async with timeout(60):
+                        page.remove_listener("requestfinished", store_request)
+                        await page.close(reason="Closing the page because the capture finished.")
+                        self.logger.debug('Page closed.')
+                        await self.context.close(reason="Closing the context because the capture finished.")  # context needs to be closed to generate the HAR
+                        self.logger.debug('Context closed.')
+                        with open(self._temp_harfile.name) as _har:
+                            to_return['har'] = json.load(_har)
+                        self.logger.debug('Got HAR.')
+                except (TimeoutError, asyncio.TimeoutError):
+                    self.logger.warning("Unable to close page and context at the end of the capture.")
+                    self.should_retry = True
                 except Exception as e:
+                    self.logger.warning("Other exception while finishingup the capture: {e}.")
                     if 'error' not in to_return:
                         to_return['error'] = f'Unable to generate HAR file: {e}'
         self.logger.debug('Capture done')
@@ -1212,17 +1177,27 @@ class Capture():
     async def _failsafe_get_screenshot(self, page: Page) -> bytes:
         self.logger.debug("Capturing a screenshot of the full page.")
         try:
-            return await page.screenshot(full_page=True, timeout=10000)
+            async with timeout(15):
+                return await page.screenshot(full_page=True, timeout=10000)
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.info("Screenshot of the full page got stuck, trying to scale it down.")
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to scale it down: {e}")
 
         try:
-            return await page.screenshot(full_page=True, scale="css", timeout=30000)
+            async with timeout(35):
+                return await page.screenshot(full_page=True, scale="css", timeout=30000)
+        except (TimeoutError, asyncio.TimeoutError):
+            self.logger.info("Screenshot of the full page got stuck, trying to get the current viewport only.")
         except Error as e:
             self.logger.info(f"Capturing a screenshot of the full page failed, trying to get the current viewport only: {e}")
 
         try:
-            return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
+            async with timeout(10):
+                return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
+        except (TimeoutError, asyncio.TimeoutError) as e:
+            self.logger.info("Screenshot of the full page got stuck, unable to get any screenshot.")
+            raise e
         except Error as e:
             self.logger.info(f"Unable to get any screenshot: {e}")
             raise e
@@ -1404,12 +1379,66 @@ class Capture():
                 _, name = exception.message.split(': ', maxsplit=1)
             exception._name = name.strip()
 
-    def _exception_is_network_error(self, exception: Error) -> bool:
+    def _retry_browser_error(self, exception: Error) -> bool:
+        if exception.name in [
+            'Download is starting',
+            'Connection closed',
+            'Connection terminated unexpectedly',
+            'Navigation interrupted by another one',
+            'Navigation failed because page was closed!',
+            'Target page, context or browser has been closed',
+            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
+            'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
+            'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
+            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
+            'Peer sent fatal TLS alert: Handshake failed',
+            'Peer sent fatal TLS alert: Internal error',
+            'Peer sent fatal TLS alert: The server name sent was not recognized',
+            'Load cannot follow more than 20 redirections',
+            'Page crashed',
+            'Error receiving data: Connection reset by peer',
+            'Internal SOCKSv5 proxy server error.',
+            'Host unreachable through SOCKSv5 server.',
+            # The browser barfed
+            'Target page, context or browser has been closed',
+        ]:
+            # Other errors, let's give it another shot
+            return True
+        elif exception.name and any(msg in exception.name for msg in ['is interrupted by another navigation to',
+                                                                      'Page.bringToFront',
+                                                                      'TypeError']):
+            # Match on partial string with variable content
+            return True
+        return False
+
+    def _retry_network_error(self, exception: Error) -> bool:
+        if exception.name in [
+                'HTTP/2 Error: NO_ERROR',
+                'HTTP/2 Error: PROTOCOL_ERROR',
+                'NS_BINDING_ABORTED',
+                'NS_BINDING_CANCELLED_OLD_LOAD',
+                'NS_ERROR_DOCUMENT_NOT_CACHED',
+                'NS_ERROR_NET_PARTIAL_TRANSFER',
+                'NS_ERROR_PARSED_DATA_CACHED',
+                'net::ERR_CONNECTION_RESET',
+                'net::ERR_EMPTY_RESPONSE',
+                'net::ERR_INVALID_RESPONSE',
+                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
+                'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
+        ]:
+            return True
+        return False
+
+    def fatal_browser_error(self, exception: Error) -> bool:
+        if exception.name and any(msg in exception.name for msg in ['Error resolving', 'Could not connect to']):
+            return True
+        return False
+
+    def _fatal_network_error(self, exception: Error) -> bool:
         if exception.name in [
                 'NS_ERROR_ABORT',
                 'NS_ERROR_CONNECTION_REFUSED',
                 'NS_ERROR_NET_INTERRUPT',
-                'NS_ERROR_NET_PARTIAL_TRANSFER',
                 'NS_ERROR_NET_RESET',
                 'NS_ERROR_NET_TIMEOUT',
                 'NS_ERROR_REDIRECT_LOOP',
@@ -1420,30 +1449,37 @@ class Capture():
                 'net::ERR_ADDRESS_UNREACHABLE',
                 'net::ERR_CONNECTION_CLOSED',
                 'net::ERR_CONNECTION_REFUSED',
-                'net::ERR_CONNECTION_RESET',
                 'net::ERR_CONNECTION_TIMED_OUT',
-                'net::ERR_EMPTY_RESPONSE',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
                 'net::ERR_INVALID_REDIRECT',
-                'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_NETWORK_ACCESS_DENIED',
                 'net::ERR_QUIC_PROTOCOL_ERROR',
-                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
                 'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
                 'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
                 'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_SSL_UNRECOGNIZED_NAME_ALERT',
-                'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
                 'net::ERR_TIMED_OUT',
                 'net::ERR_TOO_MANY_REDIRECTS',
+                'net::ERR_UNSAFE_PORT',
                 'SSL_ERROR_UNKNOWN',
         ]:
             return True
         return False
 
+    def _fatal_auth_error(self, exception: Error) -> bool:
+        if exception.name in [
+                'net::ERR_INVALID_AUTH_CREDENTIALS',
+                'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
+                'net::ERR_CERT_DATE_INVALID',
+                'net::ERR_UNEXPECTED_PROXY_AUTH',
+        ]:
+            # No need to retry, the credentials/certs are wrong/missing.
+            return True
+        return False
+
     async def _wait_for_random_timeout(self, page: Page, timeout: int) -> None:
         '''Instead of waiting for the exact same time, we wait +-500ms around the given time. The time is fiven in seconds for simplicity's sake.'''
         if timeout > 1000:

{playwrightcapture-1.25.14.dist-info → playwrightcapture-1.25.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.14
+Version: 1.25.15
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,12 +26,12 @@ Requires-Dist: aiohttp[speedups] (>=3.10.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.46.0,<2.0.0)
+Requires-Dist: playwright (>=1.47.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.27,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
-Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=74.0.0,<75.0.0)
+Requires-Dist: pytz (>=2024.2,<2025.0) ; python_version < "3.9"
+Requires-Dist: setuptools (>=74.1.2,<75.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.14.dist-info → playwrightcapture-1.25.15.dist-info}/RECORD

@@ -1,9 +1,9 @@
 playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=9ESwyexAnBtGc2QV3u4WvOKZqisMVdqHL4rko1RxJXQ,78337
+playwrightcapture/capture.py,sha256=8qoKqKnd3xt_6WuxJi2DmagC8fLx6O-RPnb5yPNkdTQ,79840
 playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
 playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
 playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.25.14.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.25.14.dist-info/METADATA,sha256=45rFgcxqSi2TAU8KwsE0dxdWja1u7tvHchj0x3H1dCM,3172
-playwrightcapture-1.25.14.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.25.14.dist-info/RECORD,,
+playwrightcapture-1.25.15.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.15.dist-info/METADATA,sha256=H_yOrQiJCbAePrkN66zoShlhvNY4AyYfdV2aHKO5uJg,3172
+playwrightcapture-1.25.15.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.15.dist-info/RECORD,,