PlaywrightCapture 1.24.11__tar.gz → 1.25.0__tar.gz

{playwrightcapture-1.24.11 → playwrightcapture-1.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.11
+Version: 1.25.0
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,14 +26,14 @@ Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.44.0,<2.0.0)
+Requires-Dist: playwright (>=1.45.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.23,<2.0)
+Requires-Dist: puremagic (>=1.25,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=70.0.0,<71.0.0)
+Requires-Dist: setuptools (>=70.2.0,<71.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
-Requires-Dist: w3lib (>=2.1.2,<3.0.0)
+Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture
 Description-Content-Type: text/markdown
 

{playwrightcapture-1.24.11 → playwrightcapture-1.25.0}/playwrightcapture/capture.py

@@ -521,7 +521,7 @@ class Capture():
             elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (alert dialog), but no button to click through.')
 
         await page.add_locator_handler(
             page.get_by_role("alertdialog").last,
@@ -542,7 +542,7 @@ class Capture():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
             page.get_by_role("dialog").last,
             handler,
@@ -584,7 +584,7 @@ class Capture():
             handler,
             times=1, no_wait_after=True
         )
-        self.logger.info('Yahoo handler added')
+        self.logger.info('Piwik handler added')
 
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
@@ -722,7 +722,6 @@ class Capture():
                     await page.bring_to_front()
                     self.logger.debug('Page moved to front.')
                 except Error as e:
-                    self.should_retry = True
                     self.logger.warning('Page in a broken state.')
                     raise e
 
@@ -757,8 +756,11 @@ class Capture():
                     if allow_tracking:
                         await self._wait_for_random_timeout(page, 2)
                         # This event is required trigger the add_locator_handler
-                        if await page.locator("body").first.is_visible():
-                            await page.locator("body").first.click(button="right", timeout=2000)
+                        try:
+                            if await page.locator("body").first.is_visible():
+                                await page.locator("body").first.click(button="right", timeout=5000)
+                        except Exception as e:
+                            self.logger.warning(f'Could not find body: {e}')
 
                     # move mouse
                     await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
@@ -920,19 +922,20 @@ class Capture():
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
                             'Target page, context or browser has been closed',
-                            'Protocol error (Page.bringToFront): Not attached to an active page',
                             'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
                             'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
-                            'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Handshake failed',
                             'Peer sent fatal TLS alert: Internal error',
+                            'Peer sent fatal TLS alert: The server name sent was not recognized',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
                             'Error receiving data: Connection reset by peer',
                             'Internal SOCKSv5 proxy server error.',
                             'Host unreachable through SOCKSv5 server.',
-                            'HTTP/2 Error: NO_ERROR']:
+                            'HTTP/2 Error: NO_ERROR',
+                            'HTTP/2 Error: PROTOCOL_ERROR']:
                 # Other errors, let's give it another shot
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
@@ -942,16 +945,18 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
                             'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
-                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
-                # No need to retry, the credentials are wrong/missing.
+                            'net::ERR_CERT_DATE_INVALID',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH',
+                            'net::ERR_UNSAFE_PORT']:
+                # No need to retry, the credentials/certs are wrong/missing.
                 pass
-            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to', 'Page.bringToFront']]):
                 self.should_retry = True
             elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
                 pass
             else:
                 # Unexpected ones
-                self.logger.exception(f'Something went poorly with {url}: {e.message}')
+                self.logger.exception(f'Something went poorly with {url}: "{e.name}" - {e.message}')
         except Exception as e:
             # we may get a non-playwright exception to.
             # The ones we try to handle here should be treated as if they were.
@@ -1194,8 +1199,13 @@ class Capture():
                 'net::ERR_EMPTY_RESPONSE',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
+                'net::ERR_INVALID_REDIRECT',
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
+                'net::ERR_NETWORK_ACCESS_DENIED',
+                'net::ERR_QUIC_PROTOCOL_ERROR',
+                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
+                'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
                 'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
                 'net::ERR_SSL_PROTOCOL_ERROR',

{playwrightcapture-1.24.11 → playwrightcapture-1.25.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.11"
+version = "1.25.0"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -19,17 +19,17 @@ classifiers=[
 
 [tool.poetry.dependencies]
 python = "^3.8"
-playwright = "^1.44.0"
+playwright = "^1.45.0"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
-w3lib = "^2.1.2"
+w3lib = "^2.2.1"
 pydub = {version = "^0.25.1", optional = true}
 SpeechRecognition = {version = "^3.10.4", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^70.0.0"
-puremagic = "^1.23"
+setuptools = "^70.2.0"
+puremagic = "^1.25"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 aiohttp = {extras = ["speedups"], version = "^3.9.5"}
 aiohttp-socks = "^0.8.4"
@@ -42,8 +42,8 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240511"
-pytest = "^8.2.1"
-mypy = "^1.10.0"
+pytest = "^8.2.2"
+mypy = "^1.10.1"
 types-dateparser = "^1.2.0.20240420"
 types-pytz = "^2024.1.0.20240417"