PlaywrightCapture 1.24.10__py3-none-any.whl → 1.25.0__py3-none-any.whl

playwrightcapture/capture.py

@@ -474,7 +474,7 @@ class Capture():
             if await page.locator("#didomi-notice-agree-button").is_visible():
                 await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
-        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler)
+        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
 
     async def __dialog_onetrust_clickthrough(self, page: Page) -> None:
@@ -484,7 +484,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#onetrust-banner-sdk').last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('OT handler added')
 
@@ -495,7 +496,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#hs-eu-cookie-confirmation').last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('HS handler added')
 
@@ -506,7 +508,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#CybotCookiebotDialogBody'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Cookiebot handler added')
 
@@ -518,11 +521,12 @@ class Capture():
             elif await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator('#onetrust-button-group').locator("#onetrust-accept-btn-handler").click(timeout=1000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (alert dialog), but no button to click through.')
 
         await page.add_locator_handler(
             page.get_by_role("alertdialog").last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('alert dialog handler added')
 
@@ -538,21 +542,23 @@ class Capture():
                 self.logger.info('Consent window found, clicking through.')
                 await page.get_by_test_id("uc-accept-all-button").click(timeout=2000)
             else:
-                self.logger.info('Consent window found, but no button to click through.')
+                self.logger.info('Consent window found (dialog), but no button to click through.')
         await page.add_locator_handler(
             page.get_by_role("dialog").last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('dialog handler added')
 
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
-                await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
+            if await page.locator('.cmplz-show').first.locator("button.cmplz-accept").is_visible():
+                await page.locator('.cmplz-show').first.locator("button.cmplz-accept").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('.cmplz-show'),
-            handler
+            page.locator('.cmplz-show').first,
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Complianz handler added')
 
@@ -563,7 +569,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('.con-wizard'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Yahoo handler added')
 
@@ -574,9 +581,10 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#ppms_cm_popup_overlay'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
-        self.logger.info('Yahoo handler added')
+        self.logger.info('Piwik handler added')
 
     async def capture_page(self, url: str, *, max_depth_capture_time: int,
                            referer: str | None=None,
@@ -644,7 +652,13 @@ class Capture():
             capturing_sub = True
         else:
             capturing_sub = False
-            page = await self.context.new_page()
+            try:
+                page = await self.context.new_page()
+            except Error as e:
+                self.logger.warning(f'The context is in a broken state: {e}')
+                self.should_retry = True
+                return to_return
+
             if allow_tracking:
                 # Add authorization clickthroughs
                 await self.__dialog_didomi_clickthrough(page)
@@ -704,8 +718,12 @@ class Capture():
                 else:
                     raise initial_error
             else:
-                await page.bring_to_front()
-                self.logger.debug('Page moved to front.')
+                try:
+                    await page.bring_to_front()
+                    self.logger.debug('Page moved to front.')
+                except Error as e:
+                    self.logger.warning('Page in a broken state.')
+                    raise e
 
                 # page instrumentation
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
@@ -738,8 +756,11 @@ class Capture():
                     if allow_tracking:
                         await self._wait_for_random_timeout(page, 2)
                         # This event is required trigger the add_locator_handler
-                        if await page.locator("body").first.is_visible():
-                            await page.locator("body").first.click(button="right", timeout=2000)
+                        try:
+                            if await page.locator("body").first.is_visible():
+                                await page.locator("body").first.click(button="right", timeout=5000)
+                        except Exception as e:
+                            self.logger.warning(f'Could not find body: {e}')
 
                     # move mouse
                     await page.mouse.move(x=random.uniform(300, 800), y=random.uniform(200, 500))
@@ -901,15 +922,20 @@ class Capture():
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
                             'Target page, context or browser has been closed',
-                            'Protocol error (Page.bringToFront): Not attached to an active page',
+                            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
                             'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
-                            'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Handshake failed',
                             'Peer sent fatal TLS alert: Internal error',
+                            'Peer sent fatal TLS alert: The server name sent was not recognized',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
-                            'Error receiving data: Connection reset by peer']:
+                            'Error receiving data: Connection reset by peer',
+                            'Internal SOCKSv5 proxy server error.',
+                            'Host unreachable through SOCKSv5 server.',
+                            'HTTP/2 Error: NO_ERROR',
+                            'HTTP/2 Error: PROTOCOL_ERROR']:
                 # Other errors, let's give it another shot
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
@@ -919,16 +945,18 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
                             'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
-                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
-                # No need to retry, the credentials are wrong/missing.
+                            'net::ERR_CERT_DATE_INVALID',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH',
+                            'net::ERR_UNSAFE_PORT']:
+                # No need to retry, the credentials/certs are wrong/missing.
                 pass
-            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to', 'Page.bringToFront']]):
                 self.should_retry = True
             elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
                 pass
             else:
                 # Unexpected ones
-                self.logger.exception(f'Something went poorly with {url}: {e.message}')
+                self.logger.exception(f'Something went poorly with {url}: "{e.name}" - {e.message}')
         except Exception as e:
             # we may get a non-playwright exception to.
             # The ones we try to handle here should be treated as if they were.
@@ -1155,6 +1183,7 @@ class Capture():
                 'NS_ERROR_ABORT',
                 'NS_ERROR_CONNECTION_REFUSED',
                 'NS_ERROR_NET_INTERRUPT',
+                'NS_ERROR_NET_PARTIAL_TRANSFER',
                 'NS_ERROR_NET_RESET',
                 'NS_ERROR_NET_TIMEOUT',
                 'NS_ERROR_REDIRECT_LOOP',
@@ -1170,8 +1199,13 @@ class Capture():
                 'net::ERR_EMPTY_RESPONSE',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
+                'net::ERR_INVALID_REDIRECT',
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
+                'net::ERR_NETWORK_ACCESS_DENIED',
+                'net::ERR_QUIC_PROTOCOL_ERROR',
+                'net::ERR_RESPONSE_HEADERS_TRUNCATED',
+                'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
                 'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
                 'net::ERR_SSL_PROTOCOL_ERROR',

{playwrightcapture-1.24.10.dist-info → playwrightcapture-1.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.10
+Version: 1.25.0
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,14 +26,14 @@ Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.44.0,<2.0.0)
+Requires-Dist: playwright (>=1.45.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.23,<2.0)
+Requires-Dist: puremagic (>=1.25,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=69.5.1,<70.0.0)
+Requires-Dist: setuptools (>=70.2.0,<71.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
-Requires-Dist: w3lib (>=2.1.2,<3.0.0)
+Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture
 Description-Content-Type: text/markdown
 

playwrightcapture-1.25.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
+playwrightcapture/capture.py,sha256=zzoZQItpKDbxpfF0PqFANfeWTmQlSwnvChuz_l1Ah-I,67333
+playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
+playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
+playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+playwrightcapture-1.25.0.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.0.dist-info/METADATA,sha256=XBYGqQxi3Qvc-ktd1lLGFBfSKRmCLkH5UkbzNPeL8kA,3173
+playwrightcapture-1.25.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.0.dist-info/RECORD,,

playwrightcapture-1.24.10.dist-info/RECORD

@@ -1,9 +0,0 @@
-playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=SQKfPz_PoySwvW3GCMRTTsElYVgZ5c9lB55srIxis8s,65604
-playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
-playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
-playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.24.10.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.24.10.dist-info/METADATA,sha256=wd_znffwaPZexymxHzn2Tzl2P0UHFHcbiWeZuSpyHpg,3174
-playwrightcapture-1.24.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.24.10.dist-info/RECORD,,