PlaywrightCapture 1.24.9__tar.gz → 1.24.11__tar.gz

{playwrightcapture-1.24.9 → playwrightcapture-1.24.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.9
+Version: 1.24.11
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,12 +26,12 @@ Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.43.0,<2.0.0)
+Requires-Dist: playwright (>=1.44.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.23,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=69.5.1,<70.0.0)
+Requires-Dist: setuptools (>=70.0.0,<71.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.1.2,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.24.9 → playwrightcapture-1.24.11}/playwrightcapture/capture.py

@@ -474,7 +474,7 @@ class Capture():
             if await page.locator("#didomi-notice-agree-button").is_visible():
                 await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
-        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler)
+        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler, times=1, no_wait_after=True)
         self.logger.info('Didomi handler added')
 
     async def __dialog_onetrust_clickthrough(self, page: Page) -> None:
@@ -484,7 +484,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#onetrust-banner-sdk').last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('OT handler added')
 
@@ -495,7 +496,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#hs-eu-cookie-confirmation').last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('HS handler added')
 
@@ -506,7 +508,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#CybotCookiebotDialogBody'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Cookiebot handler added')
 
@@ -522,7 +525,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.get_by_role("alertdialog").last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('alert dialog handler added')
 
@@ -541,18 +545,20 @@ class Capture():
                 self.logger.info('Consent window found, but no button to click through.')
         await page.add_locator_handler(
             page.get_by_role("dialog").last,
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('dialog handler added')
 
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
-                await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
+            if await page.locator('.cmplz-show').first.locator("button.cmplz-accept").is_visible():
+                await page.locator('.cmplz-show').first.locator("button.cmplz-accept").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('.cmplz-show'),
-            handler
+            page.locator('.cmplz-show').first,
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Complianz handler added')
 
@@ -563,7 +569,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('.con-wizard'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Yahoo handler added')
 
@@ -574,7 +581,8 @@ class Capture():
 
         await page.add_locator_handler(
             page.locator('#ppms_cm_popup_overlay'),
-            handler
+            handler,
+            times=1, no_wait_after=True
         )
         self.logger.info('Yahoo handler added')
 
@@ -644,7 +652,13 @@ class Capture():
             capturing_sub = True
         else:
             capturing_sub = False
-            page = await self.context.new_page()
+            try:
+                page = await self.context.new_page()
+            except Error as e:
+                self.logger.warning(f'The context is in a broken state: {e}')
+                self.should_retry = True
+                return to_return
+
             if allow_tracking:
                 # Add authorization clickthroughs
                 await self.__dialog_didomi_clickthrough(page)
@@ -704,8 +718,13 @@ class Capture():
                 else:
                     raise initial_error
             else:
-                await page.bring_to_front()
-                self.logger.debug('Page moved to front.')
+                try:
+                    await page.bring_to_front()
+                    self.logger.debug('Page moved to front.')
+                except Error as e:
+                    self.should_retry = True
+                    self.logger.warning('Page in a broken state.')
+                    raise e
 
                 # page instrumentation
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
@@ -902,6 +921,7 @@ class Capture():
                             'Navigation failed because page was closed!',
                             'Target page, context or browser has been closed',
                             'Protocol error (Page.bringToFront): Not attached to an active page',
+                            'Peer failed to perform TLS handshake: A packet with illegal or unsupported version was received.',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
                             'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
@@ -909,7 +929,10 @@ class Capture():
                             'Peer sent fatal TLS alert: Internal error',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
-                            'Error receiving data: Connection reset by peer']:
+                            'Error receiving data: Connection reset by peer',
+                            'Internal SOCKSv5 proxy server error.',
+                            'Host unreachable through SOCKSv5 server.',
+                            'HTTP/2 Error: NO_ERROR']:
                 # Other errors, let's give it another shot
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
@@ -1155,6 +1178,7 @@ class Capture():
                 'NS_ERROR_ABORT',
                 'NS_ERROR_CONNECTION_REFUSED',
                 'NS_ERROR_NET_INTERRUPT',
+                'NS_ERROR_NET_PARTIAL_TRANSFER',
                 'NS_ERROR_NET_RESET',
                 'NS_ERROR_NET_TIMEOUT',
                 'NS_ERROR_REDIRECT_LOOP',

{playwrightcapture-1.24.9 → playwrightcapture-1.24.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.9"
+version = "1.24.11"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -19,7 +19,7 @@ classifiers=[
 
 [tool.poetry.dependencies]
 python = "^3.8"
-playwright = "^1.43.0"
+playwright = "^1.44.0"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.1.2"
@@ -28,7 +28,7 @@ SpeechRecognition = {version = "^3.10.4", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^69.5.1"
+setuptools = "^70.0.0"
 puremagic = "^1.23"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 aiohttp = {extras = ["speedups"], version = "^3.9.5"}
@@ -42,7 +42,7 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240511"
-pytest = "^8.2.0"
+pytest = "^8.2.1"
 mypy = "^1.10.0"
 types-dateparser = "^1.2.0.20240420"
 types-pytz = "^2024.1.0.20240417"