PlaywrightCapture 1.24.5__tar.gz → 1.24.7__tar.gz

{playwrightcapture-1.24.5 → playwrightcapture-1.24.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.5
+Version: 1.24.7
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -21,11 +21,12 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.3,<4.0.0) ; extra == "recaptcha"
+Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
 Requires-Dist: playwright (>=1.43.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.21,<2.0)
+Requires-Dist: puremagic (>=1.22,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
 Requires-Dist: requests[socks] (>=2.31.0,<3.0.0) ; extra == "recaptcha"

{playwrightcapture-1.24.5 → playwrightcapture-1.24.7}/playwrightcapture/capture.py

@@ -43,6 +43,11 @@ else:
     from zoneinfo import available_timezones
     all_timezones_set = available_timezones()
 
+if sys.version_info < (3, 11):
+    from async_timeout import timeout
+else:
+    from asyncio import timeout
+
 if TYPE_CHECKING:
     from playwright._impl._api_structures import (SetCookieParam, Geolocation,
                                                   HttpCredentials, Headers,
@@ -469,36 +474,33 @@ class Capture():
             if await page.locator("#didomi-notice-agree-button").is_visible():
                 await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
-        await page.add_locator_handler(page.locator(".didomi-popup-view"), handler)
+        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler)
         self.logger.info('Didomi handler added')
 
     async def __dialog_onetrust_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## OT Dialog found, clicking through.')
             if await page.locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator("#onetrust-accept-btn-handler").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#onetrust-banner-sdk'),
+            page.locator('#onetrust-banner-sdk').last,
             handler
         )
         self.logger.info('OT handler added')
 
     async def __dialog_hubspot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## HS Dialog found, clicking through.')
             if await page.locator("#hs-eu-confirmation-button").is_visible():
                 await page.locator("#hs-eu-confirmation-button").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#hs-eu-cookie-confirmation'),
+            page.locator('#hs-eu-cookie-confirmation').last,
             handler
         )
         self.logger.info('HS handler added')
 
     async def __dialog_cookiebot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Cookiebot Dialog found, clicking through.')
             if await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").is_visible():
                 await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").click(timeout=2000)
 
@@ -519,7 +521,7 @@ class Capture():
                 self.logger.info('Consent window found, but no button to click through.')
 
         await page.add_locator_handler(
-            page.get_by_role("alertdialog"),
+            page.get_by_role("alertdialog").last,
             handler
         )
         self.logger.info('alert dialog handler added')
@@ -545,7 +547,6 @@ class Capture():
 
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Complianz found, clicking through.')
             if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
                 await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
 
@@ -557,7 +558,6 @@ class Capture():
 
     async def __dialog_yahoo_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Yahoo found, clicking through.')
             if await page.locator('.con-wizard').locator("button.accept-all").is_visible():
                 await page.locator('.con-wizard').locator("button.accept-all").click(timeout=2000)
 
@@ -569,7 +569,6 @@ class Capture():
 
     async def __dialog_ppms_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## piwik found, clicking through.')
             if await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").is_visible():
                 await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").click(timeout=2000)
 
@@ -610,7 +609,11 @@ class Capture():
                     multiple_downloads.append((filename, file_content))
                     self.logger.info('Done with download.')
             except Exception as e:
-                self.logger.warning(f'Unable to finish download triggered from JS: {e}')
+                if download.page.is_closed():
+                    # Page is closed, skip logging.
+                    pass
+                else:
+                    self.logger.warning(f'Unable to finish download triggered from JS: {e}')
             finally:
                 self.wait_for_download -= 1
 
@@ -664,7 +667,6 @@ class Capture():
             parsed_url = urlparse(url, allow_fragments=True)
 
             try:
-                # NOTE 2022-12-02: allow 15s less than the general timeout to get a DOM
                 await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else '')
                 page.on("download", handle_download)
             except Error as initial_error:
@@ -829,24 +831,39 @@ class Capture():
                             else:
                                 child_urls = child_urls[:max_captures]
                         self.logger.info(f'Capturing children, {max_captures} URLs')
+                        consecutive_errors = 0
                         for index, url in enumerate(child_urls):
                             self.logger.info(f'Capture child {url} - Timeout: {max_capture_time}s')
                             start_time = time.time()
+                            if page.is_closed():
+                                self.logger.info('Page is closed, unable to capture children.')
+                                break
                             try:
-                                child_capture = await asyncio.wait_for(
-                                    self.capture_page(url=url, referer=page.url,
-                                                      page=page, depth=depth,
-                                                      rendered_hostname_only=rendered_hostname_only,
-                                                      max_depth_capture_time=max_capture_time),
-                                    timeout=max_capture_time + 1)  # just adding a bit of padding so playwright has the chance to raise the exception first
-                                to_return['children'].append(child_capture)  # type: ignore[union-attr]
-                            except (TimeoutError, asyncio.exceptions.TimeoutError):
+                                async with timeout(max_capture_time + 1):  # just adding a bit of padding so playwright has the chance to raise the exception first
+                                    child_capture = await self.capture_page(
+                                        url=url, referer=page.url,
+                                        page=page, depth=depth,
+                                        rendered_hostname_only=rendered_hostname_only,
+                                        max_depth_capture_time=max_capture_time)
+                                    to_return['children'].append(child_capture)  # type: ignore[union-attr]
+                            except (TimeoutError, asyncio.exceptions.TimeoutError, asyncio.TimeoutError):
                                 self.logger.info(f'Timeout error, took more than {max_capture_time}s. Unable to capture {url}.')
+                                consecutive_errors += 1
                             except Exception as e:
                                 self.logger.warning(f'Error while capturing child "{url}": {e}. {len(child_urls) - index - 1} more to go.')
+                                consecutive_errors += 1
                             else:
+                                consecutive_errors = 0
                                 runtime = int(time.time() - start_time)
                                 self.logger.info(f'Successfully captured child URL: {url} in {runtime}s. {len(child_urls) - index - 1} to go.')
+
+                            if consecutive_errors >= 5:
+                                # if we have more than 5 consecutive errors, the capture is most probably broken, breaking.
+                                self.logger.warning('Got more than 5 consecutive errors while capturing children, breaking.')
+                                to_return['error'] = "Got more than 5 consecutive errors while capturing children"
+                                self.should_retry = True
+                                break
+
                             try:
                                 await page.go_back()
                             except PlaywrightTimeoutError:
@@ -880,12 +897,16 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['Download is starting',
                             'Connection closed',
+                            'Connection terminated unexpectedly',
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
+                            'Target page, context or browser has been closed',
                             'Protocol error (Page.bringToFront): Not attached to an active page',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
+                            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
                             'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Internal error',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
                             'Error receiving data: Connection reset by peer']:
@@ -896,9 +917,15 @@ class Capture():
                 # The browser barfed, let's try again
                 self.logger.info(f'Browser barfed on {url} (retrying): {e.message}')
                 self.should_retry = True
-            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS']:
+            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
+                            'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
                 # No need to retry, the credentials are wrong/missing.
                 pass
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+                self.should_retry = True
+            elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
+                pass
             else:
                 # Unexpected ones
                 self.logger.exception(f'Something went poorly with {url}: {e.message}')
@@ -949,7 +976,7 @@ class Capture():
         try:
             return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
         except Error as e:
-            self.logger.warning(f"Unable to get any screenshot: {e}")
+            self.logger.info(f"Unable to get any screenshot: {e}")
             raise e
 
     async def _safe_wait(self, page: Page, force_max_wait_in_sec: int | None=None) -> None:
@@ -1137,11 +1164,13 @@ class Capture():
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
+                'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
+                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_SSL_UNRECOGNIZED_NAME_ALERT',
                 'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
-                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_TIMED_OUT',
                 'net::ERR_TOO_MANY_REDIRECTS',
+                'SSL_ERROR_UNKNOWN',
         ]:
             return True
         return False

{playwrightcapture-1.24.5 → playwrightcapture-1.24.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.5"
+version = "1.24.7"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -30,7 +30,8 @@ pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
 setuptools = "^69.5.1"
-puremagic = "^1.21"
+puremagic = "^1.22"
+async-timeout = {version = "^4.0.3", python = "<3.11"}
 
 [tool.poetry.extras]
 recaptcha = ["requests", "pydub", "SpeechRecognition"]
@@ -40,9 +41,9 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240229"
-pytest = "^8.1.1"
-mypy = "^1.9.0"
-types-dateparser = "^1.1.4.20240331"
+pytest = "^8.2.0"
+mypy = "^1.10.0"
+types-dateparser = "^1.2.0.20240420"
 types-requests = "^2.31.0.20240406"
 types-pytz = "^2024.1.0.20240417"