PlaywrightCapture 1.24.5__tar.gz → 1.24.6__tar.gz

{playwrightcapture-1.24.5 → playwrightcapture-1.24.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.5
+Version: 1.24.6
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause

{playwrightcapture-1.24.5 → playwrightcapture-1.24.6}/playwrightcapture/capture.py

@@ -43,6 +43,11 @@ else:
     from zoneinfo import available_timezones
     all_timezones_set = available_timezones()
 
+if sys.version_info < (3, 11):
+    from async_timeout import timeout
+else:
+    from asyncio import timeout
+
 if TYPE_CHECKING:
     from playwright._impl._api_structures import (SetCookieParam, Geolocation,
                                                   HttpCredentials, Headers,
@@ -469,36 +474,33 @@ class Capture():
             if await page.locator("#didomi-notice-agree-button").is_visible():
                 await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
-        await page.add_locator_handler(page.locator(".didomi-popup-view"), handler)
+        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler)
         self.logger.info('Didomi handler added')
 
     async def __dialog_onetrust_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## OT Dialog found, clicking through.')
             if await page.locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator("#onetrust-accept-btn-handler").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#onetrust-banner-sdk'),
+            page.locator('#onetrust-banner-sdk').last,
             handler
         )
         self.logger.info('OT handler added')
 
     async def __dialog_hubspot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## HS Dialog found, clicking through.')
             if await page.locator("#hs-eu-confirmation-button").is_visible():
                 await page.locator("#hs-eu-confirmation-button").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#hs-eu-cookie-confirmation'),
+            page.locator('#hs-eu-cookie-confirmation').last,
             handler
         )
         self.logger.info('HS handler added')
 
     async def __dialog_cookiebot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Cookiebot Dialog found, clicking through.')
             if await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").is_visible():
                 await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").click(timeout=2000)
 
@@ -519,7 +521,7 @@ class Capture():
                 self.logger.info('Consent window found, but no button to click through.')
 
         await page.add_locator_handler(
-            page.get_by_role("alertdialog"),
+            page.get_by_role("alertdialog").last,
             handler
         )
         self.logger.info('alert dialog handler added')
@@ -545,7 +547,6 @@ class Capture():
 
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Complianz found, clicking through.')
             if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
                 await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
 
@@ -557,7 +558,6 @@ class Capture():
 
     async def __dialog_yahoo_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Yahoo found, clicking through.')
             if await page.locator('.con-wizard').locator("button.accept-all").is_visible():
                 await page.locator('.con-wizard').locator("button.accept-all").click(timeout=2000)
 
@@ -569,7 +569,6 @@ class Capture():
 
     async def __dialog_ppms_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## piwik found, clicking through.')
             if await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").is_visible():
                 await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").click(timeout=2000)
 
@@ -610,7 +609,11 @@ class Capture():
                     multiple_downloads.append((filename, file_content))
                     self.logger.info('Done with download.')
             except Exception as e:
-                self.logger.warning(f'Unable to finish download triggered from JS: {e}')
+                if download.page.is_closed():
+                    # Page is closed, skip logging.
+                    pass
+                else:
+                    self.logger.warning(f'Unable to finish download triggered from JS: {e}')
             finally:
                 self.wait_for_download -= 1
 
@@ -832,14 +835,17 @@ class Capture():
                         for index, url in enumerate(child_urls):
                             self.logger.info(f'Capture child {url} - Timeout: {max_capture_time}s')
                             start_time = time.time()
+                            if page.is_closed():
+                                self.logger.info('Page is closed, unable to capture children.')
+                                break
                             try:
-                                child_capture = await asyncio.wait_for(
-                                    self.capture_page(url=url, referer=page.url,
-                                                      page=page, depth=depth,
-                                                      rendered_hostname_only=rendered_hostname_only,
-                                                      max_depth_capture_time=max_capture_time),
-                                    timeout=max_capture_time + 1)  # just adding a bit of padding so playwright has the chance to raise the exception first
-                                to_return['children'].append(child_capture)  # type: ignore[union-attr]
+                                async with timeout(max_capture_time + 1):  # just adding a bit of padding so playwright has the chance to raise the exception first
+                                    child_capture = await self.capture_page(
+                                        url=url, referer=page.url,
+                                        page=page, depth=depth,
+                                        rendered_hostname_only=rendered_hostname_only,
+                                        max_depth_capture_time=max_capture_time)
+                                    to_return['children'].append(child_capture)  # type: ignore[union-attr]
                             except (TimeoutError, asyncio.exceptions.TimeoutError):
                                 self.logger.info(f'Timeout error, took more than {max_capture_time}s. Unable to capture {url}.')
                             except Exception as e:
@@ -880,12 +886,16 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['Download is starting',
                             'Connection closed',
+                            'Connection terminated unexpectedly',
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
+                            'Target page, context or browser has been closed',
                             'Protocol error (Page.bringToFront): Not attached to an active page',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
                             'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
+                            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
                             'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Internal error',
                             'Load cannot follow more than 20 redirections',
                             'Page crashed',
                             'Error receiving data: Connection reset by peer']:
@@ -896,9 +906,15 @@ class Capture():
                 # The browser barfed, let's try again
                 self.logger.info(f'Browser barfed on {url} (retrying): {e.message}')
                 self.should_retry = True
-            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS']:
+            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
+                            'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
                 # No need to retry, the credentials are wrong/missing.
                 pass
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+                self.should_retry = True
+            elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
+                pass
             else:
                 # Unexpected ones
                 self.logger.exception(f'Something went poorly with {url}: {e.message}')
@@ -949,7 +965,7 @@ class Capture():
         try:
             return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
         except Error as e:
-            self.logger.warning(f"Unable to get any screenshot: {e}")
+            self.logger.info(f"Unable to get any screenshot: {e}")
             raise e
 
     async def _safe_wait(self, page: Page, force_max_wait_in_sec: int | None=None) -> None:
@@ -1137,11 +1153,13 @@ class Capture():
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
+                'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
+                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_SSL_UNRECOGNIZED_NAME_ALERT',
                 'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
-                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_TIMED_OUT',
                 'net::ERR_TOO_MANY_REDIRECTS',
+                'SSL_ERROR_UNKNOWN',
         ]:
             return True
         return False

{playwrightcapture-1.24.5 → playwrightcapture-1.24.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.5"
+version = "1.24.6"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -41,8 +41,8 @@ optional = true
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240229"
 pytest = "^8.1.1"
-mypy = "^1.9.0"
-types-dateparser = "^1.1.4.20240331"
+mypy = "^1.10.0"
+types-dateparser = "^1.2.0.20240420"
 types-requests = "^2.31.0.20240406"
 types-pytz = "^2024.1.0.20240417"