PlaywrightCapture 1.24.4__tar.gz → 1.24.6__tar.gz

{playwrightcapture-1.24.4 → playwrightcapture-1.24.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.4
+Version: 1.24.6
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause

{playwrightcapture-1.24.4 → playwrightcapture-1.24.6}/playwrightcapture/capture.py

@@ -14,8 +14,9 @@ import time
 
 from base64 import b64decode
 from io import BytesIO
+from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
-from typing import Any, TypedDict, Literal, TYPE_CHECKING
+from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping
 from urllib.parse import urlparse, unquote, urljoin
 from zipfile import ZipFile
 
@@ -42,6 +43,11 @@ else:
     from zoneinfo import available_timezones
     all_timezones_set = available_timezones()
 
+if sys.version_info < (3, 11):
+    from async_timeout import timeout
+else:
+    from asyncio import timeout
+
 if TYPE_CHECKING:
     from playwright._impl._api_structures import (SetCookieParam, Geolocation,
                                                   HttpCredentials, Headers,
@@ -79,6 +85,16 @@ class CaptureResponse(TypedDict, total=False):
     potential_favicons: set[bytes] | None
 
 
+class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
+    """
+    Prepend log entry with the UUID of the capture
+    """
+    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
+        if self.extra:
+            return '[{}] {}'.format(self.extra['uuid'], msg), kwargs
+        return msg, kwargs
+
+
 class Capture():
 
     _browsers: list[BROWSER] = ['chromium', 'firefox', 'webkit']
@@ -90,7 +106,8 @@ class Capture():
 
     def __init__(self, browser: BROWSER | None=None, device_name: str | None=None,
                  proxy: str | dict[str, str] | None=None,
-                 general_timeout_in_sec: int | None = None, loglevel: str | int='INFO'):
+                 general_timeout_in_sec: int | None = None, loglevel: str | int='INFO',
+                 uuid: str | None=None):
         """Captures a page with Playwright.
 
         :param browser: The browser to use for the capture.
@@ -98,9 +115,15 @@ class Capture():
         :param proxy: The external proxy to use for the capture.
         :param general_timeout_in_sec: The general timeout for the capture, including children.
         :param loglevel: Python loglevel
+        :param uuid: The UUID of the capture.
         """
-        self.logger = logging.getLogger('playwrightcapture')
-        self.logger.setLevel(loglevel)
+        master_logger = logging.getLogger('playwrightcapture')
+        master_logger.setLevel(loglevel)
+        self.logger: Logger | PlaywrightCaptureLogAdapter
+        if uuid is not None:
+            self.logger = PlaywrightCaptureLogAdapter(master_logger, {'uuid': uuid})
+        else:
+            self.logger = master_logger
         self.browser_name: BROWSER = browser if browser else 'chromium'
 
         if general_timeout_in_sec is None:
@@ -451,36 +474,33 @@ class Capture():
             if await page.locator("#didomi-notice-agree-button").is_visible():
                 await page.locator("#didomi-notice-agree-button").click(timeout=2000)
 
-        await page.add_locator_handler(page.locator(".didomi-popup-view"), handler)
+        await page.add_locator_handler(page.locator(".didomi-popup-view").last, handler)
         self.logger.info('Didomi handler added')
 
     async def __dialog_onetrust_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## OT Dialog found, clicking through.')
             if await page.locator("#onetrust-accept-btn-handler").is_visible():
                 await page.locator("#onetrust-accept-btn-handler").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#onetrust-banner-sdk'),
+            page.locator('#onetrust-banner-sdk').last,
             handler
         )
         self.logger.info('OT handler added')
 
     async def __dialog_hubspot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## HS Dialog found, clicking through.')
             if await page.locator("#hs-eu-confirmation-button").is_visible():
                 await page.locator("#hs-eu-confirmation-button").click(timeout=2000)
 
         await page.add_locator_handler(
-            page.locator('#hs-eu-cookie-confirmation'),
+            page.locator('#hs-eu-cookie-confirmation').last,
             handler
         )
         self.logger.info('HS handler added')
 
     async def __dialog_cookiebot_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Cookiebot Dialog found, clicking through.')
             if await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").is_visible():
                 await page.locator("#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll").click(timeout=2000)
 
@@ -501,7 +521,7 @@ class Capture():
                 self.logger.info('Consent window found, but no button to click through.')
 
         await page.add_locator_handler(
-            page.get_by_role("alertdialog"),
+            page.get_by_role("alertdialog").last,
             handler
         )
         self.logger.info('alert dialog handler added')
@@ -527,7 +547,6 @@ class Capture():
 
     async def __dialog_complianz_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Complianz found, clicking through.')
             if await page.locator('.cmplz-show').locator("button.cmplz-accept").is_visible():
                 await page.locator('.cmplz-show').locator("button.cmplz-accept").click(timeout=2000)
 
@@ -539,7 +558,6 @@ class Capture():
 
     async def __dialog_yahoo_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## Yahoo found, clicking through.')
             if await page.locator('.con-wizard').locator("button.accept-all").is_visible():
                 await page.locator('.con-wizard').locator("button.accept-all").click(timeout=2000)
 
@@ -551,7 +569,6 @@ class Capture():
 
     async def __dialog_ppms_clickthrough(self, page: Page) -> None:
         async def handler() -> None:
-            self.logger.info('######## piwik found, clicking through.')
             if await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").is_visible():
                 await page.locator('.ppms_cm_popup_overlay').locator("button.ppms_cm_agree-to-all").click(timeout=2000)
 
@@ -592,7 +609,11 @@ class Capture():
                     multiple_downloads.append((filename, file_content))
                     self.logger.info('Done with download.')
             except Exception as e:
-                self.logger.warning(f'Unable to finish download triggered from JS: {e}')
+                if download.page.is_closed():
+                    # Page is closed, skip logging.
+                    pass
+                else:
+                    self.logger.warning(f'Unable to finish download triggered from JS: {e}')
             finally:
                 self.wait_for_download -= 1
 
@@ -682,9 +703,6 @@ class Capture():
                         except Exception:
                             raise e
                 else:
-                    if not self._exception_is_network_error(initial_error):
-                        # TODO: Do something?
-                        self.logger.warning(f'Unexpected error: {initial_error}')
                     raise initial_error
             else:
                 await page.bring_to_front()
@@ -817,14 +835,17 @@ class Capture():
                         for index, url in enumerate(child_urls):
                             self.logger.info(f'Capture child {url} - Timeout: {max_capture_time}s')
                             start_time = time.time()
+                            if page.is_closed():
+                                self.logger.info('Page is closed, unable to capture children.')
+                                break
                             try:
-                                child_capture = await asyncio.wait_for(
-                                    self.capture_page(url=url, referer=page.url,
-                                                      page=page, depth=depth,
-                                                      rendered_hostname_only=rendered_hostname_only,
-                                                      max_depth_capture_time=max_capture_time),
-                                    timeout=max_capture_time + 1)  # just adding a bit of padding so playwright has the chance to raise the exception first
-                                to_return['children'].append(child_capture)  # type: ignore[union-attr]
+                                async with timeout(max_capture_time + 1):  # just adding a bit of padding so playwright has the chance to raise the exception first
+                                    child_capture = await self.capture_page(
+                                        url=url, referer=page.url,
+                                        page=page, depth=depth,
+                                        rendered_hostname_only=rendered_hostname_only,
+                                        max_depth_capture_time=max_capture_time)
+                                    to_return['children'].append(child_capture)  # type: ignore[union-attr]
                             except (TimeoutError, asyncio.exceptions.TimeoutError):
                                 self.logger.info(f'Timeout error, took more than {max_capture_time}s. Unable to capture {url}.')
                             except Exception as e:
@@ -865,11 +886,19 @@ class Capture():
                 self.should_retry = True
             elif e.name in ['Download is starting',
                             'Connection closed',
+                            'Connection terminated unexpectedly',
                             'Navigation interrupted by another one',
                             'Navigation failed because page was closed!',
+                            'Target page, context or browser has been closed',
                             'Protocol error (Page.bringToFront): Not attached to an active page',
                             'Peer failed to perform TLS handshake: The TLS connection was non-properly terminated.',
-                            'Load cannot follow more than 20 redirections']:
+                            'Peer failed to perform TLS handshake: Error sending data: Connection reset by peer',
+                            'Peer failed to perform TLS handshake: Error receiving data: Connection reset by peer',
+                            'Peer sent fatal TLS alert: The server name sent was not recognized',
+                            'Peer sent fatal TLS alert: Internal error',
+                            'Load cannot follow more than 20 redirections',
+                            'Page crashed',
+                            'Error receiving data: Connection reset by peer']:
                 # Other errors, let's give it another shot
                 self.logger.info(f'Issue with {url} (retrying): {e.message}')
                 self.should_retry = True
@@ -877,9 +906,15 @@ class Capture():
                 # The browser barfed, let's try again
                 self.logger.info(f'Browser barfed on {url} (retrying): {e.message}')
                 self.should_retry = True
-            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS']:
+            elif e.name in ['net::ERR_INVALID_AUTH_CREDENTIALS',
+                            'net::ERR_BAD_SSL_CLIENT_AUTH_CERT',
+                            'net::ERR_UNEXPECTED_PROXY_AUTH']:
                 # No need to retry, the credentials are wrong/missing.
                 pass
+            elif e.name and any([msg in e.name for msg in ['is interrupted by another navigation to']]):
+                self.should_retry = True
+            elif e.name and any([msg in e.name for msg in ['Error resolving', 'Could not connect to']]):
+                pass
             else:
                 # Unexpected ones
                 self.logger.exception(f'Something went poorly with {url}: {e.message}')
@@ -930,7 +965,7 @@ class Capture():
         try:
             return await page.screenshot(scale="css", animations='disabled', caret='initial', timeout=5000)
         except Error as e:
-            self.logger.warning(f"Unable to get any screenshot: {e}")
+            self.logger.info(f"Unable to get any screenshot: {e}")
             raise e
 
     async def _safe_wait(self, page: Page, force_max_wait_in_sec: int | None=None) -> None:
@@ -1118,11 +1153,13 @@ class Capture():
                 'net::ERR_INVALID_RESPONSE',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',
+                'net::ERR_SSL_KEY_USAGE_INCOMPATIBLE',
+                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_SSL_UNRECOGNIZED_NAME_ALERT',
                 'net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH',
-                'net::ERR_SSL_PROTOCOL_ERROR',
                 'net::ERR_TIMED_OUT',
                 'net::ERR_TOO_MANY_REDIRECTS',
+                'SSL_ERROR_UNKNOWN',
         ]:
             return True
         return False

{playwrightcapture-1.24.4 → playwrightcapture-1.24.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.4"
+version = "1.24.6"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -41,10 +41,10 @@ optional = true
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240229"
 pytest = "^8.1.1"
-mypy = "^1.9.0"
-types-dateparser = "^1.1.4.20240331"
+mypy = "^1.10.0"
+types-dateparser = "^1.2.0.20240420"
 types-requests = "^2.31.0.20240406"
-types-pytz = "^2024.1.0.20240203"
+types-pytz = "^2024.1.0.20240417"
 
 
 [build-system]