PlaywrightCapture 1.21.9__tar.gz → 1.21.10__tar.gz

{playwrightcapture-1.21.9 → playwrightcapture-1.21.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.21.9
+Version: 1.21.10
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause

{playwrightcapture-1.21.9 → playwrightcapture-1.21.10}/playwrightcapture/capture.py

@@ -20,7 +20,7 @@ import requests
 
 from bs4 import BeautifulSoup
 from charset_normalizer import from_bytes
-from playwright.async_api import async_playwright, Frame, Error, Page
+from playwright.async_api import async_playwright, Frame, Error, Page, Download
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
@@ -429,7 +429,28 @@ class Capture():
                            rendered_hostname_only: bool=True,
                            with_favicon: bool=False
                            ) -> CaptureResponse:
+
         to_return: CaptureResponse = {}
+
+        self.wait_for_download = False
+
+        async def handle_download(download: Download) -> None:
+            # This method is called when a download event is triggered from JS in a page that also renders
+            try:
+                self.wait_for_download = True
+                self.logger.info('Got a download triggered from JS.')
+                tmp_f = NamedTemporaryFile(delete=False)
+                await download.save_as(tmp_f.name)
+                to_return["downloaded_filename"] = download.suggested_filename
+                with open(tmp_f.name, "rb") as f:
+                    to_return["downloaded_file"] = f.read()
+                os.unlink(tmp_f.name)
+                self.logger.info('Done with download.')
+            except Exception as e:
+                self.logger.warning(f'Unable to finish download triggered from JS: {e}')
+            finally:
+                self.wait_for_download = False
+
         if page is not None:
             capturing_sub = True
         else:
@@ -443,6 +464,7 @@ class Capture():
             try:
                 # NOTE 2022-12-02: allow 15s less than the general timeout to get a DOM
                 await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else '')
+                page.on("download", handle_download)
             except Error as initial_error:
                 self._update_exceptions(initial_error)
                 # So this one is really annoying: chromium raises a net::ERR_ABORTED when it hits a download
@@ -546,6 +568,10 @@ class Capture():
                 if 'html' in to_return and to_return['html'] is not None and with_favicon:
                     to_return['potential_favicons'] = self.get_favicons(page.url, to_return['html'])
 
+                if self.wait_for_download:
+                    self.logger.info('Waiting for download to finish...')
+                    await self._safe_wait(page)
+
                 if depth > 0 and to_return.get('html') and to_return['html']:
                     if child_urls := self._get_links_from_rendered_page(page.url, to_return['html'], rendered_hostname_only):
                         to_return['children'] = []

{playwrightcapture-1.21.9 → playwrightcapture-1.21.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.21.9"
+version = "1.21.10"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -42,8 +42,8 @@ types-beautifulsoup4 = "^4.12.0.6"
 pytest = "^7.4.2"
 mypy = "^1.5.1"
 types-dateparser = "^1.1.4.10"
-types-requests = "^2.31.0.2"
-types-pytz = "^2023.3.1.0"
+types-requests = "^2.31.0.6"
+types-pytz = "^2023.3.1.1"
 
 
 [build-system]