PlaywrightCapture 1.24.6__tar.gz → 1.24.8__tar.gz

{playwrightcapture-1.24.6 → playwrightcapture-1.24.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.24.6
+Version: 1.24.8
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -21,11 +21,12 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.3,<4.0.0) ; extra == "recaptcha"
+Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
 Requires-Dist: playwright (>=1.43.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
-Requires-Dist: puremagic (>=1.21,<2.0)
+Requires-Dist: puremagic (>=1.22,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
 Requires-Dist: requests[socks] (>=2.31.0,<3.0.0) ; extra == "recaptcha"

{playwrightcapture-1.24.6 → playwrightcapture-1.24.8}/playwrightcapture/capture.py

@@ -667,7 +667,6 @@ class Capture():
             parsed_url = urlparse(url, allow_fragments=True)
 
             try:
-                # NOTE 2022-12-02: allow 15s less than the general timeout to get a DOM
                 await page.goto(url, wait_until='domcontentloaded', referer=referer if referer else '')
                 page.on("download", handle_download)
             except Error as initial_error:
@@ -832,6 +831,7 @@ class Capture():
                             else:
                                 child_urls = child_urls[:max_captures]
                         self.logger.info(f'Capturing children, {max_captures} URLs')
+                        consecutive_errors = 0
                         for index, url in enumerate(child_urls):
                             self.logger.info(f'Capture child {url} - Timeout: {max_capture_time}s')
                             start_time = time.time()
@@ -846,13 +846,24 @@ class Capture():
                                         rendered_hostname_only=rendered_hostname_only,
                                         max_depth_capture_time=max_capture_time)
                                     to_return['children'].append(child_capture)  # type: ignore[union-attr]
-                            except (TimeoutError, asyncio.exceptions.TimeoutError):
+                            except (TimeoutError, asyncio.exceptions.TimeoutError, asyncio.TimeoutError):
                                 self.logger.info(f'Timeout error, took more than {max_capture_time}s. Unable to capture {url}.')
+                                consecutive_errors += 1
                             except Exception as e:
                                 self.logger.warning(f'Error while capturing child "{url}": {e}. {len(child_urls) - index - 1} more to go.')
+                                consecutive_errors += 1
                             else:
+                                consecutive_errors = 0
                                 runtime = int(time.time() - start_time)
                                 self.logger.info(f'Successfully captured child URL: {url} in {runtime}s. {len(child_urls) - index - 1} to go.')
+
+                            if consecutive_errors >= 5:
+                                # if we have more than 5 consecutive errors, the capture is most probably broken, breaking.
+                                self.logger.warning('Got more than 5 consecutive errors while capturing children, breaking.')
+                                to_return['error'] = "Got more than 5 consecutive errors while capturing children"
+                                self.should_retry = True
+                                break
+
                             try:
                                 await page.go_back()
                             except PlaywrightTimeoutError:
@@ -1278,8 +1289,15 @@ class Capture():
         session.verify = False
         session.headers['user-agent'] = self.user_agent
         if self.proxy and self.proxy.get('server'):
-            proxies = {'http': self.proxy['server'],
-                       'https': self.proxy['server']}
+            proxy_server = self.proxy['server']
+            # Make sure the DNS desolution is done remotely
+            # https://urllib3.readthedocs.io/en/stable/advanced-usage.html#socks-proxies
+            if proxy_server.startswith('socks5://'):
+                proxy_server = proxy_server.replace('socks5://', 'socks5h://')
+            if proxy_server.startswith('socks4://'):
+                proxy_server = proxy_server.replace('socks4://', 'socks4a://')
+
+            proxies = {'http': proxy_server, 'https': proxy_server}
             session.proxies.update(proxies)
         for u in to_fetch:
             try:

{playwrightcapture-1.24.6 → playwrightcapture-1.24.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.24.6"
+version = "1.24.8"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -30,7 +30,8 @@ pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
 setuptools = "^69.5.1"
-puremagic = "^1.21"
+puremagic = "^1.22"
+async-timeout = {version = "^4.0.3", python = "<3.11"}
 
 [tool.poetry.extras]
 recaptcha = ["requests", "pydub", "SpeechRecognition"]
@@ -40,7 +41,7 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 types-beautifulsoup4 = "^4.12.0.20240229"
-pytest = "^8.1.1"
+pytest = "^8.2.0"
 mypy = "^1.10.0"
 types-dateparser = "^1.2.0.20240420"
 types-requests = "^2.31.0.20240406"