PlaywrightCapture 1.25.4__tar.gz → 1.25.6__tar.gz

{playwrightcapture-1.25.4 → playwrightcapture-1.25.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.4
+Version: 1.25.6
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -31,7 +31,7 @@ Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.26,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=70.3.0,<71.0.0)
+Requires-Dist: setuptools (>=71.0.3,<72.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.25.4 → playwrightcapture-1.25.6}/playwrightcapture/capture.py

@@ -165,10 +165,15 @@ class Capture():
         if proxy:
             if isinstance(proxy, str):
                 self.proxy = {'server': proxy}
-            else:
+            elif isinstance(proxy, dict):
                 self.proxy = {'server': proxy['server'], 'bypass': proxy.get('bypass', ''),
                               'username': proxy.get('username', ''),
                               'password': proxy.get('password', '')}
+            elif isinstance(proxy, int):
+                # This is clearly a mistake, just ignoring it
+                self.logger.warning('Proxy is an integer, this is a mistake, ignoring it.')
+            else:
+                raise InvalidPlaywrightParameter(f'Invalid proxy parameter: "{proxy}" ({type(proxy)})')
 
         self.should_retry: bool = False
         self.__network_not_idle: int = 2  # makes sure we do not wait for network idle the max amount of time the capture is allowed to take
@@ -334,19 +339,10 @@ class Capture():
         return self._headers
 
     @headers.setter
-    def headers(self, headers: str | dict[str, str] | None) -> None:
+    def headers(self, headers: dict[str, str] | None) -> None:
         if not headers:
             return
-        if isinstance(headers, str):
-            new_headers: dict[str, str] = {}
-            for header_line in headers.splitlines():
-                if header_line and ':' in header_line:
-                    splitted = header_line.split(':', 1)
-                    if splitted and len(splitted) == 2:
-                        header, h_value = splitted
-                        if header.strip() and h_value.strip():
-                            new_headers[header.strip()] = h_value.strip()
-        elif isinstance(headers, dict):
+        if isinstance(headers, dict):
             # Check if they are valid
             new_headers = {name.strip(): value.strip() for name, value in headers.items() if isinstance(name, str) and isinstance(value, str) and name.strip() and value.strip()}
             if new_headers != headers:
@@ -812,7 +808,7 @@ class Capture():
                         try:
                             await page.locator(f'id={fragment}').first.scroll_into_view_if_needed(timeout=3000)
                             await self._wait_for_random_timeout(page, 2)
-                            async with timeout(3):
+                            async with timeout(5):
                                 await page.mouse.wheel(delta_y=random.uniform(150, 300), delta_x=0)
                             self.logger.debug('Jumped to fragment.')
                         except PlaywrightTimeoutError as e:
@@ -821,20 +817,24 @@ class Capture():
                             self.logger.warning(f'Target closed, unable to go to fragment "{fragment}": {e}')
                         except Error as e:
                             self.logger.exception(f'Unable to go to fragment "{fragment}": {e}')
-                        except TimeoutError:
+                        except (asyncio.TimeoutError, TimeoutError):
                             self.logger.debug('Unable to scroll due to timeout')
+                        except (asyncio.CancelledError):
+                            self.logger.debug('Unable to scroll due to timeout, call canceled')
                     else:
                         # scroll more
                         try:
                             # NOTE using page.mouse.wheel causes the instrumentation to fail, sometimes.
                             #   2024-07-08: Also, it sometimes get stuck.
-                            async with timeout(3):
+                            async with timeout(5):
                                 await page.mouse.wheel(delta_y=random.uniform(1500, 3000), delta_x=0)
                             self.logger.debug('Scrolled down.')
                         except Error as e:
                             self.logger.debug(f'Unable to scroll: {e}')
-                        except TimeoutError:
+                        except (TimeoutError, asyncio.TimeoutError):
                             self.logger.debug('Unable to scroll due to timeout')
+                        except (asyncio.CancelledError):
+                            self.logger.debug('Unable to scroll due to timeout, call canceled')
 
                     await self._wait_for_random_timeout(page, 3)
                     self.logger.debug('Keep going after moving on page.')
@@ -870,7 +870,7 @@ class Capture():
                 #    async with timeout(3):
                 #        await page.clock.run_for("47")
                 #        self.logger.debug('Moved time forward.')
-                # except TimeoutError:
+                # except (TimeoutError, asyncio.TimeoutError):
                 #    self.logger.warning('Unable to move time forward.')
 
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
@@ -919,7 +919,7 @@ class Capture():
                                         rendered_hostname_only=rendered_hostname_only,
                                         max_depth_capture_time=max_capture_time)
                                     to_return['children'].append(child_capture)  # type: ignore[union-attr]
-                            except (TimeoutError, asyncio.exceptions.TimeoutError, asyncio.TimeoutError):
+                            except (TimeoutError, asyncio.TimeoutError):
                                 self.logger.info(f'Timeout error, took more than {max_capture_time}s. Unable to capture {url}.')
                                 consecutive_errors += 1
                             except Exception as e:
@@ -947,6 +947,9 @@ class Capture():
         except PlaywrightTimeoutError as e:
             to_return['error'] = f"The capture took too long - {e.message}"
             self.should_retry = True
+        except (asyncio.TimeoutError, TimeoutError):
+            to_return['error'] = "Something in the capture took too long"
+            self.should_retry = True
         except TargetClosedError as e:
             to_return['error'] = f"The target was closed - {e}"
             self.should_retry = True
@@ -1084,7 +1087,7 @@ class Capture():
             try:
                 async with timeout(30):
                     return await page.content()
-            except (Error, TimeoutError):
+            except (Error, TimeoutError, asyncio.TimeoutError):
                 self.logger.debug('Unable to get page content, trying again.')
                 tries -= 1
                 await self._wait_for_random_timeout(page, 1)

{playwrightcapture-1.25.4 → playwrightcapture-1.25.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.25.4"
+version = "1.25.6"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -28,7 +28,7 @@ SpeechRecognition = {version = "^3.10.4", optional = true}
 pytz = {"version" = "^2024.1", python = "<3.9"}
 tzdata = "^2024.1"
 playwright-stealth = "^1.0.6"
-setuptools = "^70.3.0"
+setuptools = "^71.0.3"
 puremagic = "^1.26"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 aiohttp = {extras = ["speedups"], version = "^3.9.5"}