PlaywrightCapture 1.26.3__tar.gz → 1.27.1__tar.gz

{playwrightcapture-1.26.3 → playwrightcapture-1.27.1}/PKG-INFO

@@ -1,18 +1,17 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.26.3
+Version: 1.27.1
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -21,11 +20,9 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
-Requires-Dist: SpeechRecognition (<3.11) ; (python_version < "3.9") and (extra == "recaptcha")
-Requires-Dist: SpeechRecognition (>=3.11) ; (python_version >= "3.9") and (extra == "recaptcha")
+Requires-Dist: SpeechRecognition (>=3.11.0) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.9,<0.10)
-Requires-Dist: aiohttp[speedups] (<3.11) ; python_version < "3.9"
-Requires-Dist: aiohttp[speedups] (>=3.10.10,<4.0.0) ; python_version >= "3.9"
+Requires-Dist: aiohttp[speedups] (>=3.10.10,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
@@ -33,8 +30,7 @@ Requires-Dist: playwright (>=1.48.0,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.28,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
-Requires-Dist: pytz (>=2024.2,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=75.2.0,<76.0.0)
+Requires-Dist: setuptools (>=75.3.0,<76.0.0)
 Requires-Dist: tzdata (>=2024.2,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

{playwrightcapture-1.26.3 → playwrightcapture-1.27.1}/playwrightcapture/capture.py

@@ -17,7 +17,8 @@ from dataclasses import dataclass
 from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
-from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping, Generator
+from typing import Any, TypedDict, Literal, TYPE_CHECKING
+from collections.abc import MutableMapping, Generator
 from urllib.parse import urlparse, unquote, urljoin, urlsplit, urlunsplit
 from zipfile import ZipFile
 
@@ -38,11 +39,8 @@ from w3lib.url import canonicalize_url, safe_url_string
 
 from .exceptions import UnknownPlaywrightBrowser, UnknownPlaywrightDevice, InvalidPlaywrightParameter
 
-if sys.version_info < (3, 9):
-    from pytz import all_timezones_set
-else:
-    from zoneinfo import available_timezones
-    all_timezones_set = available_timezones()
+from zoneinfo import available_timezones
+all_timezones_set = available_timezones()
 
 if sys.version_info < (3, 11):
     from async_timeout import timeout
@@ -104,7 +102,7 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
 class PCStealthConfig(StealthConfig):  # type: ignore[misc]
 
     @property
-    def enabled_scripts(self) -> Generator[str, None, None]:
+    def enabled_scripts(self) -> Generator[str]:
         self.chrome_app = True
         self.chrome_csi = True
         self.chrome_runtime = True
@@ -734,7 +732,7 @@ class Capture():
                             await frame.get_by_label(label).click(timeout=2000)
                             break
                 except (TimeoutError, asyncio.TimeoutError) as e:
-                    self.logger.warning(f'Frame consent timeout: {e}')
+                    self.logger.warning(f'Consent timeout (label {label}) : {e}')
 
                 try:
                     async with timeout(5):
@@ -744,9 +742,9 @@ class Capture():
                             await frame.get_by_role("button", name=label).click(timeout=2000)
                             break
                 except (TimeoutError, asyncio.TimeoutError) as e:
-                    self.logger.warning(f'Frame consent timeout: {e}')
+                    self.logger.warning(f'Frame consent timeout (button {label}): {e}')
         except Exception as e:
-            self.logger.info(f'Issue with frame consent: {e}')
+            self.logger.info(f'Issue with consent validation: {e}')
         return got_button
 
     async def _move_time_forward(self, page: Page, time: int) -> None:
@@ -827,12 +825,18 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
-                await page.clock.install()
             except Error as e:
-                self.logger.warning(f'The context is in a broken state: {e}')
+                self.logger.warning(f'Unable to create new page, the context is in a broken state: {e}')
                 self.should_retry = True
                 return to_return
 
+            try:
+                await page.clock.install()
+                clock_set = True
+            except Error as e:
+                self.logger.warning(f'Unable to install the clock: {e}')
+                clock_set = False
+
             if allow_tracking:
                 # Add authorization clickthroughs
                 await self.__dialog_didomi_clickthrough(page)
@@ -900,8 +904,7 @@ class Capture():
                     await page.bring_to_front()
                     self.logger.debug('Page moved to front.')
                 except Error as e:
-                    self.logger.warning('Page in a broken state.')
-                    raise e
+                    self.logger.warning(f'Unable to bring the page to the front: {e}.')
 
                 # page instrumentation
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after document loaded
@@ -971,7 +974,8 @@ class Capture():
                             self.logger.debug('Got button on main frame')
                             await self._wait_for_random_timeout(page, 10)  # Wait 10 sec after click
 
-                    await self._move_time_forward(page, 10)
+                    if clock_set:
+                        await self._move_time_forward(page, 10)
 
                     if parsed_url.fragment:
                         # We got a fragment, make sure we go to it and scroll only a little bit.
@@ -1039,8 +1043,9 @@ class Capture():
                                 z.writestr(f'{i}_{filename}', file_content)
                         to_return["downloaded_file"] = mem_zip.getvalue()
 
-                # fast forward ~30s
-                await self._move_time_forward(page, 30)
+                if clock_set:
+                    # fast forward ~30s
+                    await self._move_time_forward(page, 30)
 
                 self.logger.debug('Done with instrumentation, waiting for network idle.')
                 await self._wait_for_random_timeout(page, 5)  # Wait 5 sec after instrumentation
@@ -1462,6 +1467,7 @@ class Capture():
                 'net::ERR_CONNECTION_TIMED_OUT',
                 'net::ERR_HTTP_RESPONSE_CODE_FAILURE',
                 'net::ERR_HTTP2_PROTOCOL_ERROR',
+                'net::ERR_INVALID_HTTP_RESPONSE',
                 'net::ERR_INVALID_REDIRECT',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_NETWORK_ACCESS_DENIED',

{playwrightcapture-1.26.3 → playwrightcapture-1.27.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.26.3"
+version = "1.27.1"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -18,26 +18,19 @@ classifiers=[
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 playwright = "^1.48.0"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.2.1"
 pydub = {version = "^0.25.1", optional = true}
-SpeechRecognition = [
-    {version = "<3.11", python = "<3.9", optional = true},
-    {version = ">=3.11", python = ">=3.9", optional = true}
-]
-pytz = {"version" = "^2024.2", python = "<3.9"}
+SpeechRecognition =  {version = ">=3.11.0", optional = true}
 tzdata = "^2024.2"
 playwright-stealth = "^1.0.6"
-setuptools = "^75.2.0"
+setuptools = "^75.3.0"
 puremagic = "^1.28"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
-aiohttp = [
-    {extras = ["speedups"], version = "<3.11", python = "<3.9"},
-    {extras = ["speedups"], version = "^3.10.10", python = ">=3.9"}
-]
+aiohttp = {version = "^3.10.10", extras = ["speedups"]}
 aiohttp-socks = "^0.9"
 
 [tool.poetry.extras]