PlaywrightCapture 1.25.8__py3-none-any.whl → 1.25.10__py3-none-any.whl

playwrightcapture/capture.py

@@ -18,7 +18,7 @@ from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
 from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping, Generator
-from urllib.parse import urlparse, unquote, urljoin
+from urllib.parse import urlparse, unquote, urljoin, urlsplit, urlunsplit
 from zipfile import ZipFile
 
 import aiohttp
@@ -96,28 +96,40 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
         return msg, kwargs
 
 
+# good test pages:
+# https://bot.incolumitas.com/
+# https://fingerprintjs.github.io/BotD/main/
+
 @dataclass
 class PCStealthConfig(StealthConfig):  # type: ignore[misc]
 
     @property
     def enabled_scripts(self) -> Generator[str, None, None]:
-        self.webdriver = True
-        self.webgl_vendor = True
         self.chrome_app = True
         self.chrome_csi = True
-        self.chrome_load_times = True
         self.chrome_runtime = True
+        self.chrome_load_times = True
+        self.navigator_plugins = True
+        self.hairline = True
         self.iframe_content_window = True
         self.media_codecs = True
+
+        # permissions are handled directly in playwright
+        self.navigator_permissions = False
+        # Platform is correct now
+        self.navigator_platform = False
+        # probably useless, but it will fallback to 4 regardless
         self.navigator_hardware_concurrency = 4
+        # Webgl vendor is correct now
+        self.webgl_vendor = False
+        # Set by the viewport
+        self.outerdimensions = False
+
+        # Not working with Playwright 1.45+
         self.navigator_languages = False  # Causes issue
-        self.navigator_permissions = True
-        self.navigator_platform = True
-        self.navigator_plugins = True
         self.navigator_user_agent = False  # Causes issues
         self.navigator_vendor = False  # Causes issues
-        self.outerdimensions = True
-        self.hairline = True
+
         yield from super().enabled_scripts
 
 
@@ -164,7 +176,7 @@ class Capture():
         self.proxy: ProxySettings = {}
         if proxy:
             if isinstance(proxy, str):
-                self.proxy = {'server': proxy}
+                self.proxy = self.__prepare_proxy_playwright(proxy)
             elif isinstance(proxy, dict):
                 self.proxy = {'server': proxy['server'], 'bypass': proxy.get('bypass', ''),
                               'username': proxy.get('username', ''),
@@ -184,9 +196,22 @@ class Capture():
         self._viewport: ViewportSize | None = None
         self._user_agent: str = ''
         self._timezone_id: str = ''
-        self._locale: str = ''
+        self._locale: str = 'en-US'
         self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = None
 
+    def __prepare_proxy_playwright(self, proxy: str) -> ProxySettings:
+        splitted = urlsplit(proxy)
+        if splitted.username and splitted.password:
+            return {'username': splitted.username, 'password': splitted.password,
+                    'server': urlunsplit((splitted.scheme, f'{splitted.hostname}:{splitted.port}', splitted.path, splitted.query, splitted.fragment))}
+        return {'server': proxy}
+
+    def __prepare_proxy_aiohttp(self, proxy: ProxySettings) -> str:
+        if 'username' in proxy and 'password' in proxy:
+            splitted = urlsplit(proxy['server'])
+            return urlunsplit((splitted.scheme, f'{proxy["username"]}:{proxy["password"]}@{splitted.netloc}', splitted.path, splitted.query, splitted.fragment))
+        return proxy['server']
+
     async def __aenter__(self) -> Capture:
         '''Launch the browser'''
         self._temp_harfile = NamedTemporaryFile(delete=False)
@@ -449,21 +474,22 @@ class Capture():
         # NOTE: Which perms are supported by which browsers varies
         # See https://github.com/microsoft/playwright/issues/16577
         chromium_permissions = [
-            'geolocation',
-            'midi',
-            'midi-sysex',
-            'notifications',
-            'camera',
-            'microphone',
-            'background-sync',
-            'ambient-light-sensor',
             'accelerometer',
-            'gyroscope',
-            'magnetometer',
             'accessibility-events',
+            'ambient-light-sensor',
+            'background-sync',
+            'camera',
             'clipboard-read',
             'clipboard-write',
-            'payment-handler'
+            'geolocation',
+            'gyroscope',
+            'magnetometer',
+            'microphone',
+            'midi-sysex',
+            'midi',
+            'notifications',
+            'payment-handler',
+            'storage-access'
         ]
 
         firefox_permissions = ['geolocation', 'notifications']
@@ -707,7 +733,6 @@ class Capture():
                 await self.__dialog_clickthrough(page)
 
             await stealth_async(page, PCStealthConfig())
-            # await stealth_async(page)
 
             page.set_default_timeout((self._capture_timeout - 2) * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
@@ -786,6 +811,8 @@ class Capture():
                             self.logger.warning(f'Target closed while resolving captcha on {url}: {e}')
                         except Error as e:
                             self.logger.warning(f'Error while resolving captcha on {url}: {e}')
+                        except (TimeoutError, asyncio.TimeoutError) as e:
+                            self.logger.warning(f'[Timeout] Error while resolving captcha on {url}: {e}')
                         except Exception as e:
                             self.logger.exception(f'General error with captcha solving on {url}: {e}')
                     # ======
@@ -892,8 +919,13 @@ class Capture():
                     to_return['html'] = content
 
                 if 'html' in to_return and to_return['html'] is not None and with_favicon:
-                    to_return['potential_favicons'] = await self.get_favicons(page.url, to_return['html'])
-                    got_favicons = True
+                    try:
+                        to_return['potential_favicons'] = await self.get_favicons(page.url, to_return['html'])
+                        got_favicons = True
+                    except (TimeoutError, asyncio.TimeoutError) as e:
+                        self.logger.warning(f'[Timeout] Unable to get favicons: {e}')
+                    except Exception as e:
+                        self.logger.warning(f'Unable to get favicons: {e}')
 
                 to_return['last_redirected_url'] = page.url
                 to_return['png'] = await self._failsafe_get_screenshot(page)
@@ -1196,7 +1228,8 @@ class Capture():
         if self.proxy and self.proxy.get('server'):
             connector = ProxyConnector.from_url(self.proxy['server'])
 
-        async with aiohttp.ClientSession(connector=connector) as session:
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             while True:
                 try:
                     href = await main_frame.get_by_role("link", name="Alternatively, download audio as MP3").get_attribute("href")
@@ -1206,7 +1239,7 @@ class Capture():
                 if not href:
                     self.logger.warning('Unable to find download link for captcha.')
                     return False
-                async with session.get(href, timeout=10, ssl=False) as response:
+                async with session.get(href, ssl=False) as response:
                     response.raise_for_status()
                     mp3_content = await response.read()
                 with NamedTemporaryFile() as mp3_file, NamedTemporaryFile() as wav_file:
@@ -1395,16 +1428,17 @@ class Capture():
         Method inspired by https://github.com/ail-project/ail-framework/blob/master/bin/lib/crawlers.py
         """
         connector = None
-        if self.proxy and self.proxy.get('server'):
+        if self.proxy:
             # NOTE 2024-05-17: switch to async to fetch, the lib uses socks5h by default
-            connector = ProxyConnector.from_url(self.proxy['server'])
+            connector = ProxyConnector.from_url(self.__prepare_proxy_aiohttp(self.proxy))
 
         extracted_favicons = self.__extract_favicons(rendered_content)
         if not extracted_favicons:
             return set()
         to_fetch, to_return = extracted_favicons
         to_fetch.add('/favicon.ico')
-        async with aiohttp.ClientSession(connector=connector) as session:
+        timeout = aiohttp.ClientTimeout(total=10)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             session.headers['user-agent'] = self.user_agent
             for u in to_fetch:
                 try:
@@ -1414,7 +1448,7 @@ class Capture():
                     if url_to_fetch in self._requests:
                         favicon = self._requests[url_to_fetch]
                     if not favicon:
-                        async with session.get(url_to_fetch, timeout=5, ssl=False) as favicon_response:
+                        async with session.get(url_to_fetch, ssl=False) as favicon_response:
                             favicon_response.raise_for_status()
                             favicon = await favicon_response.read()
                     if favicon:

{playwrightcapture-1.25.8.dist-info → playwrightcapture-1.25.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.8
+Version: 1.25.10
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -21,17 +21,17 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.10.4,<4.0.0) ; extra == "recaptcha"
-Requires-Dist: aiohttp-socks (>=0.8.4,<0.9.0)
-Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
+Requires-Dist: aiohttp-socks (>=0.9,<0.10)
+Requires-Dist: aiohttp[speedups] (>=3.10.1,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.45.0,<2.0.0)
+Requires-Dist: playwright (>=1.45.1,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.26,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; python_version < "3.9"
-Requires-Dist: setuptools (>=71.1.0,<72.0.0)
+Requires-Dist: setuptools (>=72.1.0,<73.0.0)
 Requires-Dist: tzdata (>=2024.1,<2025.0)
 Requires-Dist: w3lib (>=2.2.1,<3.0.0)
 Project-URL: Repository, https://github.com/Lookyloo/PlaywrightCapture

playwrightcapture-1.25.10.dist-info/RECORD

@@ -0,0 +1,9 @@
+playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
+playwrightcapture/capture.py,sha256=AayQPmsjzcN2TIBi4uuwDmV0kqIVLDdAfEaNPUg8TXY,72606
+playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
+playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
+playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+playwrightcapture-1.25.10.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.25.10.dist-info/METADATA,sha256=3nVVIzM7kcR62_XYA1mRUxfoflPZwfLS5B8KDPy3Hks,3172
+playwrightcapture-1.25.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+playwrightcapture-1.25.10.dist-info/RECORD,,

playwrightcapture-1.25.8.dist-info/RECORD

@@ -1,9 +0,0 @@
-playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=DzAx7xwF5IbYfIOt-4gzppc9Qzc1jadk93zN8PtVJZQ,70846
-playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
-playwrightcapture/helpers.py,sha256=SXQLEuxMs8-bcWykMiUVosHzzxBKuS-QC0gBV3OmKmo,1764
-playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-playwrightcapture-1.25.8.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.25.8.dist-info/METADATA,sha256=508SCjk5btHdQCfQrrivoGA7s-X-LY1TAv6Qp_HyeOE,3173
-playwrightcapture-1.25.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-playwrightcapture-1.25.8.dist-info/RECORD,,