PlaywrightCapture 1.31.0__py3-none-any.whl → 1.31.2__py3-none-any.whl

playwrightcapture/capture.py

@@ -13,12 +13,11 @@ import sys
 import time
 
 from base64 import b64decode
-from dataclasses import dataclass
 from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
 from typing import Any, Literal, TYPE_CHECKING
-from collections.abc import MutableMapping, Iterator
+from collections.abc import MutableMapping
 from urllib.parse import urlparse, unquote, urljoin, urlsplit, urlunsplit
 from zipfile import ZipFile
 
@@ -31,7 +30,7 @@ from charset_normalizer import from_bytes
 from playwright._impl._errors import TargetClosedError
 from playwright.async_api import async_playwright, Frame, Error, Page, Download, Request
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-from playwright_stealth import stealth_async, StealthConfig  # type: ignore[attr-defined]
+from playwright_stealth import Stealth, ALL_EVASIONS_DISABLED_KWARGS  # type: ignore[attr-defined]
 from puremagic import PureError, from_string
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
@@ -101,42 +100,11 @@ class PlaywrightCaptureLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
 
 
 # good test pages:
-# https://bot.incolumitas.com/
+# https://kaliiiiiiiiii.github.io/brotector/?crash=false
+# https://www.browserscan.net/bot-detection
+# https://fingerprint.com/products/bot-detection/
 # https://fingerprintjs.github.io/BotD/main/
 
-@dataclass
-class PCStealthConfig(StealthConfig):
-
-    @property
-    def enabled_scripts(self) -> Iterator[str]:
-        self.chrome_app = True
-        self.chrome_csi = True
-        self.chrome_runtime = True
-        self.chrome_load_times = True
-        self.navigator_plugins = True
-        self.hairline = True
-        self.iframe_content_window = True
-        self.media_codecs = True
-
-        # permissions are handled directly in playwright
-        self.navigator_permissions = False
-        # Platform is correct now
-        self.navigator_platform = False
-        # probably useless, but it will fallback to 4 regardless
-        self.navigator_hardware_concurrency = 4
-        # Webgl vendor is correct now
-        self.webgl_vendor = False
-        # Set by the viewport
-        self.outerdimensions = False
-
-        # Not working with Playwright 1.45+
-        self.navigator_languages = False  # Causes issue
-        self.navigator_user_agent = False  # Causes issues
-        self.navigator_vendor = False  # Causes issues
-
-        yield from super().enabled_scripts
-
-
 class Capture():
 
     _browsers: list[BROWSER] = ['chromium', 'firefox', 'webkit']
@@ -492,6 +460,52 @@ class Capture():
         )
         self.context.set_default_timeout(self._capture_timeout * 1000)
 
+        # very quick and dirty get a platform from the UA so it's not always Win32
+        # This this is deprecated and not very important.
+        # Ref: https://developer.mozilla.org/en-US/docs/Web/API/Navigator/platform
+        if any(x in ua.lower() for x in ['windows', 'win32', 'win64']):
+            _platform = 'Win32'
+        elif any(x in ua.lower() for x in ['macintosh', 'mac os x', 'macos']):
+            _platform = 'MacIntel'
+        elif any(x in ua.lower() for x in ['linux', 'ubuntu']):
+            _platform = 'Linux x86_64'
+        else:
+            _platform = 'Win32'
+
+        # Enable stealth mode
+        stealth = Stealth(
+            **{**ALL_EVASIONS_DISABLED_KWARGS,  # type: ignore[arg-type]
+               'chrome_app': True,
+               'chrome_csi': True,
+               'chrome_load_times': True,
+               'chrome_runtime': True,
+               'hairline': True,
+               'iframe_content_window': True,
+               'media_codecs': True,
+               # 'navigator_hardware_concurrency': False,
+               # 'navigator_languages': False,  # handled by playwright directly
+               # 'navigator_permissions': False,  # handled by playwright directly
+               'navigator_platform': True,
+               'navigator_plugins': True,
+               # 'navigator_user_agent': True,  # Set by playwright
+               # 'navigator_vendor': False,  # It's set correctly by playwright
+               'navigator_webdriver': True,
+               # 'sec_ch_ua': True,
+               # 'webgl_vendor': False,  # It's set correctly by playwright
+
+               # ## Overwrite the default values
+               'navigator_languages_override': None,
+               'navigator_platform_override': _platform,
+               # 'navigator_user_agent_override': ua,  # Already Set in playwright context
+               # 'navigator_vendor_override': None,
+               # 'sec_ch_ua_override': Stealth._get_greased_chrome_sec_ua_ch(ua),
+               # 'webgl_renderer_override': None,
+               # 'webgl_vendor_override': None,
+               })
+
+        # stealth.hook_playwright_context(self.playwright)
+        await stealth.apply_stealth_async(self.context)
+
         if self.cookies:
             try:
                 await self.context.add_cookies(self.cookies)
@@ -1002,6 +1016,8 @@ class Capture():
             capturing_sub = False
             try:
                 page = await self.context.new_page()
+                # client = await page.context.new_cdp_session(page)
+                # await client.detach()
             except Error as e:
                 self.logger.warning(f'Unable to create new page, the context is in a broken state: {e}')
                 self.should_retry = True
@@ -1021,8 +1037,6 @@ class Capture():
                 await self.__dialog_clickthrough(page)
                 await self.__dialog_tarteaucitron_clickthrough(page)
 
-            await stealth_async(page, PCStealthConfig())
-
             page.set_default_timeout((self._capture_timeout - 2) * 1000)
             # trigger a callback on each request to store it in a dict indexed by URL to get it back from the favicon fetcher
             page.on("requestfinished", store_request)
@@ -1547,6 +1561,7 @@ class Capture():
                 'net::ERR_INVALID_REDIRECT',
                 'net::ERR_NAME_NOT_RESOLVED',
                 'net::ERR_NETWORK_ACCESS_DENIED',
+                'net::ERR_PROXY_CONNECTION_FAILED',
                 'net::ERR_QUIC_PROTOCOL_ERROR',
                 'net::ERR_SOCKET_NOT_CONNECTED',
                 'net::ERR_SOCKS_CONNECTION_FAILED',

{playwrightcapture-1.31.0.dist-info → playwrightcapture-1.31.2.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: PlaywrightCapture
-Version: 1.31.0
+Version: 1.31.2
 Summary: A simple library to capture websites using playwright
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<4.0
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
@@ -20,13 +20,13 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.14.3) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.10.1)
-Requires-Dist: aiohttp[speedups] (>=3.12.12)
+Requires-Dist: aiohttp[speedups] (>=3.12.13)
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
-Requires-Dist: dateparser (>=1.2.1)
+Requires-Dist: dateparser (>=1.2.2)
 Requires-Dist: dnspython (>=2.7.0,<3.0.0)
-Requires-Dist: playwright (>=1.52.0)
-Requires-Dist: playwright-stealth (>=1.0.6)
+Requires-Dist: playwright (>=1.53.0)
+Requires-Dist: playwright-stealth (>=2)
 Requires-Dist: puremagic (>=1.29)
 Requires-Dist: pydub (>=0.25.1) ; (python_version < "3.10") and (extra == "recaptcha")
 Requires-Dist: pydub-ng (>=0.2.0) ; (python_version >= "3.10") and (extra == "recaptcha")

{playwrightcapture-1.31.0.dist-info → playwrightcapture-1.31.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
 playwrightcapture/__init__.py,sha256=F90Y8wYS13tDjgsfjuFrCfmzQfdnH44G-ovuilJfLEE,511
-playwrightcapture/capture.py,sha256=6taJ5k4Aolpp4Z_YRbbgt9AksWnQPoOeb3GR2x1TmCc,85259
+playwrightcapture/capture.py,sha256=lXcBWWEWRmQem6Rs_yj51PB74JFlRL4k1kggFCMxFTA,86586
 playwrightcapture/exceptions.py,sha256=LhGJQCGHzEu7Sx2Dfl28OFeDg1OmrwufFjAWXlxQnEA,366
 playwrightcapture/helpers.py,sha256=Xqs09zHhzAWnpBtQ0A9YAxg80P3Lj7aBj5M2WuEr0so,1843
 playwrightcapture/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 playwrightcapture/socks5dnslookup.py,sha256=ZpOf8tgsRQZi-WDcn9JbbG1bKz9DSfK_jz1l53UI1Ho,4058
-playwrightcapture-1.31.0.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
-playwrightcapture-1.31.0.dist-info/METADATA,sha256=kzY4EECi1UvBriPhS60Oh1plT4XbeH_hsKkXHqwHOZo,3284
-playwrightcapture-1.31.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-playwrightcapture-1.31.0.dist-info/RECORD,,
+playwrightcapture-1.31.2.dist-info/LICENSE,sha256=uwFc39fTLacBUG-XTuxX6IQKTKhg4z14gWOLt3ex4Ho,1775
+playwrightcapture-1.31.2.dist-info/METADATA,sha256=6JDSFOTJaWk1oER6gis-AbR-vKAaX3vXwOPa9sV-bFA,3285
+playwrightcapture-1.31.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+playwrightcapture-1.31.2.dist-info/RECORD,,