PlaywrightCapture 1.25.7__tar.gz → 1.25.9__tar.gz

{playwrightcapture-1.25.7 → playwrightcapture-1.25.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PlaywrightCapture
-Version: 1.25.7
+Version: 1.25.9
 Summary: A simple library to capture websites using playwright
 Home-page: https://github.com/Lookyloo/PlaywrightCapture
 License: BSD-3-Clause
@@ -26,7 +26,7 @@ Requires-Dist: aiohttp[speedups] (>=3.9.5,<4.0.0)
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.12.3,<5.0.0)
 Requires-Dist: dateparser (>=1.2.0,<2.0.0)
-Requires-Dist: playwright (>=1.45.0,<2.0.0)
+Requires-Dist: playwright (>=1.45.1,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)
 Requires-Dist: puremagic (>=1.26,<2.0)
 Requires-Dist: pydub (>=0.25.1,<0.26.0) ; extra == "recaptcha"

{playwrightcapture-1.25.7 → playwrightcapture-1.25.9}/playwrightcapture/capture.py

@@ -18,7 +18,7 @@ from io import BytesIO
 from logging import LoggerAdapter, Logger
 from tempfile import NamedTemporaryFile
 from typing import Any, TypedDict, Literal, TYPE_CHECKING, MutableMapping, Generator
-from urllib.parse import urlparse, unquote, urljoin
+from urllib.parse import urlparse, unquote, urljoin, urlsplit, urlunsplit
 from zipfile import ZipFile
 
 import aiohttp
@@ -164,7 +164,7 @@ class Capture():
         self.proxy: ProxySettings = {}
         if proxy:
             if isinstance(proxy, str):
-                self.proxy = {'server': proxy}
+                self.proxy = self.__prepare_proxy_playwright(proxy)
             elif isinstance(proxy, dict):
                 self.proxy = {'server': proxy['server'], 'bypass': proxy.get('bypass', ''),
                               'username': proxy.get('username', ''),
@@ -187,6 +187,19 @@ class Capture():
         self._locale: str = ''
         self._color_scheme: Literal['dark', 'light', 'no-preference', 'null'] | None = None
 
+    def __prepare_proxy_playwright(self, proxy: str) -> ProxySettings:
+        splitted = urlsplit(proxy)
+        if splitted.username and splitted.password:
+            return {'username': splitted.username, 'password': splitted.password,
+                    'server': urlunsplit((splitted.scheme, f'{splitted.hostname}:{splitted.port}', splitted.path, splitted.query, splitted.fragment))}
+        return {'server': proxy}
+
+    def __prepare_proxy_aiohttp(self, proxy: ProxySettings) -> str:
+        if 'username' in proxy and 'password' in proxy:
+            splitted = urlsplit(proxy['server'])
+            return urlunsplit((splitted.scheme, f'{proxy["username"]}:{proxy["password"]}@{splitted.netloc}', splitted.path, splitted.query, splitted.fragment))
+        return proxy['server']
+
     async def __aenter__(self) -> Capture:
         '''Launch the browser'''
         self._temp_harfile = NamedTemporaryFile(delete=False)
@@ -411,6 +424,9 @@ class Capture():
             if context_vp := device_context_settings.pop('viewport', self._default_viewport):
                 # Always true, but we also always want to pop it.
                 vp = self.viewport if self.viewport else context_vp
+        else:
+            ua = self.user_agent
+            vp = self.viewport
 
         self.context = await self.browser.new_context(
             record_har_path=self._temp_harfile.name,
@@ -1392,9 +1408,9 @@ class Capture():
         Method inspired by https://github.com/ail-project/ail-framework/blob/master/bin/lib/crawlers.py
         """
         connector = None
-        if self.proxy and self.proxy.get('server'):
+        if self.proxy:
             # NOTE 2024-05-17: switch to async to fetch, the lib uses socks5h by default
-            connector = ProxyConnector.from_url(self.proxy['server'])
+            connector = ProxyConnector.from_url(self.__prepare_proxy_aiohttp(self.proxy))
 
         extracted_favicons = self.__extract_favicons(rendered_content)
         if not extracted_favicons:

{playwrightcapture-1.25.7 → playwrightcapture-1.25.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PlaywrightCapture"
-version = "1.25.7"
+version = "1.25.9"
 description = "A simple library to capture websites using playwright"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -19,7 +19,7 @@ classifiers=[
 
 [tool.poetry.dependencies]
 python = "^3.8"
-playwright = "^1.45.0"
+playwright = "^1.45.1"
 dateparser = "^1.2.0"
 beautifulsoup4 = {version= "^4.12.3", extras = ["lxml", "charset_normalizer"]}
 w3lib = "^2.2.1"