scrapling 0.2.8__py3-none-any.whl → 0.2.91__py3-none-any.whl

scrapling/engines/pw.py

@@ -1,12 +1,14 @@
 import json
-import logging
 
-from scrapling.core._types import Callable, Dict, List, Optional, Union
+from scrapling.core._types import (Callable, Dict, Optional,
+                                   SelectorWaitStates, Union)
+from scrapling.core.utils import log, lru_cache
 from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
                                          NSTBROWSER_DEFAULT_QUERY)
 from scrapling.engines.toolbelt import (Response, StatusText,
+                                        async_intercept_route,
                                         check_type_validity, construct_cdp_url,
-                                        construct_proxy_dict, do_nothing,
+                                        construct_proxy_dict,
                                         generate_convincing_referer,
                                         generate_headers, intercept_route,
                                         js_bypass_path)
@@ -19,10 +21,10 @@ class PlaywrightEngine:
             useragent: Optional[str] = None,
             network_idle: Optional[bool] = False,
             timeout: Optional[float] = 30000,
-            page_action: Callable = do_nothing,
+            page_action: Callable = None,
             wait_selector: Optional[str] = None,
             locale: Optional[str] = 'en-US',
-            wait_selector_state: Optional[str] = 'attached',
+            wait_selector_state: SelectorWaitStates = 'attached',
             stealth: Optional[bool] = False,
             real_chrome: Optional[bool] = False,
             hide_canvas: Optional[bool] = False,
@@ -74,11 +76,14 @@ class PlaywrightEngine:
         self.cdp_url = cdp_url
         self.useragent = useragent
         self.timeout = check_type_validity(timeout, [int, float], 30000)
-        if callable(page_action):
-            self.page_action = page_action
+        if page_action is not None:
+            if callable(page_action):
+                self.page_action = page_action
+            else:
+                self.page_action = None
+                log.error('[Ignored] Argument "page_action" must be callable')
         else:
-            self.page_action = do_nothing
-            logging.error('[Ignored] Argument "page_action" must be callable')
+            self.page_action = None
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state
@@ -94,10 +99,8 @@ class PlaywrightEngine:
             # '--disable-extensions',
         ]
 
-    def _cdp_url_logic(self, flags: Optional[List] = None) -> str:
+    def _cdp_url_logic(self) -> str:
         """Constructs new CDP URL if NSTBrowser is enabled otherwise return CDP URL as it is
-
-        :param flags: Chrome flags to be added to NSTBrowser query
         :return: CDP URL
         """
         cdp_url = self.cdp_url
@@ -106,7 +109,8 @@ class PlaywrightEngine:
                 config = self.nstbrowser_config
             else:
                 query = NSTBROWSER_DEFAULT_QUERY.copy()
-                if flags:
+                if self.stealth:
+                    flags = self.__set_flags()
                     query.update({
                         "args": dict(zip(flags, [''] * len(flags))),  # browser args should be a dictionary
                     })
@@ -122,78 +126,104 @@ class PlaywrightEngine:
 
         return cdp_url
 
+    @lru_cache(typed=True)
+    def __set_flags(self):
+        """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
+        flags = DEFAULT_STEALTH_FLAGS
+        if self.hide_canvas:
+            flags += ('--fingerprinting-canvas-image-data-noise',)
+        if self.disable_webgl:
+            flags += ('--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2',)
+
+        return flags
+
+    def __launch_kwargs(self):
+        """Creates the arguments we will use while launching playwright's browser"""
+        launch_kwargs = {'headless': self.headless, 'ignore_default_args': self.harmful_default_args, 'channel': 'chrome' if self.real_chrome else 'chromium'}
+        if self.stealth:
+            launch_kwargs.update({'args': self.__set_flags(), 'chromium_sandbox': True})
+
+        return launch_kwargs
+
+    def __context_kwargs(self):
+        """Creates the arguments for the browser context"""
+        context_kwargs = {
+            "proxy": self.proxy,
+            "locale": self.locale,
+            "color_scheme": 'dark',  # Bypasses the 'prefersLightColor' check in creepjs
+            "device_scale_factor": 2,
+            "extra_http_headers": self.extra_headers if self.extra_headers else {},
+            "user_agent": self.useragent if self.useragent else generate_headers(browser_mode=True).get('User-Agent'),
+        }
+        if self.stealth:
+            context_kwargs.update({
+                'is_mobile': False,
+                'has_touch': False,
+                # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
+                'service_workers': 'allow',
+                'ignore_https_errors': True,
+                'screen': {'width': 1920, 'height': 1080},
+                'viewport': {'width': 1920, 'height': 1080},
+                'permissions': ['geolocation', 'notifications']
+            })
+
+        return context_kwargs
+
+    @lru_cache()
+    def __stealth_scripts(self):
+        # Basic bypasses nothing fancy as I'm still working on it
+        # But with adding these bypasses to the above config, it bypasses many online tests like
+        # https://bot.sannysoft.com/
+        # https://kaliiiiiiiiii.github.io/brotector/
+        # https://pixelscan.net/
+        # https://iphey.com/
+        # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
+        # https://arh.antoinevastel.com/bots/areyouheadless/
+        # https://prescience-data.github.io/execution-monitor.html
+        return tuple(
+            js_bypass_path(script) for script in (
+                # Order is important
+                'webdriver_fully.js', 'window_chrome.js', 'navigator_plugins.js', 'pdf_viewer.js',
+                'notification_permission.js', 'screen_props.js', 'playwright_fingerprint.js'
+            )
+        )
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
         :param url: Target url.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
+        from playwright.sync_api import Response as PlaywrightResponse
         if not self.stealth or self.real_chrome:
             # Because rebrowser_playwright doesn't play well with real browsers
             from playwright.sync_api import sync_playwright
         else:
             from rebrowser_playwright.sync_api import sync_playwright
 
-        with sync_playwright() as p:
-            # Handle the UserAgent early
-            if self.useragent:
-                extra_headers = {}
-                useragent = self.useragent
-            else:
-                extra_headers = {}
-                useragent = generate_headers(browser_mode=True).get('User-Agent')
+        # Store the final response
+        final_response = None
 
-            # Prepare the flags before diving
-            flags = DEFAULT_STEALTH_FLAGS
-            if self.hide_canvas:
-                flags += ['--fingerprinting-canvas-image-data-noise']
-            if self.disable_webgl:
-                flags += ['--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2']
+        def handle_response(finished_response: PlaywrightResponse):
+            nonlocal final_response
+            if finished_response.request.resource_type == "document":
+                final_response = finished_response
 
+        with sync_playwright() as p:
             # Creating the browser
             if self.cdp_url:
-                cdp_url = self._cdp_url_logic(flags if self.stealth else None)
+                cdp_url = self._cdp_url_logic()
                 browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
             else:
-                if self.stealth:
-                    browser = p.chromium.launch(
-                        headless=self.headless, args=flags, ignore_default_args=self.harmful_default_args, chromium_sandbox=True, channel='chrome' if self.real_chrome else 'chromium'
-                    )
-                else:
-                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=self.harmful_default_args, channel='chrome' if self.real_chrome else 'chromium')
-
-            # Creating the context
-            if self.stealth:
-                context = browser.new_context(
-                    locale=self.locale,
-                    is_mobile=False,
-                    has_touch=False,
-                    proxy=self.proxy,
-                    color_scheme='dark',  # Bypasses the 'prefersLightColor' check in creepjs
-                    user_agent=useragent,
-                    device_scale_factor=2,
-                    # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
-                    service_workers="allow",
-                    ignore_https_errors=True,
-                    extra_http_headers=extra_headers,
-                    screen={"width": 1920, "height": 1080},
-                    viewport={"width": 1920, "height": 1080},
-                    permissions=["geolocation", 'notifications'],
-                )
-            else:
-                context = browser.new_context(
-                    locale=self.locale,
-                    proxy=self.proxy,
-                    color_scheme='dark',
-                    user_agent=useragent,
-                    device_scale_factor=2,
-                    extra_http_headers=extra_headers
-                )
+                browser = p.chromium.launch(**self.__launch_kwargs())
 
+            context = browser.new_context(**self.__context_kwargs())
             # Finally we are in business
             page = context.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
+            # Listen for all responses
+            page.on("response", handle_response)
 
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
@@ -202,29 +232,16 @@ class PlaywrightEngine:
                 page.route("**/*", intercept_route)
 
             if self.stealth:
-                # Basic bypasses nothing fancy as I'm still working on it
-                # But with adding these bypasses to the above config, it bypasses many online tests like
-                # https://bot.sannysoft.com/
-                # https://kaliiiiiiiiii.github.io/brotector/
-                # https://pixelscan.net/
-                # https://iphey.com/
-                # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
-                # https://arh.antoinevastel.com/bots/areyouheadless/
-                # https://prescience-data.github.io/execution-monitor.html
-                page.add_init_script(path=js_bypass_path('webdriver_fully.js'))
-                page.add_init_script(path=js_bypass_path('window_chrome.js'))
-                page.add_init_script(path=js_bypass_path('navigator_plugins.js'))
-                page.add_init_script(path=js_bypass_path('pdf_viewer.js'))
-                page.add_init_script(path=js_bypass_path('notification_permission.js'))
-                page.add_init_script(path=js_bypass_path('screen_props.js'))
-                page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
-
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+                for script in self.__stealth_scripts():
+                    page.add_init_script(path=script)
+
+            first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
             page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
-            page = self.page_action(page)
+            if self.page_action is not None:
+                page = self.page_action(page)
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
@@ -235,25 +252,112 @@ class PlaywrightEngine:
                 if self.network_idle:
                     page.wait_for_load_state('networkidle')
 
+            response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
+            # In case we didn't catch a document type somehow
+            final_response = final_response if final_response else first_response
             # This will be parsed inside `Response`
-            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
-
-            status_text = res.status_text
+            encoding = final_response.headers.get('content-type', '') or 'utf-8'  # default encoding
             # PlayWright API sometimes give empty status text for some reason!
-            if not status_text:
-                status_text = StatusText.get(res.status)
+            status_text = final_response.status_text or StatusText.get(final_response.status)
 
             response = Response(
-                url=res.url,
+                url=final_response.url,
                 text=page.content(),
-                body=page.content().encode('utf-8'),
-                status=res.status,
+                body=response_bytes,
+                status=final_response.status,
                 reason=status_text,
                 encoding=encoding,
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
-                headers=res.all_headers(),
-                request_headers=res.request.all_headers(),
+                headers=final_response.all_headers(),
+                request_headers=final_response.request.all_headers(),
                 **self.adaptor_arguments
             )
             page.close()
         return response
+
+    async def async_fetch(self, url: str) -> Response:
+        """Async version of `fetch`
+
+        :param url: Target url.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        from playwright.async_api import Response as PlaywrightResponse
+        if not self.stealth or self.real_chrome:
+            # Because rebrowser_playwright doesn't play well with real browsers
+            from playwright.async_api import async_playwright
+        else:
+            from rebrowser_playwright.async_api import async_playwright
+
+        # Store the final response
+        final_response = None
+
+        async def handle_response(finished_response: PlaywrightResponse):
+            nonlocal final_response
+            if finished_response.request.resource_type == "document":
+                final_response = finished_response
+
+        async with async_playwright() as p:
+            # Creating the browser
+            if self.cdp_url:
+                cdp_url = self._cdp_url_logic()
+                browser = await p.chromium.connect_over_cdp(endpoint_url=cdp_url)
+            else:
+                browser = await p.chromium.launch(**self.__launch_kwargs())
+
+            context = await browser.new_context(**self.__context_kwargs())
+            # Finally we are in business
+            page = await context.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+            # Listen for all responses
+            page.on("response", handle_response)
+
+            if self.extra_headers:
+                await page.set_extra_http_headers(self.extra_headers)
+
+            if self.disable_resources:
+                await page.route("**/*", async_intercept_route)
+
+            if self.stealth:
+                for script in self.__stealth_scripts():
+                    await page.add_init_script(path=script)
+
+            first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            await page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                await page.wait_for_load_state('networkidle')
+
+            if self.page_action is not None:
+                page = await self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                await waiter.first.wait_for(state=self.wait_selector_state)
+                # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                await page.wait_for_load_state(state="load")
+                await page.wait_for_load_state(state="domcontentloaded")
+                if self.network_idle:
+                    await page.wait_for_load_state('networkidle')
+
+            response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
+            # In case we didn't catch a document type somehow
+            final_response = final_response if final_response else first_response
+            # This will be parsed inside `Response`
+            encoding = final_response.headers.get('content-type', '') or 'utf-8'  # default encoding
+            # PlayWright API sometimes give empty status text for some reason!
+            status_text = final_response.status_text or StatusText.get(final_response.status)
+
+            response = Response(
+                url=final_response.url,
+                text=await page.content(),
+                body=response_bytes,
+                status=final_response.status,
+                reason=status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
+                headers=await final_response.all_headers(),
+                request_headers=await final_response.request.all_headers(),
+                **self.adaptor_arguments
+            )
+            await page.close()
+        return response

scrapling/engines/static.py

@@ -1,34 +1,44 @@
-import logging
-
 import httpx
 from httpx._models import Response as httpxResponse
 
-from scrapling.core._types import Dict, Optional, Union
+from scrapling.core._types import Dict, Optional, Tuple, Union
+from scrapling.core.utils import log, lru_cache
 
 from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
+@lru_cache(typed=True)
 class StaticEngine:
-    def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
+    def __init__(
+            self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, follow_redirects: bool = True,
+            timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None
+    ):
         """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
 
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
+        self.url = url
+        self.proxy = proxy
+        self.stealth = stealthy_headers
         self.timeout = timeout
         self.follow_redirects = bool(follow_redirects)
+        self.retries = retries
         self._extra_headers = generate_headers(browser_mode=False)
-        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
+        # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached
+        # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`
+        self.adaptor_arguments = dict(adaptor_arguments) if adaptor_arguments else {}
 
-    @staticmethod
-    def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:
+    def _headers_job(self, headers: Optional[Dict]) -> Dict:
         """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
             finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
 
         :param headers: Current headers in the request if the user passed any
-        :param url: The Target URL.
-        :param stealth: Whether stealth mode is enabled or not.
         :return: A dictionary of the new headers.
         """
         headers = headers or {}
@@ -36,12 +46,12 @@ class StaticEngine:
         # Validate headers
         if not headers.get('user-agent') and not headers.get('User-Agent'):
             headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
-            logging.info(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
+            log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
 
-        if stealth:
+        if self.stealth:
             extra_headers = generate_headers(browser_mode=False)
             headers.update(extra_headers)
-            headers.update({'referer': generate_convincing_referer(url)})
+            headers.update({'referer': generate_convincing_referer(self.url)})
 
         return headers
 
@@ -61,69 +71,102 @@ class StaticEngine:
             cookies=dict(response.cookies),
             headers=dict(response.headers),
             request_headers=dict(response.request.headers),
+            method=response.request.method,
             **self.adaptor_arguments
         )
 
-    def get(self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    def get(self, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
-        :param url: Target url.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
-            request = client.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
+            request = client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
 
-    def post(self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    async def async_get(self, **kwargs: Dict) -> Response:
+        """Make basic async HTTP GET request for you but with some added flavors.
+
+        :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            request = await client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+
+        return self._prepare_response(request)
+
+    def post(self, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
-        :param url: Target url.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
-            request = client.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
+            request = client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
 
-    def delete(self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    async def async_post(self, **kwargs: Dict) -> Response:
+        """Make basic async HTTP POST request for you but with some added flavors.
+
+        :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            request = await client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+
+        return self._prepare_response(request)
+
+    def delete(self, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
-        :param url: Target url.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
+            request = client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+
+        return self._prepare_response(request)
+
+    async def async_delete(self, **kwargs: Dict) -> Response:
+        """Make basic async HTTP DELETE request for you but with some added flavors.
+
+        :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
-            request = client.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            request = await client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
 
-    def put(self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+    def put(self, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
-        :param url: Target url.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
+            request = client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+
+        return self._prepare_response(request)
+
+    async def async_put(self, **kwargs: Dict) -> Response:
+        """Make basic async HTTP PUT request for you but with some added flavors.
+
+        :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
-            request = client.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            request = await client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)

scrapling/engines/toolbelt/__init__.py

@@ -1,6 +1,6 @@
 from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,
-                     check_type_validity, do_nothing, get_variable_name)
+                     check_type_validity, get_variable_name)
 from .fingerprints import (generate_convincing_referer, generate_headers,
                            get_os_name)
-from .navigation import (construct_cdp_url, construct_proxy_dict,
-                         intercept_route, js_bypass_path)
+from .navigation import (async_intercept_route, construct_cdp_url,
+                         construct_proxy_dict, intercept_route, js_bypass_path)