scrapling 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

scrapling/engines/toolbelt/custom.py

@@ -2,13 +2,13 @@
 Functions related to custom types or type checking
 """
 import inspect
-import logging
 from email.message import Message
 
+from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
+                                   Type, Union)
 from scrapling.core.custom_types import MappingProxyType
+from scrapling.core.utils import log, lru_cache
 from scrapling.parser import Adaptor, SQLiteStorageSystem
-from scrapling.core.utils import setup_basic_logging, cache
-from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple
 
 
 class ResponseEncoding:
@@ -16,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @cache(maxsize=None)
+    @lru_cache(maxsize=None)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -38,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @cache(maxsize=None)
+    @lru_cache(maxsize=None)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 
@@ -84,7 +84,10 @@ class ResponseEncoding:
 class Response(Adaptor):
     """This class is returned by all engines as a way to unify response type between different libraries."""
 
-    def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, encoding: str = 'utf-8', **adaptor_arguments: Dict):
+    _is_response_result_logged = False  # Class-level flag, initialized to False
+
+    def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
+                 encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict):
         automatch_domain = adaptor_arguments.pop('automatch_domain', None)
         self.status = status
         self.reason = reason
@@ -95,6 +98,10 @@ class Response(Adaptor):
         super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
         # For back-ward compatibility
         self.adaptor = self
+        # For easier debugging while working from a Python shell
+        if not Response._is_response_result_logged:
+            log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
+            Response._is_response_result_logged = True
 
     # def __repr__(self):
     #     return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
@@ -103,8 +110,8 @@ class Response(Adaptor):
 class BaseFetcher:
     def __init__(
             self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
-            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = False,
-            automatch_domain: Optional[str] = None,
+            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,
+            automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
     ):
         """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
         are detected and passed automatically from the Fetcher based on the response for accessibility.
@@ -112,6 +119,7 @@ class BaseFetcher:
         :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
             libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
         :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+        :param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
         :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
             priority over all auto-match related arguments/functions in the class.
         :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
@@ -119,23 +127,20 @@ class BaseFetcher:
             If empty, default values will be used.
         :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
             Otherwise, the domain of the request is used by default.
-        :param debug: Enable debug mode
         """
         # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
         self.adaptor_arguments = dict(
             huge_tree=huge_tree,
             keep_comments=keep_comments,
+            keep_cdata=keep_cdata,
             auto_match=auto_match,
             storage=storage,
-            storage_args=storage_args,
-            debug=debug,
+            storage_args=storage_args
         )
-        # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
-        setup_basic_logging(level='debug' if debug else 'info')
         if automatch_domain:
             if type(automatch_domain) is not str:
-                logging.warning('[Ignored] The argument "automatch_domain" must be of string type')
+                log.warning('[Ignored] The argument "automatch_domain" must be of string type')
             else:
                 self.adaptor_arguments.update({'automatch_domain': automatch_domain})
 
@@ -211,7 +216,7 @@ class StatusText:
     })
 
     @classmethod
-    @cache(maxsize=128)
+    @lru_cache(maxsize=128)
     def get(cls, status_code: int) -> str:
         """Get the phrase for a given HTTP status code."""
         return cls._phrases.get(status_code, "Unknown Status Code")
@@ -278,7 +283,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
         error_msg = f'Argument "{var_name}" cannot be None'
         if critical:
             raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
+        log.error(f'[Ignored] {error_msg}')
         return default_value
 
     # If no valid_types specified and variable has a value, return it
@@ -291,13 +296,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
         error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
         if critical:
             raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
+        log.error(f'[Ignored] {error_msg}')
         return default_value
 
     return variable
-
-
-# Pew Pew
-def do_nothing(page):
-    # Just works as a filler for `page_action` argument in browser engines
-    return page

scrapling/engines/toolbelt/fingerprints.py

@@ -4,15 +4,15 @@ Functions related to generating headers and fingerprints generally
 
 import platform
 
-from scrapling.core.utils import cache
-from scrapling.core._types import Union, Dict
-
+from browserforge.fingerprints import Fingerprint, FingerprintGenerator
+from browserforge.headers import Browser, HeaderGenerator
 from tldextract import extract
-from browserforge.headers import HeaderGenerator, Browser
-from browserforge.fingerprints import FingerprintGenerator, Fingerprint
+
+from scrapling.core._types import Dict, Union
+from scrapling.core.utils import lru_cache
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -1,28 +1,41 @@
 """
 Functions related to files and URLs
 """
-
 import os
-import logging
-from urllib.parse import urlparse, urlencode
+from urllib.parse import urlencode, urlparse
+
+from playwright.async_api import Route as async_Route
+from playwright.sync_api import Route
 
-from scrapling.core.utils import cache
-from scrapling.core._types import Union, Dict, Optional
+from scrapling.core._types import Dict, Optional, Union
+from scrapling.core.utils import log, lru_cache
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 
-from playwright.sync_api import Route
+
+def intercept_route(route: Route):
+    """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+
+    :param route: PlayWright `Route` object of the current page
+    :return: PlayWright `Route` object
+    """
+    if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
+        log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        route.abort()
+    else:
+        route.continue_()
 
 
-def intercept_route(route: Route) -> Union[Route, None]:
+async def async_intercept_route(route: async_Route):
     """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
 
     :param route: PlayWright `Route` object of the current page
     :return: PlayWright `Route` object
     """
     if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
-        logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
-        return route.abort()
-    return route.continue_()
+        log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        await route.abort()
+    else:
+        await route.continue_()
 
 
 def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
@@ -43,7 +56,7 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict
                 }
             except ValueError:
                 # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
-                raise TypeError(f'The proxy argument\'s string is in invalid format!')
+                raise TypeError('The proxy argument\'s string is in invalid format!')
 
         elif isinstance(proxy_string, dict):
             valid_keys = ('server', 'username', 'password', )
@@ -97,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

scrapling/fetchers.py

@@ -1,7 +1,8 @@
-from scrapling.core._types import Dict, Optional, Union, Callable, List, Literal
-
-from scrapling.engines.toolbelt import Response, BaseFetcher, do_nothing
-from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, check_if_engine_usable
+from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
+                                   Union)
+from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine,
+                               check_if_engine_usable)
+from scrapling.engines.toolbelt import BaseFetcher, Response
 
 
 class Fetcher(BaseFetcher):
@@ -9,7 +10,9 @@ class Fetcher(BaseFetcher):
 
     Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
     """
-    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def get(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
         :param url: Target url.
@@ -18,13 +21,17 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).get(**kwargs)
         return response_object
 
-    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def post(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
         :param url: Target url.
@@ -33,13 +40,17 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).post(**kwargs)
         return response_object
 
-    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def put(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
         :param url: Target url
@@ -48,14 +59,96 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
 
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).put(**kwargs)
+        return response_object
+
+    def delete(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
+        """Make basic HTTP DELETE request for you but with some added flavors.
+
+        :param url: Target url
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request came from Google's search of this URL's domain.
+        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(**kwargs)
+        return response_object
+
+
+class AsyncFetcher(Fetcher):
+    async def get(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
+        """Make basic HTTP GET request for you but with some added flavors.
+
+        :param url: Target url.
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_get(**kwargs)
+        return response_object
+
+    async def post(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
+        """Make basic HTTP POST request for you but with some added flavors.
+
+        :param url: Target url.
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request came from Google's search of this URL's domain.
+        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs)
+        return response_object
+
+    async def put(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
+        """Make basic HTTP PUT request for you but with some added flavors.
+
+        :param url: Target url
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request came from Google's search of this URL's domain.
+        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs)
         return response_object
 
-    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    async def delete(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
         :param url: Target url
@@ -64,10 +157,12 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)
+        adaptor_arguments = tuple(self.adaptor_arguments.items())
+        response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_delete(**kwargs)
         return response_object
 
 
@@ -79,10 +174,10 @@ class StealthyFetcher(BaseFetcher):
     """
     def fetch(
             self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
-            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
+            timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
             wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,
-            os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True,
+            os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
@@ -98,7 +193,9 @@ class StealthyFetcher(BaseFetcher):
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
+        :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
+        :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
+            It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
@@ -112,6 +209,7 @@ class StealthyFetcher(BaseFetcher):
         """
         engine = CamoufoxEngine(
             proxy=proxy,
+            geoip=geoip,
             addons=addons,
             timeout=timeout,
             headless=headless,
@@ -132,6 +230,64 @@ class StealthyFetcher(BaseFetcher):
         )
         return engine.fetch(url)
 
+    async def async_fetch(
+            self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
+            timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,
+            os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,
+    ) -> Response:
+        """
+        Opens up a browser and do your request based on your chosen options below.
+
+        :param url: Target url.
+        :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
+        :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
+            It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        engine = CamoufoxEngine(
+            proxy=proxy,
+            geoip=geoip,
+            addons=addons,
+            timeout=timeout,
+            headless=headless,
+            humanize=humanize,
+            disable_ads=disable_ads,
+            allow_webgl=allow_webgl,
+            page_action=page_action,
+            network_idle=network_idle,
+            block_images=block_images,
+            block_webrtc=block_webrtc,
+            os_randomize=os_randomize,
+            wait_selector=wait_selector,
+            google_search=google_search,
+            extra_headers=extra_headers,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+            adaptor_arguments=self.adaptor_arguments,
+        )
+        return await engine.async_fetch(url)
+
 
 class PlayWrightFetcher(BaseFetcher):
     """A `Fetcher` class type that provide many options, all of them are based on PlayWright.
@@ -152,7 +308,7 @@ class PlayWrightFetcher(BaseFetcher):
     def fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
-            page_action: Optional[Callable] = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
+            page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
             hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
             proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
             stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
@@ -209,6 +365,66 @@ class PlayWrightFetcher(BaseFetcher):
         )
         return engine.fetch(url)
 
+    async def async_fetch(
+            self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
+            useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
+            page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
+            hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
+            stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
+            cdp_url: Optional[str] = None,
+            nstbrowser_mode: Optional[bool] = False, nstbrowser_config: Optional[Dict] = None,
+    ) -> Response:
+        """Opens up a browser and do your request based on your chosen options below.
+
+        :param url: Target url.
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
+        :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        engine = PlaywrightEngine(
+            proxy=proxy,
+            locale=locale,
+            timeout=timeout,
+            stealth=stealth,
+            cdp_url=cdp_url,
+            headless=headless,
+            useragent=useragent,
+            real_chrome=real_chrome,
+            page_action=page_action,
+            hide_canvas=hide_canvas,
+            network_idle=network_idle,
+            google_search=google_search,
+            extra_headers=extra_headers,
+            wait_selector=wait_selector,
+            disable_webgl=disable_webgl,
+            nstbrowser_mode=nstbrowser_mode,
+            nstbrowser_config=nstbrowser_config,
+            disable_resources=disable_resources,
+            wait_selector_state=wait_selector_state,
+            adaptor_arguments=self.adaptor_arguments,
+        )
+        return await engine.async_fetch(url)
+
 
 class CustomFetcher(BaseFetcher):
     def fetch(self, url: str, browser_engine, **kwargs) -> Response: