scrapling 0.2.96__py3-none-any.whl → 0.2.98__py3-none-any.whl

scrapling/engines/pw.py

@@ -19,20 +19,20 @@ class PlaywrightEngine:
             self, headless: Union[bool, str] = True,
             disable_resources: bool = False,
             useragent: Optional[str] = None,
-            network_idle: Optional[bool] = False,
+            network_idle: bool = False,
             timeout: Optional[float] = 30000,
             page_action: Callable = None,
             wait_selector: Optional[str] = None,
             locale: Optional[str] = 'en-US',
             wait_selector_state: SelectorWaitStates = 'attached',
-            stealth: Optional[bool] = False,
-            real_chrome: Optional[bool] = False,
-            hide_canvas: Optional[bool] = False,
-            disable_webgl: Optional[bool] = False,
+            stealth: bool = False,
+            real_chrome: bool = False,
+            hide_canvas: bool = False,
+            disable_webgl: bool = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: Optional[bool] = False,
+            nstbrowser_mode: bool = False,
             nstbrowser_config: Optional[Dict] = None,
-            google_search: Optional[bool] = True,
+            google_search: bool = True,
             extra_headers: Optional[Dict[str, str]] = None,
             proxy: Optional[Union[str, Dict[str, str]]] = None,
             adaptor_arguments: Dict = None
@@ -126,7 +126,7 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    @lru_cache(typed=True)
+    @lru_cache(32, typed=True)
     def __set_flags(self):
         """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
         flags = DEFAULT_STEALTH_FLAGS
@@ -169,7 +169,7 @@ class PlaywrightEngine:
 
         return context_kwargs
 
-    @lru_cache()
+    @lru_cache(1)
     def __stealth_scripts(self):
         # Basic bypasses nothing fancy as I'm still working on it
         # But with adding these bypasses to the above config, it bypasses many online tests like
@@ -188,6 +188,38 @@ class PlaywrightEngine:
             )
         )
 
+    def _process_response_history(self, first_response):
+        """Process response history to build a list of Response objects"""
+        history = []
+        current_request = first_response.request.redirected_from
+
+        try:
+            while current_request:
+                try:
+                    current_response = current_request.response()
+                    history.insert(0, Response(
+                        url=current_request.url,
+                        # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                        text='',
+                        body=b'',
+                        status=current_response.status if current_response else 301,
+                        reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                        encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                        cookies={},
+                        headers=current_response.all_headers() if current_response else {},
+                        request_headers=current_request.all_headers(),
+                        **self.adaptor_arguments
+                    ))
+                except Exception as e:
+                    log.error(f"Error processing redirect: {e}")
+                    break
+
+                current_request = current_request.redirected_from
+        except Exception as e:
+            log.error(f"Error processing response history: {e}")
+
+        return history
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
@@ -201,8 +233,8 @@ class PlaywrightEngine:
         else:
             from rebrowser_playwright.sync_api import sync_playwright
 
-        # Store the final response
         final_response = None
+        referer = generate_convincing_referer(url) if self.google_search else None
 
         def handle_response(finished_response: PlaywrightResponse):
             nonlocal final_response
@@ -218,11 +250,9 @@ class PlaywrightEngine:
                 browser = p.chromium.launch(**self.__launch_kwargs())
 
             context = browser.new_context(**self.__context_kwargs())
-            # Finally we are in business
             page = context.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
-            # Listen for all responses
             page.on("response", handle_response)
 
             if self.extra_headers:
@@ -235,54 +265,51 @@ class PlaywrightEngine:
                 for script in self.__stealth_scripts():
                     page.add_init_script(path=script)
 
-            first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            first_response = page.goto(url, referer=referer)
             page.wait_for_load_state(state="domcontentloaded")
+
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
             if self.page_action is not None:
-                page = self.page_action(page)
+                try:
+                    page = self.page_action(page)
+                except Exception as e:
+                    log.error(f"Error executing page_action: {e}")
 
             if self.wait_selector and type(self.wait_selector) is str:
-                waiter = page.locator(self.wait_selector)
-                waiter.first.wait_for(state=self.wait_selector_state)
-                # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                page.wait_for_load_state(state="load")
-                page.wait_for_load_state(state="domcontentloaded")
-                if self.network_idle:
-                    page.wait_for_load_state('networkidle')
+                try:
+                    waiter = page.locator(self.wait_selector)
+                    waiter.first.wait_for(state=self.wait_selector_state)
+                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                    page.wait_for_load_state(state="load")
+                    page.wait_for_load_state(state="domcontentloaded")
+                    if self.network_idle:
+                        page.wait_for_load_state('networkidle')
+                except Exception as e:
+                    log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
+            if not final_response:
+                raise ValueError("Failed to get a response from the page")
+
             # This will be parsed inside `Response`
             encoding = final_response.headers.get('content-type', '') or 'utf-8'  # default encoding
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = []
-            current_request = first_response.request.redirected_from
-            while current_request:
-                current_response = current_request.response()
-
-                history.insert(0, Response(
-                    url=current_request.url,
-                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
-                    text='',
-                    body=b'',
-                    status=current_response.status if current_response else 301,
-                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
-                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
-                    cookies={},
-                    headers=current_response.all_headers() if current_response else {},
-                    request_headers=current_request.all_headers(),
-                    **self.adaptor_arguments
-                ))
-                current_request = current_request.redirected_from
+            history = self._process_response_history(first_response)
+            try:
+                page_content = page.content()
+            except Exception as e:
+                log.error(f"Error getting page content: {e}")
+                page_content = ""
 
             response = Response(
                 url=page.url,
-                text=page.content(),
-                body=page.content().encode('utf-8'),
+                text=page_content,
+                body=page_content.encode('utf-8'),
                 status=final_response.status,
                 reason=status_text,
                 encoding=encoding,
@@ -293,6 +320,7 @@ class PlaywrightEngine:
                 **self.adaptor_arguments
             )
             page.close()
+            context.close()
         return response
 
     async def async_fetch(self, url: str) -> Response:
@@ -308,8 +336,8 @@ class PlaywrightEngine:
         else:
             from rebrowser_playwright.async_api import async_playwright
 
-        # Store the final response
         final_response = None
+        referer = generate_convincing_referer(url) if self.google_search else None
 
         async def handle_response(finished_response: PlaywrightResponse):
             nonlocal final_response
@@ -325,11 +353,9 @@ class PlaywrightEngine:
                 browser = await p.chromium.launch(**self.__launch_kwargs())
 
             context = await browser.new_context(**self.__context_kwargs())
-            # Finally we are in business
             page = await context.new_page()
             page.set_default_navigation_timeout(self.timeout)
             page.set_default_timeout(self.timeout)
-            # Listen for all responses
             page.on("response", handle_response)
 
             if self.extra_headers:
@@ -342,54 +368,51 @@ class PlaywrightEngine:
                 for script in self.__stealth_scripts():
                     await page.add_init_script(path=script)
 
-            first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            first_response = await page.goto(url, referer=referer)
             await page.wait_for_load_state(state="domcontentloaded")
+
             if self.network_idle:
                 await page.wait_for_load_state('networkidle')
 
             if self.page_action is not None:
-                page = await self.page_action(page)
+                try:
+                    page = await self.page_action(page)
+                except Exception as e:
+                    log.error(f"Error executing async page_action: {e}")
 
             if self.wait_selector and type(self.wait_selector) is str:
-                waiter = page.locator(self.wait_selector)
-                await waiter.first.wait_for(state=self.wait_selector_state)
-                # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                await page.wait_for_load_state(state="load")
-                await page.wait_for_load_state(state="domcontentloaded")
-                if self.network_idle:
-                    await page.wait_for_load_state('networkidle')
+                try:
+                    waiter = page.locator(self.wait_selector)
+                    await waiter.first.wait_for(state=self.wait_selector_state)
+                    # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                    await page.wait_for_load_state(state="load")
+                    await page.wait_for_load_state(state="domcontentloaded")
+                    if self.network_idle:
+                        await page.wait_for_load_state('networkidle')
+                except Exception as e:
+                    log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
+            if not final_response:
+                raise ValueError("Failed to get a response from the page")
+
             # This will be parsed inside `Response`
             encoding = final_response.headers.get('content-type', '') or 'utf-8'  # default encoding
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = []
-            current_request = first_response.request.redirected_from
-            while current_request:
-                current_response = await current_request.response()
-
-                history.insert(0, Response(
-                    url=current_request.url,
-                    # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
-                    text='',
-                    body=b'',
-                    status=current_response.status if current_response else 301,
-                    reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
-                    encoding=current_response.headers.get('content-type', '') or 'utf-8',
-                    cookies={},
-                    headers=await current_response.all_headers() if current_response else {},
-                    request_headers=await current_request.all_headers(),
-                    **self.adaptor_arguments
-                ))
-                current_request = current_request.redirected_from
+            history = self._process_response_history(first_response)
+            try:
+                page_content = await page.content()
+            except Exception as e:
+                log.error(f"Error getting page content in async: {e}")
+                page_content = ""
 
             response = Response(
                 url=page.url,
-                text=await page.content(),
-                body=(await page.content()).encode('utf-8'),
+                text=page_content,
+                body=page_content.encode('utf-8'),
                 status=final_response.status,
                 reason=status_text,
                 encoding=encoding,
@@ -400,4 +423,6 @@ class PlaywrightEngine:
                 **self.adaptor_arguments
             )
             await page.close()
+            await context.close()
+
         return response

scrapling/engines/static.py

@@ -7,10 +7,10 @@ from scrapling.core.utils import log, lru_cache
 from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
-@lru_cache(typed=True)
+@lru_cache(2, typed=True)  # Singleton easily
 class StaticEngine:
     def __init__(
-            self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, follow_redirects: bool = True,
+            self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,
             timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None
     ):
         """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
@@ -79,17 +79,25 @@ class StaticEngine:
             **self.adaptor_arguments
         )
 
+    def _make_request(self, method: str, **kwargs) -> Response:
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
+            request = getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)
+
+    async def _async_make_request(self, method: str, **kwargs) -> Response:
+        headers = self._headers_job(kwargs.pop('headers', {}))
+        async with httpx.AsyncClient(proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)) as client:
+            request = await getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)
+
     def get(self, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
         :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
-            request = client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return self._make_request('get', **kwargs)
 
     async def async_get(self, **kwargs: Dict) -> Response:
         """Make basic async HTTP GET request for you but with some added flavors.
@@ -97,11 +105,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            request = await client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return await self._async_make_request('get', **kwargs)
 
     def post(self, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
@@ -109,11 +113,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
-            request = client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return self._make_request('post', **kwargs)
 
     async def async_post(self, **kwargs: Dict) -> Response:
         """Make basic async HTTP POST request for you but with some added flavors.
@@ -121,11 +121,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            request = await client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return await self._async_make_request('post', **kwargs)
 
     def delete(self, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
@@ -133,11 +129,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
-            request = client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return self._make_request('delete', **kwargs)
 
     async def async_delete(self, **kwargs: Dict) -> Response:
         """Make basic async HTTP DELETE request for you but with some added flavors.
@@ -145,11 +137,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            request = await client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return await self._async_make_request('delete', **kwargs)
 
     def put(self, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
@@ -157,11 +145,7 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
-            request = client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return self._make_request('put', **kwargs)
 
     async def async_put(self, **kwargs: Dict) -> Response:
         """Make basic async HTTP PUT request for you but with some added flavors.
@@ -169,8 +153,4 @@ class StaticEngine:
         :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        headers = self._headers_job(kwargs.pop('headers', {}))
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            request = await client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
-
-        return self._prepare_response(request)
+        return await self._async_make_request('put', **kwargs)

scrapling/engines/toolbelt/custom.py

@@ -16,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @lru_cache(maxsize=None)
+    @lru_cache(maxsize=128)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -38,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @lru_cache(maxsize=None)
+    @lru_cache(maxsize=128)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 

scrapling/engines/toolbelt/fingerprints.py

@@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union
 from scrapling.core.utils import lru_cache
 
 
-@lru_cache(None, typed=True)
+@lru_cache(10, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@lru_cache(None, typed=True)
+@lru_cache(1, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@lru_cache(None, typed=True)
+@lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

scrapling/fetchers.py

@@ -11,7 +11,7 @@ class Fetcher(BaseFetcher):
     Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
     """
     def get(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
@@ -30,7 +30,7 @@ class Fetcher(BaseFetcher):
         return response_object
 
     def post(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
@@ -49,7 +49,7 @@ class Fetcher(BaseFetcher):
         return response_object
 
     def put(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
@@ -69,7 +69,7 @@ class Fetcher(BaseFetcher):
         return response_object
 
     def delete(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
@@ -90,7 +90,7 @@ class Fetcher(BaseFetcher):
 
 class AsyncFetcher(Fetcher):
     async def get(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
@@ -109,7 +109,7 @@ class AsyncFetcher(Fetcher):
         return response_object
 
     async def post(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
@@ -128,7 +128,7 @@ class AsyncFetcher(Fetcher):
         return response_object
 
     async def put(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
@@ -147,7 +147,7 @@ class AsyncFetcher(Fetcher):
         return response_object
 
     async def delete(
-            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
             proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
@@ -173,11 +173,11 @@ class StealthyFetcher(BaseFetcher):
      Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
     """
     def fetch(
-            self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
+            self, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
+            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
-            wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
-            proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = False, geoip: Optional[bool] = False,
+            wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
+            proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
@@ -231,11 +231,11 @@ class StealthyFetcher(BaseFetcher):
         return engine.fetch(url)
 
     async def async_fetch(
-            self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
+            self, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
+            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
-            wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
-            proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = False, geoip: Optional[bool] = False,
+            wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
+            proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
@@ -307,13 +307,13 @@ class PlayWrightFetcher(BaseFetcher):
     """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
-            useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
+            useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,
             page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
-            hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
             proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
-            stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
+            stealth: bool = False, real_chrome: bool = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: Optional[bool] = False, nstbrowser_config: Optional[Dict] = None,
+            nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
         """Opens up a browser and do your request based on your chosen options below.
 
@@ -367,13 +367,13 @@ class PlayWrightFetcher(BaseFetcher):
 
     async def async_fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
-            useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
+            useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,
             page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
-            hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
             proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
-            stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
+            stealth: bool = False, real_chrome: bool = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: Optional[bool] = False, nstbrowser_config: Optional[Dict] = None,
+            nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
         """Opens up a browser and do your request based on your chosen options below.