scrapling 0.1.2__py3-none-any.whl → 0.2__py3-none-any.whl

scrapling/engines/pw.py

@@ -0,0 +1,232 @@
+import json
+import logging
+from scrapling.core._types import Union, Callable, Optional, List, Dict
+
+from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULT_QUERY
+from scrapling.engines.toolbelt import (
+    Response,
+    do_nothing,
+    js_bypass_path,
+    intercept_route,
+    generate_headers,
+    check_type_validity,
+    construct_cdp_url,
+    generate_convincing_referer,
+)
+
+
+class PlaywrightEngine:
+    def __init__(
+            self, headless: Union[bool, str] = True,
+            disable_resources: bool = False,
+            useragent: Optional[str] = None,
+            network_idle: Optional[bool] = False,
+            timeout: Optional[float] = 30000,
+            page_action: Callable = do_nothing,
+            wait_selector: Optional[str] = None,
+            wait_selector_state: Optional[str] = 'attached',
+            stealth: bool = False,
+            hide_canvas: bool = True,
+            disable_webgl: bool = False,
+            cdp_url: Optional[str] = None,
+            nstbrowser_mode: bool = False,
+            nstbrowser_config: Optional[Dict] = None,
+            google_search: Optional[bool] = True,
+            extra_headers: Optional[Dict[str, str]] = None,
+            adaptor_arguments: Dict = None
+    ):
+        """An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
+
+        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+        :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
+        self.headless = headless
+        self.disable_resources = disable_resources
+        self.network_idle = bool(network_idle)
+        self.stealth = bool(stealth)
+        self.hide_canvas = bool(hide_canvas)
+        self.disable_webgl = bool(disable_webgl)
+        self.google_search = bool(google_search)
+        self.extra_headers = extra_headers or {}
+        self.cdp_url = cdp_url
+        self.useragent = useragent
+        self.timeout = check_type_validity(timeout, [int, float], 30000)
+        if callable(page_action):
+            self.page_action = page_action
+        else:
+            self.page_action = do_nothing
+            logging.error('[Ignored] Argument "page_action" must be callable')
+
+        self.wait_selector = wait_selector
+        self.wait_selector_state = wait_selector_state
+        self.nstbrowser_mode = bool(nstbrowser_mode)
+        self.nstbrowser_config = nstbrowser_config
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
+
+    def _cdp_url_logic(self, flags: Optional[List] = None) -> str:
+        """Constructs new CDP URL if NSTBrowser is enabled otherwise return CDP URL as it is
+
+        :param flags: Chrome flags to be added to NSTBrowser query
+        :return: CDP URL
+        """
+        cdp_url = self.cdp_url
+        if self.nstbrowser_mode:
+            if self.nstbrowser_config and type(self.nstbrowser_config) is Dict:
+                config = self.nstbrowser_config
+            else:
+                query = NSTBROWSER_DEFAULT_QUERY.copy()
+                if flags:
+                    query.update({
+                        "args": dict(zip(flags, [''] * len(flags))),  # browser args should be a dictionary
+                    })
+
+                config = {
+                    'config': json.dumps(query),
+                    # 'token': ''
+                }
+            cdp_url = construct_cdp_url(cdp_url, config)
+        else:
+            # To validate it
+            cdp_url = construct_cdp_url(cdp_url)
+
+        return cdp_url
+
+    def fetch(self, url: str) -> Response:
+        """Opens up the browser and do your request based on your chosen options.
+
+        :param url: Target url.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        if not self.stealth:
+            from playwright.sync_api import sync_playwright
+        else:
+            from rebrowser_playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            # Handle the UserAgent early
+            if self.useragent:
+                extra_headers = {}
+                useragent = self.useragent
+            else:
+                extra_headers = generate_headers(browser_mode=True)
+                useragent = extra_headers.get('User-Agent')
+
+            # Prepare the flags before diving
+            flags = DEFAULT_STEALTH_FLAGS
+            if self.hide_canvas:
+                flags += ['--fingerprinting-canvas-image-data-noise']
+            if self.disable_webgl:
+                flags += ['--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2']
+
+            # Creating the browser
+            if self.cdp_url:
+                cdp_url = self._cdp_url_logic(flags if self.stealth else None)
+                browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
+            else:
+                if self.stealth:
+                    browser = p.chromium.launch(headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True)
+                else:
+                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'])
+
+            # Creating the context
+            if self.stealth:
+                context = browser.new_context(
+                    locale='en-US',
+                    is_mobile=False,
+                    has_touch=False,
+                    color_scheme='dark',  # Bypasses the 'prefersLightColor' check in creepjs
+                    user_agent=useragent,
+                    device_scale_factor=2,
+                    # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
+                    service_workers="allow",
+                    ignore_https_errors=True,
+                    extra_http_headers=extra_headers,
+                    screen={"width": 1920, "height": 1080},
+                    viewport={"width": 1920, "height": 1080},
+                    permissions=["geolocation", 'notifications'],
+                )
+            else:
+                context = browser.new_context(
+                    color_scheme='dark',
+                    user_agent=useragent,
+                    device_scale_factor=2,
+                    extra_http_headers=extra_headers
+                )
+
+            # Finally we are in business
+            page = context.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+
+            if self.extra_headers:
+                page.set_extra_http_headers(self.extra_headers)
+
+            if self.disable_resources:
+                page.route("**/*", intercept_route)
+
+            if self.stealth:
+                # Basic bypasses nothing fancy as I'm still working on it
+                # But with adding these bypasses to the above config, it bypasses many online tests like
+                # https://bot.sannysoft.com/
+                # https://kaliiiiiiiiii.github.io/brotector/
+                # https://pixelscan.net/
+                # https://iphey.com/
+                # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
+                # https://arh.antoinevastel.com/bots/areyouheadless/
+                # https://prescience-data.github.io/execution-monitor.html
+                page.add_init_script(path=js_bypass_path('webdriver_fully.js'))
+                page.add_init_script(path=js_bypass_path('window_chrome.js'))
+                page.add_init_script(path=js_bypass_path('navigator_plugins.js'))
+                page.add_init_script(path=js_bypass_path('pdf_viewer.js'))
+                page.add_init_script(path=js_bypass_path('notification_permission.js'))
+                page.add_init_script(path=js_bypass_path('screen_props.js'))
+                page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
+
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                page.wait_for_load_state('networkidle')
+
+            page = self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                waiter.wait_for(state=self.wait_selector_state)
+
+            content_type = res.headers.get('content-type', '')
+            # Parse charset from content-type
+            encoding = 'utf-8'  # default encoding
+            if 'charset=' in content_type.lower():
+                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+
+            response = Response(
+                url=res.url,
+                text=page.content(),
+                content=res.body(),
+                status=res.status,
+                reason=res.status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
+                headers=res.all_headers(),
+                request_headers=res.request.all_headers(),
+                adaptor_arguments=self.adaptor_arguments
+            )
+            page.close()
+        return response

scrapling/engines/static.py

@@ -0,0 +1,112 @@
+import logging
+
+from scrapling.core._types import Union, Optional, Dict
+from .toolbelt import Response, generate_convincing_referer, generate_headers
+
+import httpx
+from httpx._models import Response as httpxResponse
+
+
+class StaticEngine:
+    def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
+        """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
+
+        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
+        self.timeout = timeout
+        self.follow_redirects = bool(follow_redirects)
+        self._extra_headers = generate_headers(browser_mode=False)
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
+
+    @staticmethod
+    def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:
+        """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
+        finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
+
+        :param headers: Current headers in the request if the user passed any
+        :param url: The Target URL.
+        :param stealth: Whether stealth mode is enabled or not.
+        :return: A dictionary of the new headers.
+        """
+        headers = headers or {}
+
+        # Validate headers
+        if not headers.get('user-agent') and not headers.get('User-Agent'):
+            headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
+            logging.info(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
+
+        if stealth:
+            extra_headers = generate_headers(browser_mode=False)
+            headers.update(extra_headers)
+            headers.update({'referer': generate_convincing_referer(url)})
+
+        return headers
+
+    def _prepare_response(self, response: httpxResponse) -> Response:
+        """Takes httpx response and generates `Response` object from it.
+
+        :param response: httpx response object
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        return Response(
+            url=str(response.url),
+            text=response.text,
+            content=response.content,
+            status=response.status_code,
+            reason=response.reason_phrase,
+            encoding=response.encoding or 'utf-8',
+            cookies=dict(response.cookies),
+            headers=dict(response.headers),
+            request_headers=dict(response.request.headers),
+            adaptor_arguments=self.adaptor_arguments
+        )
+
+    def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP GET request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)
+
+    def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP POST request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)
+
+    def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP DELETE request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)
+
+    def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
+        """Make basic HTTP PUT request for you but with some added flavors.
+        :param url: Target url.
+        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+            create a referer header as if this request had came from Google's search of this URL's domain.
+        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+        :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
+        """
+        headers = self._headers_job(kwargs.get('headers'), url, stealthy_headers)
+        request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
+        return self._prepare_response(request)

scrapling/engines/toolbelt/__init__.py

@@ -0,0 +1,18 @@
+from .fingerprints import (
+    get_os_name,
+    generate_headers,
+    generate_convincing_referer,
+)
+from .custom import (
+    Response,
+    do_nothing,
+    BaseFetcher,
+    get_variable_name,
+    check_type_validity,
+    check_if_engine_usable,
+)
+from .navigation import (
+    js_bypass_path,
+    intercept_route,
+    construct_cdp_url,
+)

scrapling/engines/toolbelt/custom.py

@@ -0,0 +1,168 @@
+"""
+Functions related to custom types or type checking
+"""
+import inspect
+import logging
+from dataclasses import dataclass, field
+
+from scrapling.core.utils import setup_basic_logging
+from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
+
+
+@dataclass(frozen=True)
+class Response:
+    """This class is returned by all engines as a way to unify response type between different libraries."""
+    url: str
+    text: str
+    content: bytes
+    status: int
+    reason: str
+    encoding: str = 'utf-8'  # default encoding
+    cookies: Dict = field(default_factory=dict)
+    headers: Dict = field(default_factory=dict)
+    request_headers: Dict = field(default_factory=dict)
+    adaptor_arguments: Dict = field(default_factory=dict)
+
+    @property
+    def adaptor(self) -> Union[Adaptor, None]:
+        """Generate Adaptor instance from this response if possible, otherwise return None"""
+        automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
+        if self.text:
+            # For playwright that will be the response after all JS executed
+            return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.content:
+            # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
+            # To get response Bytes after the load states
+            # Reference: https://playwright.dev/python/docs/api/class-page
+            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+        return None
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
+
+
+class BaseFetcher:
+    def __init__(
+            self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
+            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
+            automatch_domain: Optional[str] = None,
+    ):
+        """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
+        are detected and passed automatically from the Fetcher based on the response for accessibility.
+
+        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
+            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
+        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
+            priority over all auto-match related arguments/functions in the class.
+        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
+        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
+            If empty, default values will be used.
+        :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
+            Otherwise, the domain of the request is used by default.
+        :param debug: Enable debug mode
+        """
+        # Adaptor class parameters
+        # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
+        self.adaptor_arguments = dict(
+            huge_tree=huge_tree,
+            keep_comments=keep_comments,
+            auto_match=auto_match,
+            storage=storage,
+            storage_args=storage_args,
+            debug=debug,
+        )
+        # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
+        setup_basic_logging(level='debug' if debug else 'info')
+        if automatch_domain:
+            if type(automatch_domain) is not str:
+                logging.warning('[Ignored] The argument "automatch_domain" must be of string type')
+            else:
+                self.adaptor_arguments.update({'automatch_domain': automatch_domain})
+
+
+def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
+    """This function check if the passed engine can be used by a Fetcher-type class or not.
+
+    :param engine: The engine class itself
+    :return: The engine class again if all checks out, otherwise raises error
+    :raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
+    """
+    # if isinstance(engine, type):
+    #     raise TypeError("Expected an engine instance, not a class definition of the engine")
+
+    if hasattr(engine, 'fetch'):
+        fetch_function = getattr(engine, "fetch")
+        if callable(fetch_function):
+            if len(inspect.signature(fetch_function).parameters) > 0:
+                return engine
+            else:
+                # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
+                raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
+        else:
+            # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
+            raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
+    else:
+        # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
+        raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
+
+
+def get_variable_name(var: Any) -> Optional[str]:
+    """Get the name of a variable using global and local scopes.
+    :param var: The variable to find the name for
+    :return: The name of the variable if found, None otherwise
+    """
+    for scope in [globals(), locals()]:
+        for name, value in scope.items():
+            if value is var:
+                return name
+    return None
+
+
+def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
+    """Check if a variable matches the specified type constraints.
+    :param variable: The variable to check
+    :param valid_types: List of valid types for the variable
+    :param default_value: Value to return if type check fails
+    :param critical: If True, raises TypeError instead of logging error
+    :param param_name: Optional parameter name for error messages
+    :return: The original variable if valid, default_value if invalid
+    :raise TypeError: If critical=True and type check fails
+    """
+    # Use provided param_name or try to get it automatically
+    var_name = param_name or get_variable_name(variable) or "Unknown"
+
+    # Convert valid_types to a list if None
+    valid_types = valid_types or []
+
+    # Handle None value
+    if variable is None:
+        if type(None) in valid_types:
+            return variable
+        error_msg = f'Argument "{var_name}" cannot be None'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    # If no valid_types specified and variable has a value, return it
+    if not valid_types:
+        return variable
+
+    # Check if variable type matches any of the valid types
+    if not any(isinstance(variable, t) for t in valid_types):
+        type_names = [t.__name__ for t in valid_types]
+        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
+        if critical:
+            raise TypeError(error_msg)
+        logging.error(f'[Ignored] {error_msg}')
+        return default_value
+
+    return variable
+
+
+# Pew Pew
+def do_nothing(page):
+    # Just works as a filler for `page_action` argument in browser engines
+    return page

scrapling/engines/toolbelt/fingerprints.py

@@ -0,0 +1,81 @@
+"""
+Functions related to generating headers and fingerprints generally
+"""
+
+import platform
+
+from scrapling.core.utils import cache
+from scrapling.core._types import Union, Dict
+
+from tldextract import extract
+from browserforge.headers import HeaderGenerator, Browser
+from browserforge.fingerprints import FingerprintGenerator, Fingerprint
+
+
+@cache(None, typed=True)
+def generate_convincing_referer(url: str) -> str:
+    """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
+
+    >>> generate_convincing_referer('https://www.somewebsite.com/blah')
+    'https://www.google.com/search?q=somewebsite'
+
+    :param url: The URL you are about to fetch.
+    :return: Google's search URL of the domain name
+    """
+    website_name = extract(url).domain
+    return f'https://www.google.com/search?q={website_name}'
+
+
+@cache(None, typed=True)
+def get_os_name() -> Union[str, None]:
+    """Get the current OS name in the same format needed for browserforge
+
+    :return: Current OS name or `None` otherwise
+    """
+    #
+    os_name = platform.system()
+    return {
+        'Linux': 'linux',
+        'Darwin': 'macos',
+        'Windows': 'windows',
+        # For the future? because why not
+        'iOS': 'ios',
+    }.get(os_name)
+
+
+def generate_suitable_fingerprint() -> Fingerprint:
+    """Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.
+
+    This function was originally created to test Browserforge's injector.
+    :return: `Fingerprint` object
+    """
+    return FingerprintGenerator(
+        browser=[Browser(name='chrome', min_version=128)],
+        os=get_os_name(),  # None is ignored
+        device='desktop'
+    ).generate()
+
+
+def generate_headers(browser_mode: bool = False) -> Dict:
+    """Generate real browser-like headers using browserforge's generator
+
+    :param browser_mode: If enabled, the headers created are used for playwright so it have to match everything
+    :return: A dictionary of the generated headers
+    """
+    if browser_mode:
+        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
+        # So we don't raise any inconsistency red flags while websites fingerprinting us
+        os_name = get_os_name()
+        return HeaderGenerator(
+            browser=[Browser(name='chrome', min_version=128)],
+            os=os_name,  # None is ignored
+            device='desktop'
+        ).generate()
+    else:
+        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
+        browsers = [
+            Browser(name='chrome', min_version=120),
+            Browser(name='firefox', min_version=120),
+            Browser(name='edge', min_version=120),
+        ]
+        return HeaderGenerator(browser=browsers, device='desktop').generate()