scrapling 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl

scrapling/__init__.py

@@ -1,28 +1,38 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3.5"
+__version__ = "0.3.6"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
+from typing import Any, TYPE_CHECKING
 
-# A lightweight approach to create a lazy loader for each import for backward compatibility
-# This will reduces initial memory footprint significantly (only loads what's used)
-def __getattr__(name):
-    lazy_imports = {
-        "Fetcher": ("scrapling.fetchers", "Fetcher"),
-        "Selector": ("scrapling.parser", "Selector"),
-        "Selectors": ("scrapling.parser", "Selectors"),
-        "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
-        "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
-        "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
-        "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
-        "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
-    }
-
-    if name in lazy_imports:
-        module_path, class_name = lazy_imports[name]
+if TYPE_CHECKING:
+    from scrapling.parser import Selector, Selectors
+    from scrapling.core.custom_types import AttributesHandler, TextHandler
+    from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+
+
+# Lazy import mapping
+_LAZY_IMPORTS = {
+    "Fetcher": ("scrapling.fetchers", "Fetcher"),
+    "Selector": ("scrapling.parser", "Selector"),
+    "Selectors": ("scrapling.parser", "Selectors"),
+    "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
+    "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
+    "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
+    "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
+    "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
+}
+__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+
+
+def __getattr__(name: str) -> Any:
+    if name in _LAZY_IMPORTS:
+        module_path, class_name = _LAZY_IMPORTS[name]
         module = __import__(module_path, fromlist=[class_name])
         return getattr(module, class_name)
     else:
-        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
-__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+def __dir__() -> list[str]:
+    """Support for dir() and autocomplete."""
+    return sorted(__all__ + ["fetchers", "parser", "cli", "core", "__author__", "__version__", "__copyright__"])

scrapling/cli.py

@@ -2,8 +2,9 @@ from pathlib import Path
 from subprocess import check_output
 from sys import executable as python_executable
 
+from scrapling.core.utils import log
 from scrapling.engines.toolbelt.custom import Response
-from scrapling.core.utils import log, _CookieParser, _ParseHeaders
+from scrapling.core.utils._shell import _CookieParser, _ParseHeaders
 from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable
 
 from orjson import loads as json_loads, JSONDecodeError
@@ -135,10 +136,26 @@ def install(force):  # pragma: no cover
 
 
 @command(help="Run Scrapling's MCP server (Check the docs for more info).")
-def mcp():
+@option(
+    "--http",
+    is_flag=True,
+    default=False,
+    help="Whether to run the MCP server in streamable-http transport or leave it as stdio (Default: False)",
+)
+@option(
+    "--host",
+    type=str,
+    default="0.0.0.0",
+    help="The host to use if streamable-http transport is enabled (Default: '0.0.0.0')",
+)
+@option(
+    "--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)"
+)
+def mcp(http, host, port):
     from scrapling.core.ai import ScraplingMCPServer
 
-    ScraplingMCPServer().serve()
+    server = ScraplingMCPServer()
+    server.serve(http, host, port)
 
 
 @command(help="Interactive scraping console")
@@ -766,7 +783,7 @@ def stealthy_fetch(
     :param disable_resources: Drop requests of unnecessary resources for a speed boost.
     :param block_webrtc: Blocks WebRTC entirely.
     :param humanize: Humanize the cursor movement.
-    :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page.
+    :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
     :param allow_webgl: Allow WebGL (recommended to keep enabled).
     :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
     :param disable_ads: Install the uBlock Origin addon on the browser.

scrapling/core/_types.py

@@ -39,6 +39,4 @@ except ImportError:  # pragma: no cover
     try:
         from typing_extensions import Self  # Backport
     except ImportError:
-        from typing import TypeVar
-
         Self = object

scrapling/core/ai.py

@@ -42,10 +42,7 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
 
 
 class ScraplingMCPServer:
-    _server = FastMCP(name="Scrapling")
-
     @staticmethod
-    @_server.tool()
     def get(
         url: str,
         impersonate: Optional[BrowserTypeLiteral] = "chrome",
@@ -124,7 +121,6 @@ class ScraplingMCPServer:
         )
 
     @staticmethod
-    @_server.tool()
     async def bulk_get(
         urls: Tuple[str, ...],
         impersonate: Optional[BrowserTypeLiteral] = "chrome",
@@ -211,7 +207,6 @@ class ScraplingMCPServer:
             ]
 
     @staticmethod
-    @_server.tool()
     async def fetch(
         url: str,
         extraction_type: extraction_types = "markdown",
@@ -263,7 +258,7 @@ class ScraplingMCPServer:
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -300,7 +295,6 @@ class ScraplingMCPServer:
         )
 
     @staticmethod
-    @_server.tool()
     async def bulk_fetch(
         urls: Tuple[str, ...],
         extraction_type: extraction_types = "markdown",
@@ -352,7 +346,7 @@ class ScraplingMCPServer:
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -394,7 +388,6 @@ class ScraplingMCPServer:
             ]
 
     @staticmethod
-    @_server.tool()
     async def stealthy_fetch(
         url: str,
         extraction_type: extraction_types = "markdown",
@@ -443,7 +436,7 @@ class ScraplingMCPServer:
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
@@ -494,7 +487,6 @@ class ScraplingMCPServer:
         )
 
     @staticmethod
-    @_server.tool()
     async def bulk_stealthy_fetch(
         urls: Tuple[str, ...],
         extraction_type: extraction_types = "markdown",
@@ -543,7 +535,7 @@ class ScraplingMCPServer:
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
@@ -598,6 +590,22 @@ class ScraplingMCPServer:
                 for page in responses
             ]
 
-    def serve(self):
+    def serve(self, http: bool, host: str, port: int):
         """Serve the MCP server."""
-        self._server.run(transport="stdio")
+        server = FastMCP(name="Scrapling", host=host, port=port)
+        server.add_tool(self.get, title="get", description=self.get.__doc__, structured_output=True)
+        server.add_tool(self.bulk_get, title="bulk_get", description=self.bulk_get.__doc__, structured_output=True)
+        server.add_tool(self.fetch, title="fetch", description=self.fetch.__doc__, structured_output=True)
+        server.add_tool(
+            self.bulk_fetch, title="bulk_fetch", description=self.bulk_fetch.__doc__, structured_output=True
+        )
+        server.add_tool(
+            self.stealthy_fetch, title="stealthy_fetch", description=self.stealthy_fetch.__doc__, structured_output=True
+        )
+        server.add_tool(
+            self.bulk_stealthy_fetch,
+            title="bulk_stealthy_fetch",
+            description=self.bulk_stealthy_fetch.__doc__,
+            structured_output=True,
+        )
+        server.run(transport="stdio" if not http else "streamable-http")

scrapling/core/shell.py

@@ -22,10 +22,11 @@ from logging import (
 from orjson import loads as json_loads, JSONDecodeError
 
 from scrapling import __version__
+from scrapling.core.utils import log
 from scrapling.parser import Selector, Selectors
 from scrapling.core.custom_types import TextHandler
 from scrapling.engines.toolbelt.custom import Response
-from scrapling.core.utils import log, _ParseHeaders, _CookieParser
+from scrapling.core.utils._shell import _ParseHeaders, _CookieParser
 from scrapling.core._types import (
     Optional,
     Dict,

scrapling/core/storage.py

@@ -6,7 +6,6 @@ from sqlite3 import connect as db_connect
 
 from orjson import dumps, loads
 from lxml.html import HtmlElement
-from tldextract import extract as tld
 
 from scrapling.core.utils import _StorageTools, log
 from scrapling.core._types import Dict, Optional, Any
@@ -26,6 +25,8 @@ class StorageSystemMixin(ABC):  # pragma: no cover
             return default_value
 
         try:
+            from tldextract import extract as tld
+
             extracted = tld(self.url)
             return extracted.top_domain_under_public_suffix or extracted.domain or default_value
         except AttributeError:

scrapling/core/utils/__init__.py

@@ -7,4 +7,3 @@ from ._utils import (
     clean_spaces,
     html_forbidden,
 )
-from ._shell import _CookieParser, _ParseHeaders

scrapling/engines/_browsers/__init__.py

@@ -1,2 +0,0 @@
-from ._controllers import DynamicSession, AsyncDynamicSession
-from ._camoufox import StealthySession, AsyncStealthySession

scrapling/engines/_browsers/_base.py

@@ -12,17 +12,13 @@ from camoufox.utils import (
     installed_verstr as camoufox_version,
 )
 
-from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
-from scrapling.core._types import (
-    Any,
-    Dict,
-    Optional,
-)
 from ._page import PageInfo, PagePool
-from ._config_tools import _compiled_stealth_scripts
-from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.parser import Selector
+from scrapling.core._types import Dict, Optional
 from scrapling.engines.toolbelt.fingerprints import get_os_name
 from ._validators import validate, PlaywrightConfig, CamoufoxConfig
+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
 
 __ff_version_str__ = camoufox_version().split(".", 1)[0]
 
@@ -268,4 +264,9 @@ class StealthySessionMixin:
             if f"cType: '{ctype}'" in page_content:
                 return ctype
 
+        # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)
+        selector = Selector(content=page_content)
+        if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):
+            return "embedded"
+
         return None

scrapling/engines/_browsers/_camoufox.py

@@ -116,7 +116,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
@@ -237,26 +237,33 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 return
 
             else:
-                while "Verifying you are human." in self._get_page_content(page):
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
-                    page.wait_for_timeout(500)
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+                if challenge_type != "embedded":
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in self._get_page_content(page):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        page.wait_for_timeout(500)
 
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
+                    log.error("Didn't find Cloudflare iframe!")
                     return
 
-                while not iframe.frame_element().is_visible():
-                    # Double-checking that the iframe is loaded
-                    page.wait_for_timeout(500)
+                if challenge_type != "embedded":
+                    while not iframe.frame_element().is_visible():
+                        # Double-checking that the iframe is loaded
+                        page.wait_for_timeout(500)
 
+                iframe.wait_for_load_state(state="domcontentloaded")
+                iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = page.locator(".main-content p+div>div>div").bounding_box()
+                outer_box = page.locator(box_selector).last.bounding_box()
                 captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                page.locator(".zone-name-title").wait_for(state="hidden")
+                if challenge_type != "embedded":
+                    page.locator(".zone-name-title").wait_for(state="hidden")
                 page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
@@ -293,7 +300,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
@@ -435,7 +442,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param cookies: Set cookies for the next request.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
@@ -556,26 +563,33 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 return
 
             else:
-                while "Verifying you are human." in (await self._get_page_content(page)):
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
-                    await page.wait_for_timeout(500)
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+                if challenge_type != "embedded":
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in (await self._get_page_content(page)):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        await page.wait_for_timeout(500)
 
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
+                    log.error("Didn't find Cloudflare iframe!")
                     return
 
-                while not await (await iframe.frame_element()).is_visible():
-                    # Double-checking that the iframe is loaded
-                    await page.wait_for_timeout(500)
+                if challenge_type != "embedded":
+                    while not await (await iframe.frame_element()).is_visible():
+                        # Double-checking that the iframe is loaded
+                        await page.wait_for_timeout(500)
 
+                await iframe.wait_for_load_state(state="domcontentloaded")
+                await iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
+                outer_box = await page.locator(box_selector).last.bounding_box()
                 captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                await page.locator(".zone-name-title").wait_for(state="hidden")
+                if challenge_type != "embedded":
+                    await page.locator(".zone-name-title").wait_for(state="hidden")
                 await page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
@@ -612,7 +626,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
-        :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+        :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """

scrapling/engines/_browsers/_controllers.py

@@ -117,7 +117,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -360,7 +360,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
-        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+        :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.

scrapling/engines/constants.py

@@ -101,18 +101,3 @@ DEFAULT_STEALTH_FLAGS = (
     "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
     "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
 )
-
-# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
-NSTBROWSER_DEFAULT_QUERY = {
-    "once": True,
-    "headless": True,
-    "autoClose": True,
-    "fingerprint": {
-        "flags": {"timezone": "BasedOnIp", "screen": "Custom"},
-        "platform": "linux",  # support: windows, mac, linux
-        "kernel": "chromium",  # only support: chromium
-        "kernelMilestone": "128",
-        "hardwareConcurrency": 8,
-        "deviceMemory": 8,
-    },
-}