seleniumbase 4.32.1__py3-none-any.whl → 4.32.2__py3-none-any.whl

seleniumbase/undetected/cdp_driver/cdp_util.py

@@ -0,0 +1,317 @@
+"""CDP-Driver is based on NoDriver"""
+from __future__ import annotations
+import asyncio
+import logging
+import time
+import types
+import typing
+from typing import Optional, List, Union, Callable
+from .element import Element
+from .browser import Browser
+from .browser import PathLike
+from .config import Config
+from .tab import Tab
+import mycdp as cdp
+
+logger = logging.getLogger(__name__)
+T = typing.TypeVar("T")
+
+
+async def start(
+    config: Optional[Config] = None,
+    *,
+    user_data_dir: Optional[PathLike] = None,
+    headless: Optional[bool] = False,
+    incognito: Optional[bool] = False,
+    guest: Optional[bool] = False,
+    browser_executable_path: Optional[PathLike] = None,
+    browser_args: Optional[List[str]] = None,
+    sandbox: Optional[bool] = True,
+    lang: Optional[str] = None,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
+    expert: Optional[bool] = None,
+    **kwargs: Optional[dict],
+) -> Browser:
+    """
+    Helper function to launch a browser. It accepts several keyword parameters.
+    Conveniently, you can just call it bare (no parameters) to quickly launch
+    an instance with best practice defaults.
+    Note: Due to a Chrome-130 bug, use start_async or start_sync instead.
+     (Calling this method directly could lead to an unresponsive browser)
+    Note: New args are expected: Use kwargs only!
+    Note: This should be called ``await start()``
+    :param user_data_dir:
+    :type user_data_dir: PathLike
+    :param headless:
+    :type headless: bool
+    :param browser_executable_path:
+    :type browser_executable_path: PathLike
+    :param browser_args:
+     ["--some-chromeparam=somevalue", "some-other-param=someval"]
+    :type browser_args: List[str]
+    :param sandbox: Default True, but when set to False it adds --no-sandbox
+     to the params, also when using linux under a root user,
+     it adds False automatically (else Chrome won't start).
+    :type sandbox: bool
+    :param lang: language string
+    :type lang: str
+    :param port: If you connect to an existing debuggable session,
+     you can specify the port here.
+     If both host and port are provided,
+     then a local Chrome browser will not be started!
+    :type port: int
+    :param host: If you connect to an existing debuggable session,
+     you can specify the host here.
+     If both host and port are provided,
+     then a local Chrome browser will not be started!
+    :type host: str
+    :param expert: When set to True, "expert" mode is enabled.
+     This means adding: --disable-web-security --disable-site-isolation-trials,
+     as well as some scripts and patching useful for debugging.
+     (For example, ensuring shadow-root is always in "open" mode.)
+    :type expert: bool
+    """
+    if not config:
+        config = Config(
+            user_data_dir,
+            headless,
+            incognito,
+            guest,
+            browser_executable_path,
+            browser_args,
+            sandbox,
+            lang,
+            host=host,
+            port=port,
+            expert=expert,
+            **kwargs,
+        )
+    return await Browser.create(config)
+
+
+async def start_async(*args, **kwargs) -> Browser:
+    headless = False
+    if "headless" in kwargs:
+        headless = kwargs["headless"]
+    decoy_args = kwargs
+    decoy_args["headless"] = True
+    driver = await start(**decoy_args)
+    kwargs["headless"] = headless
+    kwargs["user_data_dir"] = driver.config.user_data_dir
+    driver.stop()  # Due to Chrome-130, must stop & start
+    time.sleep(0.15)
+    return await start(*args, **kwargs)
+
+
+def start_sync(*args, **kwargs) -> Browser:
+    loop = asyncio.get_event_loop()
+    headless = False
+    if "headless" in kwargs:
+        headless = kwargs["headless"]
+    decoy_args = kwargs
+    decoy_args["headless"] = True
+    driver = loop.run_until_complete(start(**decoy_args))
+    kwargs["headless"] = headless
+    kwargs["user_data_dir"] = driver.config.user_data_dir
+    driver.stop()  # Due to Chrome-130, must stop & start
+    time.sleep(0.15)
+    return loop.run_until_complete(start(*args, **kwargs))
+
+
+async def create_from_driver(driver) -> Browser:
+    """Create a Browser instance from a running driver instance."""
+    from .config import Config
+
+    conf = Config()
+    host, port = driver.options.debugger_address.split(":")
+    conf.host, conf.port = host, int(port)
+    # Create Browser instance
+    browser = await start(conf)
+    browser._process_pid = driver.browser_pid
+    # Stop chromedriver binary
+    driver.service.stop()
+    driver.browser_pid = -1
+    driver.user_data_dir = None
+    return browser
+
+
+def free_port() -> int:
+    """Determines a free port using sockets."""
+    import socket
+
+    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    free_socket.bind(("127.0.0.1", 0))
+    free_socket.listen(5)
+    port: int = free_socket.getsockname()[1]
+    free_socket.close()
+    return port
+
+
+def filter_recurse_all(
+    doc: T, predicate: Callable[[cdp.dom.Node, Element], bool]
+) -> List[T]:
+    """
+    Test each child using predicate(child),
+    and return all children for which predicate(child) == True
+    :param doc: The cdp.dom.Node object or :py:class:`cdp_driver.Element`
+    :param predicate: A function which takes a node as first parameter
+     and returns a boolean, where True means include.
+    """
+    if not hasattr(doc, "children"):
+        raise TypeError("Object should have a .children attribute!")
+    out = []
+    if doc and doc.children:
+        for child in doc.children:
+            if predicate(child):
+                out.append(child)
+            if child.shadow_roots is not None:
+                out.extend(
+                    filter_recurse_all(child.shadow_roots[0], predicate)
+                )
+            out.extend(filter_recurse_all(child, predicate))
+    return out
+
+
+def filter_recurse(
+    doc: T, predicate: Callable[[cdp.dom.Node, Element], bool]
+) -> T:
+    """
+    Test each child using predicate(child),
+    and return the first child of which predicate(child) == True
+    :param doc: the cdp.dom.Node object or :py:class:`cdp_driver.Element`
+    :param predicate: a function which takes a node as first parameter
+     and returns a boolean, where True means include.
+    """
+    if not hasattr(doc, "children"):
+        raise TypeError("Object should have a .children attribute!")
+    if doc and doc.children:
+        for child in doc.children:
+            if predicate(child):
+                return child
+            if child.shadow_roots:
+                shadow_root_result = filter_recurse(
+                    child.shadow_roots[0], predicate
+                )
+                if shadow_root_result:
+                    return shadow_root_result
+            result = filter_recurse(child, predicate)
+            if result:
+                return result
+
+
+def circle(
+    x, y=None, radius=10, num=10, dir=0
+) -> typing.Generator[typing.Tuple[float, float], None, None]:
+    """
+    A generator will calculate coordinates around a circle.
+    :param x: start x position
+    :type x: int
+    :param y: start y position
+    :type y: int
+    :param radius: size of the circle
+    :type radius: int
+    :param num: the amount of points calculated
+     (higher => slower, more cpu, but more detailed)
+    :type num: int
+    """
+    import math
+
+    r = radius
+    w = num
+    if not y:
+        y = x
+    a = int(x - r * 2)
+    b = int(y - r * 2)
+    m = (2 * math.pi) / w
+    if dir == 0:
+        # Regular direction
+        ran = 0, w + 1, 1
+    else:
+        # Opposite direction
+        ran = w + 1, 0, -1
+    for i in range(*ran):
+        x = a + r * math.sin(m * i)
+        y = b + r * math.cos(m * i)
+        yield x, y
+
+
+def remove_from_tree(tree: cdp.dom.Node, node: cdp.dom.Node) -> cdp.dom.Node:
+    if not hasattr(tree, "children"):
+        raise TypeError("Object should have a .children attribute!")
+    if tree and tree.children:
+        for child in tree.children:
+            if child.backend_node_id == node.backend_node_id:
+                tree.children.remove(child)
+            remove_from_tree(child, node)
+    return tree
+
+
+async def html_from_tree(
+    tree: Union[cdp.dom.Node, Element], target: Tab
+):
+    if not hasattr(tree, "children"):
+        raise TypeError("Object should have a .children attribute!")
+    out = ""
+    if tree and tree.children:
+        for child in tree.children:
+            if isinstance(child, Element):
+                out += await child.get_html()
+            else:
+                out += await target.send(
+                    cdp.dom.get_outer_html(
+                        backend_node_id=child.backend_node_id
+                    )
+                )
+            out += await html_from_tree(child, target)
+    return out
+
+
+def compare_target_info(
+    info1: cdp.target.TargetInfo, info2: cdp.target.TargetInfo
+) -> List[typing.Tuple[str, typing.Any, typing.Any]]:
+    """
+    When logging mode is set to debug, browser object will log when target info
+    is changed. To provide more meaningful log messages,
+    this function is called to check what has actually changed
+    between the 2 (by simple dict comparison).
+    It returns a list of tuples
+        [ ... ( key_which_has_changed, old_value, new_value) ]
+    :param info1:
+    :param info2:
+    """
+    d1 = info1.__dict__
+    d2 = info2.__dict__
+    return [(k, v, d2[k]) for (k, v) in d1.items() if d2[k] != v]
+
+
+def loop():
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    return loop
+
+
+def cdp_get_module(domain: Union[str, types.ModuleType]):
+    """
+    Get cdp module by given string.
+    :param domain:
+    """
+    import importlib
+
+    if isinstance(domain, types.ModuleType):
+        domain_mod = domain
+    else:
+        try:
+            if domain in ("input",):
+                domain = "input_"
+            domain_mod = getattr(cdp, domain)
+            if not domain_mod:
+                raise AttributeError
+        except AttributeError:
+            try:
+                domain_mod = importlib.import_module(domain)
+            except ModuleNotFoundError:
+                raise ModuleNotFoundError(
+                    "Could not find cdp module from input '%s'" % domain
+                )
+    return domain_mod

seleniumbase/undetected/cdp_driver/config.py

@@ -0,0 +1,322 @@
+import logging
+import os
+import pathlib
+import secrets
+import sys
+import tempfile
+import zipfile
+from typing import Union, List, Optional
+
+__all__ = [
+    "Config",
+    "find_chrome_executable",
+    "temp_profile_dir",
+    "is_root",
+    "is_posix",
+    "PathLike",
+]
+
+logger = logging.getLogger(__name__)
+is_posix = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
+
+PathLike = Union[str, pathlib.Path]
+AUTO = None
+
+
+class Config:
+    """Config object"""
+
+    def __init__(
+        self,
+        user_data_dir: Optional[PathLike] = AUTO,
+        headless: Optional[bool] = False,
+        incognito: Optional[bool] = False,
+        guest: Optional[bool] = False,
+        browser_executable_path: Optional[PathLike] = AUTO,
+        browser_args: Optional[List[str]] = AUTO,
+        sandbox: Optional[bool] = True,
+        lang: Optional[str] = "en-US",
+        host: str = AUTO,
+        port: int = AUTO,
+        expert: bool = AUTO,
+        **kwargs: dict,
+    ):
+        """
+        Creates a config object.
+        Can be called without any arguments to generate a best-practice config,
+        which is recommended.
+        Calling the object, eg: myconfig(), returns the list of arguments which
+        are provided to the browser.
+        Additional args can be added using the :py:obj:`~add_argument method`.
+        Instances of this class are usually not instantiated by end users.
+        :param user_data_dir: the data directory to use
+        :param headless: set to True for headless mode
+        :param browser_executable_path:
+         Specify browser executable, instead of using autodetect.
+        :param browser_args: Forwarded to browser executable.
+         Eg: ["--some-chromeparam=somevalue", "some-other-param=someval"]
+        :param sandbox: disables sandbox
+        :param autodiscover_targets: use autodiscovery of targets
+        :param lang:
+         Language string to use other than the default "en-US,en;q=0.9"
+        :param expert: When set to True, "expert" mode is enabled.
+         This adds: --disable-web-security --disable-site-isolation-trials,
+         as well as some scripts and patching useful for debugging.
+         (For example, ensuring shadow-root is always in "open" mode.)
+        :param kwargs:
+        :type user_data_dir: PathLike
+        :type headless: bool
+        :type browser_executable_path: PathLike
+        :type browser_args: list[str]
+        :type sandbox: bool
+        :type lang: str
+        :type kwargs: dict
+        """
+        if not browser_args:
+            browser_args = []
+        if not user_data_dir:
+            self._user_data_dir = temp_profile_dir()
+            self._custom_data_dir = False
+        else:
+            self.user_data_dir = user_data_dir
+        if not browser_executable_path:
+            browser_executable_path = find_chrome_executable()
+        self._browser_args = browser_args
+        self.browser_executable_path = browser_executable_path
+        self.headless = headless
+        self.incognito = incognito
+        self.guest = guest
+        self.sandbox = sandbox
+        self.host = host
+        self.port = port
+        self.expert = expert
+        self._extensions = []
+        # When using posix-ish operating system and running as root,
+        # you must use no_sandbox=True
+        if is_posix and is_root() and sandbox:
+            logger.info("Detected root usage, auto-disabling sandbox mode.")
+            self.sandbox = False
+        self.autodiscover_targets = True
+        self.lang = lang
+        # Other keyword args will be accessible by attribute
+        self.__dict__.update(kwargs)
+        super().__init__()
+        self._default_browser_args = [
+            "--remote-allow-origins=*",
+            "--no-first-run",
+            "--no-service-autorun",
+            "--no-default-browser-check",
+            "--homepage=about:blank",
+            "--no-pings",
+            "--safebrowsing-disable-download-protection",
+            '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"',
+            "--password-store=basic",
+            "--deny-permission-prompts",
+            "--disable-infobars",
+            "--disable-breakpad",
+            "--disable-component-update",
+            "--disable-prompt-on-repost",
+            "--disable-password-generation",
+            "--disable-ipc-flooding-protection",
+            "--disable-search-engine-choice-screen",
+            "--disable-backgrounding-occluded-windows",
+            "--disable-client-side-phishing-detection",
+            "--disable-top-sites",
+            "--disable-renderer-backgrounding",
+            "--disable-background-networking",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process,Translate,"
+            "InsecureDownloadWarnings,DownloadBubble,DownloadBubbleV2,"
+            "OptimizationTargetPrediction,OptimizationGuideModelDownloading,"
+            "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4",
+        ]
+
+    @property
+    def browser_args(self):
+        return sorted(self._default_browser_args + self._browser_args)
+
+    @property
+    def user_data_dir(self):
+        return self._user_data_dir
+
+    @user_data_dir.setter
+    def user_data_dir(self, path: PathLike):
+        self._user_data_dir = str(path)
+        self._custom_data_dir = True
+
+    @property
+    def uses_custom_data_dir(self) -> bool:
+        return self._custom_data_dir
+
+    def add_extension(self, extension_path: PathLike):
+        """
+        Adds an extension to load. You can set the extension_path to a
+        folder (containing the manifest), or an extension zip file (.crx)
+        :param extension_path:
+        """
+        path = pathlib.Path(extension_path)
+        if not path.exists():
+            raise FileNotFoundError(
+                "Could not find anything here: %s" % str(path)
+            )
+        if path.is_file():
+            tf = tempfile.mkdtemp(
+                prefix="extension_", suffix=secrets.token_hex(4)
+            )
+            with zipfile.ZipFile(path, "r") as z:
+                z.extractall(tf)
+                self._extensions.append(tf)
+        elif path.is_dir():
+            for item in path.rglob("manifest.*"):
+                path = item.parent
+            self._extensions.append(path)
+
+    def __call__(self):
+        # The host and port will be added when starting the browser.
+        # By the time it starts, the port is probably already taken.
+        args = self._default_browser_args.copy()
+        args += ["--user-data-dir=%s" % self.user_data_dir]
+        args += ["--disable-features=IsolateOrigins,site-per-process"]
+        args += ["--disable-session-crashed-bubble"]
+        if self.expert:
+            args += [
+                "--disable-web-security",
+                "--disable-site-isolation-trials",
+            ]
+        if self._browser_args:
+            args.extend([arg for arg in self._browser_args if arg not in args])
+        if self.headless:
+            args.append("--headless=new")
+        if self.incognito:
+            args.append("--incognito")
+        if self.guest:
+            args.append("--guest")
+        if not self.sandbox:
+            args.append("--no-sandbox")
+        if self.host:
+            args.append("--remote-debugging-host=%s" % self.host)
+        if self.port:
+            args.append("--remote-debugging-port=%s" % self.port)
+        return args
+
+    def add_argument(self, arg: str):
+        if any(
+            x in arg.lower()
+            for x in [
+                "headless",
+                "data-dir",
+                "data_dir",
+                "no-sandbox",
+                "no_sandbox",
+                "lang",
+            ]
+        ):
+            raise ValueError(
+                '"%s" is not allowed. Please use one of the '
+                'attributes of the Config object to set it.'
+                % arg
+            )
+        self._browser_args.append(arg)
+
+    def __repr__(self):
+        s = f"{self.__class__.__name__}"
+        for k, v in ({**self.__dict__, **self.__class__.__dict__}).items():
+            if k[0] == "_":
+                continue
+            if not v:
+                continue
+            if isinstance(v, property):
+                v = getattr(self, k)
+            if callable(v):
+                continue
+            s += f"\n\t{k} = {v}"
+        return s
+
+
+def is_root():
+    """
+    Helper function to determine if the user is trying to launch chrome
+    under linux as root, which needs some alternative handling.
+    """
+    import ctypes
+    import os
+
+    try:
+        return os.getuid() == 0
+    except AttributeError:
+        return ctypes.windll.shell32.IsUserAnAdmin() != 0
+
+
+def temp_profile_dir():
+    """Generate a temp dir (path)"""
+    path = os.path.normpath(tempfile.mkdtemp(prefix="uc_"))
+    return path
+
+
+def find_chrome_executable(return_all=False):
+    """
+    Finds the chrome, beta, canary, chromium executable
+    and returns the disk path.
+    """
+    candidates = []
+    if is_posix:
+        for item in os.environ.get("PATH").split(os.pathsep):
+            for subitem in (
+                "google-chrome",
+                "chromium",
+                "chromium-browser",
+                "chrome",
+                "google-chrome-stable",
+            ):
+                candidates.append(os.sep.join((item, subitem)))
+        if "darwin" in sys.platform:
+            candidates += [
+                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "/Applications/Chromium.app/Contents/MacOS/Chromium",
+            ]
+    else:
+        for item in map(
+            os.environ.get,
+            (
+                "PROGRAMFILES",
+                "PROGRAMFILES(X86)",
+                "LOCALAPPDATA",
+                "PROGRAMW6432",
+            ),
+        ):
+            if item is not None:
+                for subitem in (
+                    "Google/Chrome/Application",
+                    "Google/Chrome Beta/Application",
+                    "Google/Chrome Canary/Application",
+                ):
+                    candidates.append(
+                        os.sep.join((item, subitem, "chrome.exe"))
+                    )
+    rv = []
+    for candidate in candidates:
+        if os.path.exists(candidate) and os.access(candidate, os.X_OK):
+            logger.debug("%s is a valid candidate... " % candidate)
+            rv.append(candidate)
+        else:
+            logger.debug(
+                "%s is not a valid candidate because it doesn't exist "
+                "or isn't an executable."
+                % candidate
+            )
+    winner = None
+    if return_all and rv:
+        return rv
+    if rv and len(rv) > 1:
+        # Assuming the shortest path wins
+        winner = min(rv, key=lambda x: len(x))
+    elif len(rv) == 1:
+        winner = rv[0]
+    if winner:
+        return os.path.normpath(winner)
+    raise FileNotFoundError(
+        "Could not find a valid chrome browser binary. "
+        "Please make sure Chrome is installed. "
+        "Or use the keyword argument: "
+        "'browser_executable_path=/path/to/your/browser'."
+    )