seleniumbase 4.41.3__py3-none-any.whl → 4.45.10__py3-none-any.whl

seleniumbase/undetected/cdp_driver/tab.py

@@ -4,9 +4,15 @@ import base64
 import datetime
 import logging
 import pathlib
+import re
 import urllib.parse
 import warnings
+from contextlib import suppress
+from filelock import FileLock
 from seleniumbase import config as sb_config
+from seleniumbase.fixtures import constants
+from seleniumbase.fixtures import js_utils
+from seleniumbase.fixtures import shared_utils
 from typing import Dict, List, Union, Optional, Tuple
 from . import browser as cdp_browser
 from . import element
@@ -244,23 +250,7 @@ class Tab(Connection):
          Raise timeout exception when after this many seconds nothing is found.
         :type timeout: float,int
         """
-        loop = asyncio.get_running_loop()
-        start_time = loop.time()
-        selector = selector.strip()
-        item = None
-        try:
-            item = await self.query_selector(selector)
-        except (Exception, TypeError):
-            pass
-        while not item:
-            await self
-            item = await self.query_selector(selector)
-            if loop.time() - start_time > timeout:
-                raise asyncio.TimeoutError(
-                    "Time ran out while waiting for: {%s}" % selector
-                )
-            await self.sleep(0.5)
-        return item
+        return await self.wait_for(selector=selector, timeout=timeout)
 
     async def find_all(
         self,
@@ -358,13 +348,19 @@ class Tab(Connection):
         if new_window and not new_tab:
             new_tab = True
         if new_tab:
-            if hasattr(sb_config, "incognito") and sb_config.incognito:
+            if (
+                getattr(sb_config, "incognito", None)
+                or (
+                    getattr(sb_config, "_cdp_browser", None)
+                    in ["comet", "atlas"]
+                )
+            ):
                 return await self.browser.get(
                     url, new_tab=False, new_window=True, **kwargs
                 )
             else:
                 return await self.browser.get(
-                    url, new_tab, new_window, **kwargs
+                    url, new_tab=True, new_window=False, **kwargs
                 )
         else:
             if not kwargs:
@@ -375,9 +371,12 @@ class Tab(Connection):
                 return self
             else:
                 return await self.browser.get(
-                    url, new_tab, new_window, **kwargs
+                    url, new_tab=False, new_window=False, **kwargs
                 )
 
+    async def open(self, url="about:blank"):
+        return await self.get(url=url)
+
     async def query_selector_all(
         self,
         selector: str,
@@ -705,10 +704,12 @@ class Tab(Connection):
             raise ProtocolException(errors)
         if remote_object:
             if return_by_value:
-                if remote_object.value:
+                if remote_object.value is not None:
                     return remote_object.value
             else:
-                return remote_object, errors
+                if remote_object.deep_serialized_value is not None:
+                    return remote_object.deep_serialized_value.value
+        return None
 
     async def js_dumps(
         self, obj_name: str, return_by_value: Optional[bool] = True
@@ -899,7 +900,10 @@ class Tab(Connection):
         """Gets the current page source content (html)"""
         doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
         return await self.send(
-            cdp.dom.get_outer_html(backend_node_id=doc.backend_node_id)
+            cdp.dom.get_outer_html(
+                backend_node_id=doc.backend_node_id,
+                include_shadow_dom=True,
+            )
         )
 
     async def maximize(self):
@@ -1075,7 +1079,7 @@ class Tab(Connection):
                     raise asyncio.TimeoutError(
                         "Time ran out while waiting for: {%s}" % selector
                     )
-                await self.sleep(0.5)
+                await self.sleep(0.068)
             return item
         if text:
             item = await self.find_element_by_text(text)
@@ -1085,9 +1089,42 @@ class Tab(Connection):
                     raise asyncio.TimeoutError(
                         "Time ran out while waiting for: {%s}" % text
                     )
-                await self.sleep(0.5)
+                await self.sleep(0.068)
             return item
 
+    async def set_attributes(self, selector, attribute, value):
+        """This method uses JavaScript to set/update a common attribute.
+        All matching selectors from querySelectorAll() are used.
+        Example => (Make all links on a website redirect to Google):
+        self.set_attributes("a", "href", "https://google.com")"""
+        attribute = re.escape(attribute)
+        attribute = js_utils.escape_quotes_if_needed(attribute)
+        value = re.escape(value)
+        value = js_utils.escape_quotes_if_needed(value)
+        if selector.startswith(("/", "./", "(")):
+            with suppress(Exception):
+                selector = js_utils.convert_to_css_selector(selector, "xpath")
+        css_selector = selector
+        css_selector = re.escape(css_selector)  # Add "\\" to special chars
+        css_selector = js_utils.escape_quotes_if_needed(css_selector)
+        js_code = (
+            """var $elements = document.querySelectorAll('%s');
+            var index = 0, length = $elements.length;
+            for(; index < length; index++){
+            $elements[index].setAttribute('%s','%s');}""" % (
+                css_selector,
+                attribute,
+                value,
+            )
+        )
+        with suppress(Exception):
+            await self.evaluate(js_code)
+
+    async def internalize_links(self):
+        """All `target="_blank"` links become `target="_self"`.
+        This prevents those links from opening in a new tab."""
+        await self.set_attributes('[target="_blank"]', "target", "_self")
+
     async def download_file(
         self, url: str, filename: Optional[PathLike] = None
     ):
@@ -1287,19 +1324,355 @@ class Tab(Connection):
                         res.append(abs_url)
         return res
 
-    async def verify_cf(self):
-        """(An attempt)"""
-        checkbox = None
-        checkbox_sibling = await self.wait_for(text="verify you are human")
-        if checkbox_sibling:
-            parent = checkbox_sibling.parent
-            while parent:
-                checkbox = await parent.query_selector("input[type=checkbox]")
-                if checkbox:
-                    break
-                parent = parent.parent
-        await checkbox.mouse_move()
-        await checkbox.mouse_click()
+    async def get_html(self):
+        element = await self.find("html", timeout=1)
+        return await element.get_html_async()
+
+    async def get_page_source(self):
+        return await self.get_html()
+
+    async def is_element_present(self, selector):
+        try:
+            await self.select(selector, timeout=0.01)
+            return True
+        except Exception:
+            return False
+
+    async def is_element_visible(self, selector):
+        try:
+            element = await self.select(selector, timeout=0.01)
+        except Exception:
+            return False
+        if not element:
+            return False
+        try:
+            position = await element.get_position_async()
+            return (position.width != 0 or position.height != 0)
+        except Exception:
+            return False
+
+    async def __on_a_cf_turnstile_page(self, source=None):
+        if not source or len(source) < 400:
+            await self.sleep(0.22)
+            source = await self.get_html()
+        if (
+            (
+                'data-callback="onCaptchaSuccess"' in source
+                and 'title="reCAPTCHA"' not in source
+                and 'id="recaptcha-token"' not in source
+            )
+            or "/challenge-platform/h/b/" in source
+            or 'id="challenge-widget-' in source
+            or "challenges.cloudf" in source
+            or "cf-turnstile-" in source
+        ):
+            return True
+        return False
+
+    async def __on_a_g_recaptcha_page(self, *args, **kwargs):
+        await self.sleep(0.4)  # reCAPTCHA may need a moment to appear
+        source = await self.get_html()
+        if (
+            (
+                'id="recaptcha-token"' in source
+                or 'title="reCAPTCHA"' in source
+            )
+            and await self.is_element_present('iframe[title="reCAPTCHA"]')
+        ):
+            await self.sleep(0.1)
+            return True
+        elif "com/recaptcha/api.js" in source:
+            await self.sleep(1.6)  # Still loading
+            return True
+        return False
+
+    async def __gui_click_recaptcha(self):
+        selector = None
+        if await self.is_element_present('iframe[title="reCAPTCHA"]'):
+            selector = 'iframe[title="reCAPTCHA"]'
+        else:
+            return
+        await self.sleep(0.5)
+        with suppress(Exception):
+            element_rect = await self.get_gui_element_rect(selector, timeout=1)
+            e_x = element_rect["x"]
+            e_y = element_rect["y"]
+            x_offset = 26
+            y_offset = 35
+            if await asyncio.to_thread(shared_utils.is_windows):
+                x_offset = 29
+            x = e_x + x_offset
+            y = e_y + y_offset
+            sb_config._saved_cf_x_y = (x, y)  # For debugging later
+            await self.sleep(0.11)
+            gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
+            with await asyncio.to_thread(gui_lock.acquire):
+                await self.bring_to_front()
+                await self.sleep(0.05)
+                await self.click_with_offset(
+                    selector, x_offset, y_offset, timeout=1
+                )
+                await self.sleep(0.22)
+
+    async def get_element_rect(self, selector, timeout=5):
+        element = await self.select(selector, timeout=timeout)
+        coordinates = None
+        if ":contains(" in selector:
+            position = await element.get_position_async()
+            x = position.x
+            y = position.y
+            width = position.width
+            height = position.height
+            coordinates = {"x": x, "y": y, "width": width, "height": height}
+        else:
+            coordinates = await self.js_dumps(
+                """document.querySelector('%s').getBoundingClientRect()"""
+                % js_utils.escape_quotes_if_needed(re.escape(selector))
+            )
+        return coordinates
+
+    async def get_window_rect(self):
+        coordinates = {}
+        innerWidth = await self.evaluate("window.innerWidth")
+        innerHeight = await self.evaluate("window.innerHeight")
+        outerWidth = await self.evaluate("window.outerWidth")
+        outerHeight = await self.evaluate("window.outerHeight")
+        pageXOffset = await self.evaluate("window.pageXOffset")
+        pageYOffset = await self.evaluate("window.pageYOffset")
+        scrollX = await self.evaluate("window.scrollX")
+        scrollY = await self.evaluate("window.scrollY")
+        screenLeft = await self.evaluate("window.screenLeft")
+        screenTop = await self.evaluate("window.screenTop")
+        x = await self.evaluate("window.screenX")
+        y = await self.evaluate("window.screenY")
+        coordinates["innerWidth"] = innerWidth
+        coordinates["innerHeight"] = innerHeight
+        coordinates["outerWidth"] = outerWidth
+        coordinates["outerHeight"] = outerHeight
+        coordinates["width"] = outerWidth
+        coordinates["height"] = outerHeight
+        coordinates["pageXOffset"] = pageXOffset if pageXOffset else 0
+        coordinates["pageYOffset"] = pageYOffset if pageYOffset else 0
+        coordinates["scrollX"] = scrollX if scrollX else 0
+        coordinates["scrollY"] = scrollY if scrollY else 0
+        coordinates["screenLeft"] = screenLeft if screenLeft else 0
+        coordinates["screenTop"] = screenTop if screenTop else 0
+        coordinates["x"] = x if x else 0
+        coordinates["y"] = y if y else 0
+        return coordinates
+
+    async def get_gui_element_rect(self, selector, timeout=5):
+        """(Coordinates are relative to the screen. Not the window.)"""
+        element_rect = await self.get_element_rect(selector, timeout=timeout)
+        e_width = element_rect["width"]
+        e_height = element_rect["height"]
+        window_rect = await self.get_window_rect()
+        w_bottom_y = window_rect["y"] + window_rect["height"]
+        viewport_height = window_rect["innerHeight"]
+        x = window_rect["x"] + element_rect["x"]
+        y = w_bottom_y - viewport_height + element_rect["y"]
+        y_scroll_offset = window_rect["pageYOffset"]
+        if (
+            hasattr(sb_config, "_cdp_browser")
+            and sb_config._cdp_browser == "opera"
+        ):
+            # Handle special case where Opera side panel shifts coordinates
+            x_offset = window_rect["outerWidth"] - window_rect["innerWidth"]
+            if x_offset > 56:
+                x_offset = 56
+            elif x_offset < 22:
+                x_offset = 0
+            x = x + x_offset
+        y = y - y_scroll_offset
+        x = x + window_rect["scrollX"]
+        y = y + window_rect["scrollY"]
+        return ({"height": e_height, "width": e_width, "x": x, "y": y})
+
+    async def get_title(self):
+        return await self.evaluate("document.title")
+
+    async def get_current_url(self):
+        return await self.evaluate("window.location.href")
+
+    async def send_keys(self, selector, text, timeout=5):
+        element = await self.find(selector, timeout=timeout)
+        await element.send_keys_async(text)
+
+    async def type(self, selector, text, timeout=5):
+        await self.send_keys(selector, text, timeout=timeout)
+
+    async def click(self, selector, timeout=5):
+        element = await self.find(selector, timeout=timeout)
+        await element.click_async()
+
+    async def click_with_offset(self, selector, x, y, center=False, timeout=5):
+        element = await self.find(selector, timeout=timeout)
+        await element.scroll_into_view_async()
+        await element.mouse_click_with_offset_async(x=x, y=y, center=center)
+
+    async def solve_captcha(self):
+        await self.sleep(0.11)
+        source = await self.get_html()
+        if await self.__on_a_cf_turnstile_page(source):
+            pass
+        elif await self.__on_a_g_recaptcha_page(source):
+            await self.__gui_click_recaptcha()
+            return
+        else:
+            return
+        selector = None
+        if await self.is_element_present('[class="cf-turnstile"]'):
+            selector = '[class="cf-turnstile"]'
+        elif await self.is_element_present("#challenge-form div > div"):
+            selector = "#challenge-form div > div"
+        elif await self.is_element_present('[style="display: grid;"] div div'):
+            selector = '[style="display: grid;"] div div'
+        elif await self.is_element_present("[class*=spacer] + div div"):
+            selector = '[class*=spacer] + div div'
+        elif await self.is_element_present(".spacer div:not([class])"):
+            selector = ".spacer div:not([class])"
+        elif await self.is_element_present('[data-testid*="challenge-"] div'):
+            selector = '[data-testid*="challenge-"] div'
+        elif await self.is_element_present(
+            "div#turnstile-widget div:not([class])"
+        ):
+            selector = "div#turnstile-widget div:not([class])"
+        elif await self.is_element_present("ngx-turnstile div:not([class])"):
+            selector = "ngx-turnstile div:not([class])"
+        elif await self.is_element_present(
+            'form div:not([class]):has(input[name*="cf-turn"])'
+        ):
+            selector = 'form div:not([class]):has(input[name*="cf-turn"])'
+        elif await self.is_element_present("form div:not(:has(*))"):
+            selector = "form div:not(:has(*))"
+        elif await self.is_element_present(
+            "body > div#check > div:not([class])"
+        ):
+            selector = "body > div#check > div:not([class])"
+        elif await self.is_element_present(".cf-turnstile-wrapper"):
+            selector = ".cf-turnstile-wrapper"
+        elif await self.is_element_present(
+            '[id*="turnstile"] div:not([class])'
+        ):
+            selector = '[id*="turnstile"] div:not([class])'
+        elif await self.is_element_present(
+            '[class*="turnstile"] div:not([class])'
+        ):
+            selector = '[class*="turnstile"] div:not([class])'
+        elif await self.is_element_present(
+            '[data-callback="onCaptchaSuccess"]'
+        ):
+            selector = '[data-callback="onCaptchaSuccess"]'
+        elif await self.is_element_present(
+            "div:not([class]) > div:not([class])"
+        ):
+            selector = "div:not([class]) > div:not([class])"
+        else:
+            return
+        if not selector:
+            return
+        if (
+            await self.is_element_present("form")
+            and (
+                await self.is_element_present('form[class*="center"]')
+                or await self.is_element_present('form[class*="right"]')
+                or await self.is_element_present('form div[class*="center"]')
+                or await self.is_element_present('form div[class*="right"]')
+            )
+        ):
+            script = (
+                """var $elements = document.querySelectorAll(
+                'form[class], form div[class]');
+                var index = 0, length = $elements.length;
+                for(; index < length; index++){
+                the_class = $elements[index].getAttribute('class');
+                new_class = the_class.replaceAll('center', 'left');
+                new_class = new_class.replaceAll('right', 'left');
+                $elements[index].setAttribute('class', new_class);}"""
+            )
+            with suppress(Exception):
+                await self.evaluate(script)
+        elif (
+            await self.is_element_present("form")
+            and (
+                await self.is_element_present('form div[style*="center"]')
+                or await self.is_element_present('form div[style*="right"]')
+            )
+        ):
+            script = (
+                """var $elements = document.querySelectorAll(
+                'form[style], form div[style]');
+                var index = 0, length = $elements.length;
+                for(; index < length; index++){
+                the_style = $elements[index].getAttribute('style');
+                new_style = the_style.replaceAll('center', 'left');
+                new_style = new_style.replaceAll('right', 'left');
+                $elements[index].setAttribute('style', new_style);}"""
+            )
+            with suppress(Exception):
+                await self.evaluate(script)
+        elif (
+            await self.is_element_present(
+                'form [id*="turnstile"] div:not([class])'
+            )
+            or await self.is_element_present(
+                'form [class*="turnstile"] div:not([class])'
+            )
+        ):
+            script = (
+                """var $elements = document.querySelectorAll(
+                'form [id*="turnstile"]');
+                var index = 0, length = $elements.length;
+                for(; index < length; index++){
+                $elements[index].setAttribute('align', 'left');}
+                var $elements = document.querySelectorAll(
+                'form [class*="turnstile"]');
+                var index = 0, length = $elements.length;
+                for(; index < length; index++){
+                $elements[index].setAttribute('align', 'left');}"""
+            )
+            with suppress(Exception):
+                await self.evaluate(script)
+        elif (
+            await self.is_element_present(
+                '[style*="text-align: center;"] div:not([class])'
+            )
+        ):
+            script = (
+                """var $elements = document.querySelectorAll(
+                '[style*="text-align: center;"]');
+                var index = 0, length = $elements.length;
+                for(; index < length; index++){
+                the_style = $elements[index].getAttribute('style');
+                new_style = the_style.replaceAll('center', 'left');
+                $elements[index].setAttribute('style', new_style);}"""
+            )
+            with suppress(Exception):
+                await self.evaluate(script)
+        with suppress(Exception):
+            await self.sleep(0.05)
+            element_rect = await self.get_gui_element_rect(selector, timeout=1)
+            e_x = element_rect["x"]
+            e_y = element_rect["y"]
+            x_offset = 32
+            y_offset = 32
+            if await asyncio.to_thread(shared_utils.is_windows):
+                y_offset = 28
+            x = e_x + x_offset
+            y = e_y + y_offset
+            sb_config._saved_cf_x_y = (x, y)  # For debugging later
+            await self.sleep(0.11)
+            gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
+            with await asyncio.to_thread(gui_lock.acquire):
+                await self.bring_to_front()
+                await self.sleep(0.05)
+                await self.click_with_offset(
+                    selector, x_offset, y_offset, timeout=1
+                )
+                await self.sleep(0.22)
+
+    async def click_captcha(self):
+        await self.solve_captcha()
 
     async def get_document(self):
         return await self.send(cdp.dom.get_document())
@@ -1365,7 +1738,9 @@ class Tab(Connection):
         :param selector: css selector string
         :type selector: str
         """
-        return self.wait_for(text, selector, timeout)
+        return self.wait_for(
+            selector=selector, text=text, timeout=timeout
+        )
 
     def __eq__(self, other: Tab):
         try:

seleniumbase/utilities/selenium_grid/download_selenium_server.py

@@ -19,7 +19,7 @@ FULL_DOWNLOAD_PATH = os.getcwd() + "/" + RENAMED_JAR_FILE
 def download_selenium_server():
     """Downloads the Selenium Server JAR file."""
     try:
-        local_file = open(JAR_FILE, "wb")
+        local_file = open(JAR_FILE, mode="wb")
         remote_file = urlopen(SELENIUM_JAR)
         print("Downloading the Selenium Server JAR file...\n")
         local_file.write(remote_file.read())

seleniumbase/utilities/selenium_grid/grid_hub.py

@@ -1,4 +1,3 @@
-import codecs
 import os
 import subprocess
 import sys
@@ -78,7 +77,7 @@ def main():
     data = []
     data.append(verbose)
     file_path = os.path.join(dir_path, "verbose_hub_server.dat")
-    file = codecs.open(file_path, "w+", "utf-8")
+    file = open(file_path, mode="w+", encoding="utf-8")
     file.writelines("\r\n".join(data))
     file.close()
 

seleniumbase/utilities/selenium_grid/grid_node.py

@@ -1,4 +1,3 @@
-import codecs
 import os
 import subprocess
 import sys
@@ -88,14 +87,14 @@ def main():
     data = []
     data.append(server_ip)
     file_path = os.path.join(dir_path, "ip_of_grid_hub.dat")
-    file = codecs.open(file_path, "w+", "utf-8")
+    file = open(file_path, mode="w+", encoding="utf-8")
     file.writelines("\r\n".join(data))
     file.close()
 
     data = []
     data.append(verbose)
     file_path = os.path.join(dir_path, "verbose_node_server.dat")
-    file = codecs.open(file_path, "w+", "utf-8")
+    file = open(file_path, mode="w+", encoding="utf-8")
     file.writelines("\r\n".join(data))
     file.close()
 

seleniumbase/utilities/selenium_ide/convert_ide.py

@@ -8,7 +8,6 @@ Usage:
 Output:
         [NEW_FILE_SB].py  (adds "_SB" to the original file name)
                           (the original file is kept intact)"""
-import codecs
 import re
 import sys
 from seleniumbase.fixtures import js_utils
@@ -60,7 +59,7 @@ def main():
     uses_keys = False
     uses_select = False
 
-    with open(webdriver_python_file, "r", encoding="utf-8") as f:
+    with open(webdriver_python_file, mode="r", encoding="utf-8") as f:
         all_code = f.read()
     if "def test_" not in all_code:
         raise Exception(
@@ -894,7 +893,7 @@ def main():
     # Create SeleniumBase test file
     base_file_name = webdriver_python_file.split(".py")[0]
     converted_file_name = base_file_name + "_SB.py"
-    out_file = codecs.open(converted_file_name, "w+", encoding="utf-8")
+    out_file = open(converted_file_name, mode="w+", encoding="utf-8")
     out_file.writelines(seleniumbase_code)
     out_file.close()
     print(