requestium 0.5.1__py3-none-any.whl

requestium/__init__.py

@@ -0,0 +1,6 @@
+from selenium.common import exceptions  # noqa: F401
+from selenium.webdriver.common.by import By  # noqa: F401
+from selenium.webdriver.common.keys import Keys  # noqa: F401
+from selenium.webdriver.support.ui import Select  # noqa: F401
+
+from .requestium import Session  # noqa: F401

requestium/requestium.py

@@ -0,0 +1,6 @@
+from .requestium_mixin import (  # noqa: F401
+    DriverMixin,
+    _ensure_click,
+)
+from .requestium_response import RequestiumResponse  # noqa: F401
+from .requestium_session import Session  # noqa: F401

requestium/requestium_mixin.py

@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import functools
+import time
+import warnings
+from typing import TYPE_CHECKING, Any
+
+import tldextract
+from parsel.selector import Selector, SelectorList
+from selenium.common.exceptions import NoSuchWindowException, WebDriverException
+from selenium.webdriver.common.by import By, ByType
+from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
+from selenium.webdriver.support import expected_conditions
+from selenium.webdriver.support.ui import WebDriverWait
+
+if TYPE_CHECKING:
+    from selenium.webdriver.remote.webelement import WebElement
+
+
+DEFAULT_TIMEOUT: float = 0.5
+
+
+def _ensure_click(self: WebElement) -> None:
+    """
+    Ensure a click gets made, because Selenium can be a bit buggy about clicks.
+
+    This method gets added to the selenium element returned in '__ensure_element_by_xpath'.
+    We should probably add it to more selenium methods, such as all the 'find**' methods though.
+
+    I wrote this method out of frustration with chromedriver and its problems with clicking
+    items that need to be scrolled to in order to be clickable. In '__ensure_element_by_xpath' we
+    scroll to the item before returning it, but chrome has some problems if it doesn't get some
+    time to scroll to the item. This method ensures chromes gets enough time to scroll to the item
+    before clicking it. I tried SEVERAL more 'correct' methods to get around this, but none of them
+    worked 100% of the time (waiting for the element to be 'clickable' does not work).
+    """
+    # We ensure the element is scrolled into the middle of the viewport to ensure that
+    # it is clickable. There are two main ways an element may not be clickable:
+    #   - It is outside of the viewport
+    #   - It is under a banner or toolbar
+    # This script solves both cases
+    script = (
+        "var viewPortHeight = Math.max("
+        "document.documentElement.clientHeight, window.innerHeight || 0);"
+        "var elementTop = arguments[0].getBoundingClientRect().top;"
+        "window.scrollBy(0, elementTop-(viewPortHeight/2));"
+    )
+    self.parent.execute_script(script, self)  # parent = the webdriver
+
+    exception_message = ""
+    for _ in range(10):
+        try:
+            self.click()
+            return
+        except WebDriverException as e:
+            exception_message = str(e)
+            time.sleep(0.2)
+    msg = f"Couldn't click item after trying 10 times, got error message: \n{exception_message}"
+    raise WebDriverException(msg)
+
+
+class DriverMixin(RemoteWebDriver):
+    """Provides helper methods to our driver classes."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.default_timeout = kwargs.pop("default_timeout", DEFAULT_TIMEOUT)
+        super().__init__(*args, **kwargs)
+
+    def try_add_cookie(self, cookie: dict[str, Any]) -> bool:
+        """
+        Attempt to add the cookie.
+
+        Suppress any errors, and simply detect success or failure.
+        """
+        try:
+            self.add_cookie(cookie)
+        except WebDriverException as e:
+            if e.msg and not e.msg.__contains__("Couldn't add the following cookie to the webdriver"):
+                raise WebDriverException from e
+        return self.is_cookie_in_driver(cookie)
+
+    def ensure_add_cookie(self, cookie: dict[str, Any], override_domain: str | None = None) -> None:
+        """
+        Add a cookie to the driver and check to ensure it has been added.
+
+        Selenium needs the driver to be currently at the domain of the cookie
+        before allowing you to add it, so we need to get through this limitation.
+
+        The cookie parameter is a dict which must contain the keys (name, value, domain) and
+        may contain the keys (path, expiry).
+
+        We first check that we aren't currently in the cookie's domain, if we aren't, we GET
+        the cookie's domain and then add the cookies to the driver.
+
+        We can override the cookie's domain using 'override_domain'. The use for this
+        parameter is that sometimes GETting the cookie's domain redirects you to a different
+        sub domain, and therefore adding the cookie fails. So sometimes the user may
+        need to override the cookie's domain to a less strict one, Eg.: 'site.com' instead of
+        'home.site.com', in this way even if the site redirects us to a subdomain, the cookie will
+        stick. If you set the domain to '', the cookie gets added with whatever domain the browser
+        is currently at (at least in chrome it does), so this ensures the cookie gets added.
+
+        It also retries adding the cookie with a more permissive domain if it fails in the first
+        try, and raises an exception if that fails. The standard selenium behaviour in this case
+        was to not do anything, which was very hard to debug.
+        """
+        if override_domain:
+            cookie["domain"] = override_domain
+
+        cookie_domain = cookie["domain"] if cookie["domain"][0] != "." else cookie["domain"][1:]
+        try:
+            browser_domain = tldextract.extract(self.current_url).fqdn
+        except (AttributeError, NoSuchWindowException):
+            browser_domain = ""
+        if cookie_domain not in browser_domain:
+            # TODO @joaqo: Check if hardcoding 'http' causes trouble.
+            # https://github.com/tryolabs/requestium/issues/97
+            # Consider using a new proxy for this next request to not cause an anomalous
+            # request. This way their server sees our ip address as continuously having the
+            # same cookies and not have a request mid-session with no cookies
+            self.get("http://" + cookie_domain)
+
+        cookie_added = self.try_add_cookie(cookie)
+
+        # If we fail adding the cookie, retry with a more permissive domain
+        if not cookie_added:
+            cookie["domain"] = tldextract.extract(cookie["domain"]).top_domain_under_public_suffix
+            cookie_added = self.try_add_cookie(cookie)
+            if not cookie_added:
+                msg = f"Couldn't add the following cookie to the webdriver: {cookie}"
+                raise WebDriverException(msg)
+
+    def is_cookie_in_driver(self, cookie: dict[str, Any]) -> bool:
+        """
+        We check that the cookie is correctly added to the driver.
+
+        We only compare name, value and domain, as the rest can produce false negatives.
+        We are a bit lenient when comparing domains.
+        """
+        for driver_cookie in self.get_cookies():
+            name_matches = cookie["name"] == driver_cookie["name"]
+            value_matches = cookie["value"] == driver_cookie["value"]
+            domain_matches = driver_cookie["domain"] in (cookie["domain"], "." + cookie["domain"])
+            if name_matches and value_matches and domain_matches:
+                return True
+        return False
+
+    def ensure_element_by_id(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.ID, selector, state, timeout)
+
+    def ensure_element_by_name(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.NAME, selector, state, timeout)
+
+    def ensure_element_by_xpath(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.XPATH, selector, state, timeout)
+
+    def ensure_element_by_link_text(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.LINK_TEXT, selector, state, timeout)
+
+    def ensure_element_by_partial_link_text(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.PARTIAL_LINK_TEXT, selector, state, timeout)
+
+    def ensure_element_by_tag_name(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.TAG_NAME, selector, state, timeout)
+
+    def ensure_element_by_class_name(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.CLASS_NAME, selector, state, timeout)
+
+    def ensure_element_by_css_selector(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        return self.ensure_element(By.CSS_SELECTOR, selector, state, timeout)
+
+    def ensure_element(self, locator: ByType | str, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
+        """
+        Wait until an element appears or disappears in the browser.
+
+        The webdriver runs in parallel with our scripts, so we must wait for it everytime it
+        runs javascript. Selenium automatically waits till a page loads when GETing it,
+        but it doesn't do this when it runs javascript and makes AJAX requests.
+        So we must explicitly wait in that case.
+
+        The 'locator' argument defines what strategy we use to search for the element.
+        It expects standard names from the By class in selenium.webdriver.common.by.
+        https://www.selenium.dev/selenium/docs/api/py/webdriver/selenium.webdriver.common.by.html
+
+        The 'state' argument allows us to chose between waiting for the element to be visible,
+        clickable, present, or invisible. Presence is more inclusive, but sometimes we want to
+        know if the element is visible. Careful, its not always intuitive what Selenium considers
+        to be a visible element. We can also wait for it to be clickable, although this method
+        is a bit buggy in selenium, an element can be 'clickable' according to selenium and
+        still fail when we try to click it.
+
+        More info at: http://selenium-python.readthedocs.io/waits.html
+        """
+        locators_compatibility = {
+            "link_text": By.LINK_TEXT,
+            "partial_link_text": By.PARTIAL_LINK_TEXT,
+            "tag_name": By.TAG_NAME,
+            "class_name": By.CLASS_NAME,
+            "css_selector": By.CSS_SELECTOR,
+        }
+        if locator in locators_compatibility:
+            warnings.warn(
+                """
+                Support for locator strategy names with underscores is deprecated.
+                Use strategies from Selenium's By class (importable from selenium.webdriver.common.by).
+                """,
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            locator = locators_compatibility[locator]
+
+        if not timeout:
+            timeout = self.default_timeout or DEFAULT_TIMEOUT
+
+        if state == "visible":
+            element = WebDriverWait(self, timeout).until(expected_conditions.visibility_of_element_located((locator, selector)))
+        elif state == "clickable":
+            element = WebDriverWait(self, timeout).until(expected_conditions.element_to_be_clickable((locator, selector)))
+        elif state == "present":
+            element = WebDriverWait(self, timeout).until(expected_conditions.presence_of_element_located((locator, selector)))
+        elif state == "invisible":
+            WebDriverWait(self, timeout).until(expected_conditions.invisibility_of_element_located((locator, selector)))
+            element = None
+        else:
+            msg = f"The 'state' argument must be 'visible', 'clickable', 'present' or 'invisible', not '{state}'"
+            raise ValueError(msg)
+
+        # We add this method to our element to provide a more robust click. Chromedriver
+        # sometimes needs some time before it can click an item, specially if it needs to
+        # scroll into it first. This method ensures clicks don't fail because of this.
+        if element:
+            element.ensure_click = functools.partial(_ensure_click, element)  # type: ignore[attr-defined]
+        return element
+
+    @property
+    def selector(self) -> Selector:
+        """
+        Returns the current state of the browser in a Selector.
+
+        We re-parse the site on each xpath, css, re call because we are running a web browser
+        and the site may change between calls
+        """
+        return Selector(text=self.page_source)
+
+    def xpath(self, *args, **kwargs) -> SelectorList[Selector]:
+        return self.selector.xpath(*args, **kwargs)
+
+    def css(self, *args, **kwargs) -> SelectorList[Selector]:
+        return self.selector.css(*args, **kwargs)
+
+    def re(self, *args, **kwargs) -> list[str]:
+        return self.selector.re(*args, **kwargs)
+
+    def re_first(self, *args, **kwargs) -> str | None:
+        return self.selector.re_first(*args, **kwargs)

requestium/requestium_response.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import requests
+from parsel.selector import Selector, SelectorList
+from requests import Response
+
+
+class RequestiumResponse(requests.Response):
+    """Adds xpath, css, and regex methods to a normal requests response object."""
+
+    def __init__(self, response: Response) -> None:
+        super().__init__()
+        self.__class__ = type(response.__class__.__name__, (self.__class__, response.__class__), response.__dict__)
+
+    @property
+    def selector(self) -> Selector:
+        """
+        Returns the response text in a Selector.
+
+        We re-parse the text on each xpath, css, re call in case the encoding has changed.
+        """
+        return Selector(text=self.text)
+
+    def xpath(self, *args, **kwargs) -> SelectorList[Selector]:
+        return self.selector.xpath(*args, **kwargs)
+
+    def css(self, *args, **kwargs) -> SelectorList[Selector]:
+        return self.selector.css(*args, **kwargs)
+
+    def re(self, *args, **kwargs) -> list[str]:
+        return self.selector.re(*args, **kwargs)
+
+    def re_first(self, *args, **kwargs) -> str | None:
+        return self.selector.re_first(*args, **kwargs)

requestium/requestium_session.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import functools
+import types
+from typing import Any
+
+import requests
+import tldextract
+from selenium import webdriver
+from selenium.common import InvalidCookieDomainException
+from selenium.webdriver import ChromeService
+
+from .requestium_mixin import DriverMixin
+from .requestium_response import RequestiumResponse
+
+RequestiumChrome = type("RequestiumChrome", (DriverMixin, webdriver.Chrome), {})
+
+
+class Session(requests.Session):
+    """
+    Class that adds a Selenium Webdriver and helper methods to a  Requests Session.
+
+    This session class is a normal Requests Session that has the ability to switch back
+    and forth between this session and a webdriver, allowing us to run js when needed.
+
+    Cookie transfer is done with the 'transfer' methods.
+
+    Header and proxy transfer is done only one time when the driver process starts.
+
+    Some useful helper methods and object wrappings have been added.
+    """
+
+    def _start_chrome_browser(self, headless: bool | None = False):  # noqa C901
+        # TODO @joaqo: Transfer of proxies and headers.
+        # https://github.com/tryolabs/requestium/issues/96
+        # Not currently supported by chromedriver. Choosing not to use plug-ins
+        # for this as I don't want to worry about the extra dependencies and
+        # plug-ins don't work in headless mode. :-(
+        chrome_options = webdriver.ChromeOptions()
+
+        if headless:
+            chrome_options.add_argument("headless=new")
+
+        if "binary_location" in self.webdriver_options:
+            chrome_options.binary_location = self.webdriver_options["binary_location"]
+
+        if "arguments" in self.webdriver_options:
+            if isinstance(self.webdriver_options["arguments"], list):
+                for arg in self.webdriver_options["arguments"]:
+                    chrome_options.add_argument(arg)
+            else:
+                msg = f"'arguments' option must be a list, but got {type(self.webdriver_options['arguments']).__name__}"
+                raise TypeError(msg)
+
+        if "extensions" in self.webdriver_options and isinstance(self.webdriver_options["extensions"], list):
+            for arg in self.webdriver_options["extensions"]:
+                chrome_options.add_extension(arg)
+
+        if "prefs" in self.webdriver_options:
+            prefs = self.webdriver_options["prefs"]
+            chrome_options.add_experimental_option("prefs", prefs)
+
+        experimental_options = self.webdriver_options.get("experimental_options")
+        if isinstance(experimental_options, dict):
+            for name, value in experimental_options.items():
+                chrome_options.add_experimental_option(name, value)
+
+        # Selenium updated webdriver.Chrome's arg and kwargs, to accept options, service, keep_alive
+        # since ChromeService is the only object where webdriver_path is mapped to executable_path, it must be
+        # initialized and passed in as a kwarg to RequestiumChrome so it can be passed in as a kwarg
+        # when passed into webdriver.Chrome in super(DriverMixin, self).__init__(*args, **kwargs)
+        service = ChromeService(executable_path=self.webdriver_path)
+        return RequestiumChrome(service=service, options=chrome_options, default_timeout=self.default_timeout)
+
+    def __init__(
+        self,
+        *,
+        webdriver_path: str | None = None,
+        headless: bool | None = None,
+        default_timeout: float = 5,
+        webdriver_options: dict[str, Any] | None = None,
+        driver: DriverMixin | None = None,
+    ) -> None:
+        super().__init__()
+
+        if webdriver_options is None:
+            webdriver_options = {}
+
+        self.webdriver_path = webdriver_path
+        self.default_timeout = default_timeout
+        self.webdriver_options = webdriver_options
+        self._driver = driver
+        self._last_requests_url: str | None = None
+
+        if not self._driver:
+            self._driver_initializer = functools.partial(self._start_chrome_browser, headless=headless)
+        else:
+            for name in DriverMixin.__dict__:
+                name_private = name.startswith("__") and name.endswith("__")
+                name_function = isinstance(DriverMixin.__dict__[name], types.FunctionType)
+                name_in_driver = name in dir(self._driver)
+                if name_private or not name_function or name_in_driver:
+                    continue
+                self._driver.__dict__[name] = DriverMixin.__dict__[name].__get__(self._driver)
+            self._driver.default_timeout = self.default_timeout
+
+    @property
+    def driver(self) -> DriverMixin:
+        if self._driver is None:
+            self._driver = self._driver_initializer()
+        return self._driver
+
+    def transfer_session_cookies_to_driver(self, domain: str | None = None) -> None:
+        """
+        Copy the Session's cookies into the webdriver.
+
+        Using the 'domain' parameter we choose the cookies we wish to transfer, we only
+        transfer the cookies which belong to that domain. The domain defaults to our last visited
+        site if not provided.
+        """
+        if not domain and self._last_requests_url:
+            domain = tldextract.extract(self._last_requests_url).top_domain_under_public_suffix
+
+        if not domain:
+            msg = "Trying to transfer cookies to selenium without specifying a domain and without having visited any page in the current session"
+            raise InvalidCookieDomainException(msg)
+
+        # Transfer cookies
+        for c in [c for c in self.cookies if domain in c.domain]:
+            cookie = {"name": c.name, "value": c.value, "path": c.path, "expiry": c.expires, "domain": c.domain}
+
+            self.driver.ensure_add_cookie({k: v for k, v in cookie.items() if v is not None})
+
+    def transfer_driver_cookies_to_session(self, *, copy_user_agent: bool | None = True) -> None:
+        if copy_user_agent:
+            self.copy_user_agent_from_driver()
+
+        for cookie in self.driver.get_cookies():
+            self.cookies.set(cookie["name"], cookie["value"], domain=cookie["domain"])
+
+    def get(self, *args, **kwargs) -> RequestiumResponse:
+        resp = super().get(*args, **kwargs)
+        self._last_requests_url = resp.url
+        return RequestiumResponse(resp)
+
+    def post(self, *args, **kwargs) -> RequestiumResponse:
+        resp = super().post(*args, **kwargs)
+        self._last_requests_url = resp.url
+        return RequestiumResponse(resp)
+
+    def put(self, *args, **kwargs) -> RequestiumResponse:
+        resp = super().put(*args, **kwargs)
+        self._last_requests_url = resp.url
+        return RequestiumResponse(resp)
+
+    def copy_user_agent_from_driver(self) -> None:
+        """
+        Update requests' session user-agent with the driver's user agent.
+
+        This method will start the browser process if its not already running.
+        """
+        selenium_user_agent = self.driver.execute_script("return navigator.userAgent;")
+        self.headers.update({"user-agent": selenium_user_agent})

requestium-0.5.1.dist-info/METADATA

@@ -0,0 +1,376 @@
+Metadata-Version: 2.4
+Name: requestium
+Version: 0.5.1
+Author-email: Joaquin Alori <joaquin@tryolabs.com>
+Maintainer-email: Judson Neer <jkudson.neer@gmail.com>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2017, Tryolabs
+        All rights reserved.
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        * Redistributions of source code must retain the above copyright notice, this
+          list of conditions and the following disclaimer.
+        
+        * Redistributions in binary form must reproduce the above copyright notice,
+          this list of conditions and the following disclaimer in the documentation
+          and/or other materials provided with the distribution.
+        
+        * Neither the name of the copyright holder nor the names of its
+          contributors may be used to endorse or promote products derived from
+          this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: source, https://github.com/tryolabs/requestium
+Project-URL: download, https://pypi.org/project/requestium/#files
+Project-URL: issues, https://github.com/tryolabs/requestium/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: OS Independent
+Classifier: Environment :: Web Environment
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: parsel>=1.7
+Requires-Dist: requests>=2.27
+Requires-Dist: selenium>=4.32
+Requires-Dist: tldextract>=5.3
+Provides-Extra: dev
+Requires-Dist: bandit[sarif]==1.9.2; extra == "dev"
+Requires-Dist: mypy==1.19.1; extra == "dev"
+Requires-Dist: pre-commit~=4.5.0; extra == "dev"
+Requires-Dist: pytest-cov~=7.0.0; extra == "dev"
+Requires-Dist: pytest-xdist~=3.8.0; extra == "dev"
+Requires-Dist: pytest~=9.0.0; extra == "dev"
+Requires-Dist: ruff==0.14.11; extra == "dev"
+Requires-Dist: ssort==0.16.0; extra == "dev"
+Requires-Dist: types-requests==2.32.4.20260107; extra == "dev"
+Dynamic: license-file
+
+![Requestium](https://user-images.githubusercontent.com/14966348/32966130-8bb15b00-cbb7-11e7-9faf-85963ec5bd82.png)
+========
+
+[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/requestium)](https://pypi.org/project/requestium/)
+[![PyPI](https://img.shields.io/pypi/v/requestium?color=blue)](https://pypi.org/project/requestium/)
+[![Coverage](https://raw.githubusercontent.com/tryolabs/requestium/python-coverage-comment-action-data/badge.svg)](https://htmlpreview.github.io/?https://github.com/tryolabs/requestium/blob/python-coverage-comment-action-data/htmlcov/index.html)
+[![Python checks](https://github.com/tryolabs/requestium/actions/workflows/python.yml/badge.svg)](https://github.com/tryolabs/requestium/actions/workflows/python.yml)
+
+Requestium is a Python library that merges the power of [Requests](https://github.com/requests/requests), [Selenium](https://github.com/SeleniumHQ/selenium), and [Parsel](https://github.com/scrapy/parsel) into a single integrated tool for automatizing web actions.
+
+The library was created for writing web automation scripts that are written using mostly Requests but that are able to seamlessly switch to Selenium for the JavaScript heavy parts of the website, while maintaining the session.
+
+Requestium adds independent improvements to both Requests and Selenium, and every new feature is lazily evaluated, so its useful even if writing scripts that use only Requests or Selenium.
+
+Read more about the motivation behind creating this library in this [blopost](https://tryolabs.com/blog/2017/11/22/requestium-integration-layer-requests-selenium-web-automation/).
+
+## Features
+- Enables switching between a Requests' Session and a Selenium webdriver while maintaining the current web session.
+- Integrates Parsel's parser into the library, making xpath, css, and regex much cleaner to write.
+- Improves Selenium's handling of dynamically loading elements.
+- Makes cookie handling more flexible in Selenium.
+- Makes clicking elements in Selenium more reliable.
+- Supports Chromedriver natively plus adding a custom webdriver.
+
+## Installation
+```bash
+pip install requestium
+```
+
+You should then download your preferred Selenium webdriver if you plan to use the Selenium part of Requestium, such as [Chromedriver](https://sites.google.com/a/chromium.org/chromedriver/).
+
+## Usage
+First create a session as you would do on Requests, and optionally add arguments for the web-driver if you plan to use one.
+```python
+from requestium import Session, Keys
+
+options = {'arguments': ['headless']}
+s = Session(webdriver_path='./chromedriver', default_timeout=15, webdriver_options=options)
+```
+
+Since headless mode is common, there's a shortcut for it by specifying `headless=True`.
+
+```python
+from requestium import Session, Keys
+
+s = Session(webdriver_path='./chromedriver' headless=True)
+```
+
+You can also create a Selenium webdriver outside Requestium and have it use that instead:
+
+```python
+from selenium import webdriver
+from requestium import Session, Keys
+
+firefox_driver = webdriver.Firefox()
+
+s = Session(driver=firefox_driver)
+```
+
+You can also specify a 3rd party Chrome webdriver class and use it by specifying the `browser` argument as well. This will allow, for example, to use [Selenium-Wire](https://github.com/wkeeling/selenium-wire) to get XHR requests of a web page:
+
+```python
+from seleniumwire import webdriver
+from requestium import Session, Keys
+
+seleniumwire_driver = webdriver.Chrome()
+
+s = Session(webdriver_path='./chromedriver', driver=seleniumwire_driver)
+
+```
+
+You don't need to parse the response, it is done automatically when calling xpath, css or re.
+```python
+title = s.get('http://samplesite.com').xpath('//title/text()').extract_first(default='Default Title')
+```
+
+Regex require less boilerplate when compared to Python's standard `re` module.
+```python
+response = s.get('http://samplesite.com/sample_path')
+
+# Extracts the first match
+identifier = response.re_first(r'ID_\d\w\d', default='ID_1A1')
+
+# Extracts all matches as a list
+users = response.re(r'user_\d\d\d')
+```
+
+The Session object is just a regular Requests's session object, so you can use all of its methods.
+```python
+s.post('http://www.samplesite.com/sample', data={'field1': 'data1'})
+s.proxies.update({'http': 'http://10.11.4.254:3128', 'https': 'https://10.11.4.252:3128'})
+```
+
+And you can switch to using the Selenium webdriver to run any js code.
+```python
+s.transfer_session_cookies_to_driver()  # You can maintain the session if needed
+s.driver.get('http://www.samplesite.com/sample/process')
+```
+
+The driver object is a Selenium webdriver object, so you can use any of the normal selenium methods plus new methods added by Requestium.
+```python
+s.driver.find_element("xpath", "//input[@class='user_name']").send_keys('James Bond', Keys.ENTER)
+
+# New methods which wait for element to load instead of failing, useful for single page web apps
+s.driver.ensure_element("xpath", "//div[@attribute='button']").click()
+s.driver.ensure_element_by_xpath("//div[@attribute='button']").click()
+```
+
+Requestium also adds xpath, css, and re methods to the Selenium driver object.
+```python
+if s.driver.re(r'ID_\d\w\d some_pattern'):
+    print('Found it!')
+```
+
+And finally you can switch back to using Requests.
+```python
+s.transfer_driver_cookies_to_session()
+s.post('http://www.samplesite.com/sample2', data={'key1': 'value1'})
+```
+
+## Selenium workarounds
+Requestium adds several 'ensure' methods to the driver object, as Selenium is known to be very finicky about selecting elements and cookie handling.
+
+### Wait for element
+The `ensure_element` and `ensure_element_by_` methods wait for the element to be loaded in the browser and returns it as soon as it loads. They're named after Selenium's `find_element` and `find_element_by_` methods (which immediately raise an exception if they can't find the element).
+
+Requestium can wait for an element to be in any of the following states:
+- present (default)
+- clickable
+- visible
+- invisible (useful for things like waiting for *loading...* gifs to disappear)
+
+These methods are very useful for single page web apps where the site is dynamically changing its elements. We usually end up completely replacing our `find_element` and `find_element_by_` calls with `ensure_element` and `ensure_element_by_` calls as they are more flexible.
+
+Elements you get using these methods have the new `ensure_click` method which makes the click less prone to failure. This helps with getting through a lot of the problems with Selenium clicking.
+
+```python
+s.driver.ensure_element("xpath", "//li[@class='b1']", state='clickable', timeout=5).ensure_click()
+
+# === We also added these methods named in accordance to Selenium's api design ===
+# ensure_element_by_id
+# ensure_element_by_name
+# ensure_element_by_xpath
+# ensure_element_by_link_text
+# ensure_element_by_partial_link_text
+# ensure_element_by_tag_name
+# ensure_element_by_class_name
+# ensure_element_by_css_selector
+```
+
+### Add cookie
+The `ensure_add_cookie` method makes adding cookies much more robust. Selenium needs the browser to be at the cookie's domain before being able to add the cookie, this method offers several workarounds for this. If the browser is not in the cookies domain, it GETs the domain before adding the cookie. It also allows you to override the domain before adding it, and avoid making this GET. The domain can be overridden to `''`, this sets the cookie's domain to whatever domain the driver is currently in.
+
+If it can't add the cookie it tries to add it with a less restrictive domain (Eg.: home.site.com -> site.com) before failing.
+
+```python
+cookie = {"domain": "www.site.com",
+          "secure": false,
+          "value": "sd2451dgd13",
+          "expiry": 1516824855.759154,
+          "path": "/",
+          "httpOnly": true,
+          "name": "sessionid"}
+s.driver.ensure_add_cookie(cookie, override_domain='')
+```
+
+## Considerations
+New features are lazily evaluated, meaning:
+- The Selenium webdriver process is only started if you call the driver object. So if you don't need to use the webdriver, you could use the library with no overhead. Very useful if you just want to use the library for its integration with Parsel.
+- Parsing of the responses is only done if you call the `xpath`, `css`, or `re` methods of the response. So again there is no overhead if you don't need to use this feature.
+
+A byproduct of this is that the Selenium webdriver could be used just as a tool to ease in the development of regular Requests code: You can start writing your script using just the Requests' session, and at the last step of the script (the one you are currently working on) transfer the session to the Chrome webdriver. This way, a Chrome process starts in your machine, and acts as a real time "visor" for the last step of your code. You can see in what state your session is currently in, inspect it with Chrome's excellent inspect tools, and decide what's the next step your session object should take. Very useful to try code in an IPython interpreter and see how the site reacts in real time.
+
+When `transfer_driver_cookies_to_session` is called, Requestium automatically updates your Requests session user-agent to match that of the browser used in Selenium. This doesn't happen when running Requests without having switched from a Selenium session first though. So if you just want to run Requests but want it to use your browser's user agent instead of the default one (which sites love to block), just run:
+```python
+s.copy_user_agent_from_driver()
+```
+Take into account that doing this will launch a browser process.
+
+Note: The Selenium Chrome webdriver doesn't support automatic transfer of proxies from the Session to the Webdriver at the moment.
+
+## Comparison with Requests + Selenium + lxml
+A silly working example of a script that runs on Reddit. We'll then show how it compares to using Requests + Selenium + lxml instead of Requestium.
+
+### Using Requestium
+```python
+from requestium import Session, Keys
+
+# If you want requestium to type your username in the browser for you, write it in here:
+reddit_user_name = ''
+
+s = Session('./chromedriver', default_timeout=15)
+s.driver.get('http://reddit.com')
+s.driver.find_element("xpath", "//a[@href='https://www.reddit.com/login']").click()
+
+print('Waiting for elements to load...')
+s.driver.ensure_element("class name", "desktop-onboarding-sign-up__form-toggler",
+				      state='visible').click()
+
+if reddit_user_name:
+    s.driver.ensure_element('id', 'user_login').send_keys(reddit_user_name)
+    s.driver.ensure_element('id', 'passwd_login').send_keys(Keys.BACKSPACE)
+print('Please log-in in the chrome browser')
+
+s.driver.ensure_element("class name", "desktop-onboarding__title", timeout=60, state='invisible')
+print('Thanks!')
+
+if not reddit_user_name:
+    reddit_user_name = s.driver.xpath("//span[@class='user']//text()").extract_first()
+
+if reddit_user_name:
+    s.transfer_driver_cookies_to_session()
+    response = s.get("https://www.reddit.com/user/{}/".format(reddit_user_name))
+    cmnt_karma = response.xpath("//span[@class='karma comment-karma']//text()").extract_first()
+    reddit_golds_given = response.re_first(r"(\d+) gildings given out")
+    print("Comment karma: {}".format(cmnt_karma))
+    print("Reddit golds given: {}".format(reddit_golds_given))
+else:
+    print("Couldn't get user name")
+```
+
+### Using Requests + Selenium + lxml
+
+```python
+import re
+from lxml import etree
+from requests import Session
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# If you want requestium to type your username in the browser for you, write it in here:
+reddit_user_name = ''
+
+driver = webdriver.Chrome('./chromedriver')
+driver.get('http://reddit.com')
+driver.find_element("xpath", "//a[@href='https://www.reddit.com/login']").click()
+
+print('Waiting for elements to load...')
+WebDriverWait(driver, 15).until(
+    EC.visibility_of_element_located((By.CLASS_NAME, "desktop-onboarding-sign-up__form-toggler"))
+).click()
+
+if reddit_user_name:
+    WebDriverWait(driver, 15).until(
+        EC.presence_of_element_located((By.ID, 'user_login'))
+    ).send_keys(reddit_user_name)
+    driver.find_element('id', 'passwd_login').send_keys(Keys.BACKSPACE)
+print('Please log-in in the chrome browser')
+
+try:
+    WebDriverWait(driver, 3).until(
+        EC.presence_of_element_located((By.CLASS_NAME, "desktop-onboarding__title"))
+    )
+except TimeoutException:
+    pass
+WebDriverWait(driver, 60).until(
+    EC.invisibility_of_element_located((By.CLASS_NAME, "desktop-onboarding__title"))
+)
+print('Thanks!')
+
+if not reddit_user_name:
+    tree = etree.HTML(driver.page_source)
+    try:
+        reddit_user_name = tree.xpath("//span[@class='user']//text()")[0]
+    except IndexError:
+        reddit_user_name = None
+
+if reddit_user_name:
+    s = Session()
+    # Reddit will think we are a bot if we have the wrong user agent
+    selenium_user_agent = driver.execute_script("return navigator.userAgent;")
+    s.headers.update({"user-agent": selenium_user_agent})
+    for cookie in driver.get_cookies():
+        s.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'])
+    response = s.get("https://www.reddit.com/user/{}/".format(reddit_user_name))
+    try:
+        cmnt_karma = etree.HTML(response.content).xpath(
+            "//span[@class='karma comment-karma']//text()")[0]
+    except IndexError:
+        cmnt_karma = None
+    match = re.search(r"(\d+) gildings given out", str(response.content))
+    if match:
+        reddit_golds_given = match.group(1)
+    else:
+        reddit_golds_given = None
+    print("Comment karma: {}".format(cmnt_karma))
+    print("Reddit golds given: {}".format(reddit_golds_given))
+else:
+    print("Couldn't get user name")
+```
+
+## Similar Projects
+This project intends to be a drop-in replacement of Requests' Session object, with added functionality. If your use case is a drop in replacement for a Selenium webdriver, but that also has some of Requests' functionality, [Selenium-Requests](https://github.com/cryzed/Selenium-Requests) does just that.
+
+
+## License
+Copyright © 2018, [Tryolabs](https://tryolabs.com/). Released under the [BSD 3-Clause](https://github.com/tryolabs/requestium/blob/master/LICENSE).

requestium-0.5.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+requestium/__init__.py,sha256=BEjUrsq_EjA5osONG3zj3n_x2B2SgemDqY1XHYRkfsQ,283
+requestium/requestium.py,sha256=nsSQEmVFIcRHz0YvoepB3deXy46B06eqrv339O4FLWs,204
+requestium/requestium_mixin.py,sha256=6tdyRPX1qvP_BlZAaKLtTMKHGTFWRku0Ot459gFMwZM,12754
+requestium/requestium_response.py,sha256=5nta1i3x2Ya4tZiYGPznr15Guc36j3p3q8B5p1bddc0,1154
+requestium/requestium_session.py,sha256=Ia5PlswYZQXwwzRkqxqzz2IIHFX4RCSQ8dR3a6lozKU,7051
+requestium-0.5.1.dist-info/licenses/LICENSE,sha256=_KRtksFg3PbSlgNIn1UoPfJu-VboyW2Phdo3oPyosCk,1508
+requestium-0.5.1.dist-info/METADATA,sha256=WWKGloqUs6s2pOIuUT4sJT5-tedCAjTjLDewG3RW0Yg,18231
+requestium-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+requestium-0.5.1.dist-info/top_level.txt,sha256=6lHJKsZ2tUqzyHVJWBZkIAq0OX8aTbMCTwqCd0yHdV0,11
+requestium-0.5.1.dist-info/RECORD,,

requestium-0.5.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

requestium-0.5.1.dist-info/licenses/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2017, Tryolabs
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

requestium-0.5.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+requestium