webscout 5.6__py3-none-any.whl → 5.7__py3-none-any.whl

webscout/Provider/meta.py

@@ -8,7 +8,7 @@ from typing import Dict, Generator, Iterator, List, Union
 import random
 
 
-from requests_html import HTMLSession
+from webscout.requestsHTMLfix import HTMLSession
 import requests
 from bs4 import BeautifulSoup
 

webscout/__init__.py

@@ -3,6 +3,7 @@ from .webscout_search_async import AsyncWEBS
 from .version import __version__
 from .DWEBS import *
 from .transcriber import *
+from .requestsHTMLfix import *
 from .tempid import *
 from .websx_search import WEBSX
 from .LLM import VLM, LLM

webscout/requestsHTMLfix.py

@@ -0,0 +1,775 @@
+import sys
+import asyncio
+from urllib.parse import urlparse, urlunparse, urljoin
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures._base import TimeoutError
+from functools import partial
+from typing import Set, Union, List, MutableMapping, Optional
+
+import pyppeteer
+import requests
+from pyquery import PyQuery
+
+from fake_useragent import UserAgent
+from lxml_html_clean import Cleaner
+import lxml
+from lxml import etree
+from lxml.html import HtmlElement
+from lxml.html import tostring as lxml_html_tostring
+from lxml.html.soupparser import fromstring as soup_parse
+from parse import search as parse_search
+from parse import findall, Result
+from w3lib.encoding import html_to_unicode
+
+DEFAULT_ENCODING = 'utf-8'
+DEFAULT_URL = 'https://example.org/'
+DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8'
+DEFAULT_NEXT_SYMBOL = ['next', 'more', 'older']
+
+cleaner = Cleaner()
+cleaner.javascript = True
+cleaner.style = True
+
+useragent = None
+
+# Typing.
+_Find = Union[List['Element'], 'Element']
+_XPath = Union[List[str], List['Element'], str, 'Element']
+_Result = Union[List['Result'], 'Result']
+_HTML = Union[str, bytes]
+_BaseHTML = str
+_UserAgent = str
+_DefaultEncoding = str
+_URL = str
+_RawHTML = bytes
+_Encoding = str
+_LXML = HtmlElement
+_Text = str
+_Search = Result
+_Containing = Union[str, List[str]]
+_Links = Set[str]
+_Attrs = MutableMapping
+_Next = Union['HTML', List[str]]
+_NextSymbol = List[str]
+
+# Sanity checking.
+try:
+    assert sys.version_info.major == 3
+    assert sys.version_info.minor > 5
+except AssertionError:
+    raise RuntimeError('Requests-HTML requires Python 3.6+!')
+
+
+class MaxRetries(Exception):
+
+    def __init__(self, message):
+        self.message = message
+
+
+class BaseParser:
+    """A basic HTML/Element Parser, for Humans.
+
+    :param element: The element from which to base the parsing upon.
+    :param default_encoding: Which encoding to default to.
+    :param html: HTML from which to base the parsing upon (optional).
+    :param url: The URL from which the HTML originated, used for ``absolute_links``.
+
+    """
+
+    def __init__(self, *, element, default_encoding: _DefaultEncoding = None, html: _HTML = None, url: _URL) -> None:
+        self.element = element
+        self.url = url
+        self.skip_anchors = True
+        self.default_encoding = default_encoding
+        self._encoding = None
+        self._html = html.encode(DEFAULT_ENCODING) if isinstance(html, str) else html
+        self._lxml = None
+        self._pq = None
+
+    @property
+    def raw_html(self) -> _RawHTML:
+        """Bytes representation of the HTML content.
+        (`learn more <http://www.diveintopython3.net/strings.html>`_).
+        """
+        if self._html:
+            return self._html
+        else:
+            return etree.tostring(self.element, encoding='unicode').strip().encode(self.encoding)
+
+    @property
+    def html(self) -> _BaseHTML:
+        """Unicode representation of the HTML content
+        (`learn more <http://www.diveintopython3.net/strings.html>`_).
+        """
+        if self._html:
+            return self.raw_html.decode(self.encoding, errors='replace')
+        else:
+            return etree.tostring(self.element, encoding='unicode').strip()
+
+    @html.setter
+    def html(self, html: str) -> None:
+        self._html = html.encode(self.encoding)
+
+    @raw_html.setter
+    def raw_html(self, html: bytes) -> None:
+        """Property setter for self.html."""
+        self._html = html
+
+    @property
+    def encoding(self) -> _Encoding:
+        """The encoding string to be used, extracted from the HTML and
+        :class:`HTMLResponse <HTMLResponse>` headers.
+        """
+        if self._encoding:
+            return self._encoding
+
+        # Scan meta tags for charset.
+        if self._html:
+            self._encoding = html_to_unicode(self.default_encoding, self._html)[0]
+            # Fall back to requests' detected encoding if decode fails.
+            try:
+                self.raw_html.decode(self.encoding, errors='replace')
+            except UnicodeDecodeError:
+                self._encoding = self.default_encoding
+
+
+        return self._encoding if self._encoding else self.default_encoding
+
+    @encoding.setter
+    def encoding(self, enc: str) -> None:
+        """Property setter for self.encoding."""
+        self._encoding = enc
+
+    @property
+    def pq(self) -> PyQuery:
+        """`PyQuery <https://pythonhosted.org/pyquery/>`_ representation
+        of the :class:`Element <Element>` or :class:`HTML <HTML>`.
+        """
+        if self._pq is None:
+            self._pq = PyQuery(self.lxml)
+
+        return self._pq
+
+    @property
+    def lxml(self) -> HtmlElement:
+        """`lxml <http://lxml.de>`_ representation of the
+        :class:`Element <Element>` or :class:`HTML <HTML>`.
+        """
+        if self._lxml is None:
+            try:
+                self._lxml = soup_parse(self.html, features='html.parser')
+            except ValueError:
+                self._lxml = lxml.html.fromstring(self.raw_html)
+
+        return self._lxml
+
+    @property
+    def text(self) -> _Text:
+        """The text content of the
+        :class:`Element <Element>` or :class:`HTML <HTML>`.
+        """
+        return self.pq.text()
+
+    @property
+    def full_text(self) -> _Text:
+        """The full text content (including links) of the
+        :class:`Element <Element>` or :class:`HTML <HTML>`.
+        """
+        return self.lxml.text_content()
+
+    def find(self, selector: str = "*", *, containing: _Containing = None, clean: bool = False, first: bool = False, _encoding: str = None) -> _Find:
+        """Given a CSS Selector, returns a list of
+        :class:`Element <Element>` objects or a single one.
+
+        :param selector: CSS Selector to use.
+        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
+        :param containing: If specified, only return elements that contain the provided text.
+        :param first: Whether or not to return just the first result.
+        :param _encoding: The encoding format.
+
+        Example CSS Selectors:
+
+        - ``a``
+        - ``a.someClass``
+        - ``a#someID``
+        - ``a[target=_blank]``
+
+        See W3School's `CSS Selectors Reference
+        <https://www.w3schools.com/cssref/css_selectors.asp>`_
+        for more details.
+
+        If ``first`` is ``True``, only returns the first
+        :class:`Element <Element>` found.
+        """
+
+        # Convert a single containing into a list.
+        if isinstance(containing, str):
+            containing = [containing]
+
+        encoding = _encoding or self.encoding
+        elements = [
+            Element(element=found, url=self.url, default_encoding=encoding)
+            for found in self.pq(selector)
+        ]
+
+        if containing:
+            elements_copy = elements.copy()
+            elements = []
+
+            for element in elements_copy:
+                if any([c.lower() in element.full_text.lower() for c in containing]):
+                    elements.append(element)
+
+            elements.reverse()
+
+        # Sanitize the found HTML.
+        if clean:
+            elements_copy = elements.copy()
+            elements = []
+
+            for element in elements_copy:
+                element.raw_html = lxml_html_tostring(cleaner.clean_html(element.lxml))
+                elements.append(element)
+
+        return _get_first_or_list(elements, first)
+
+    def xpath(self, selector: str, *, clean: bool = False, first: bool = False, _encoding: str = None) -> _XPath:
+        """Given an XPath selector, returns a list of
+        :class:`Element <Element>` objects or a single one.
+
+        :param selector: XPath Selector to use.
+        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
+        :param first: Whether or not to return just the first result.
+        :param _encoding: The encoding format.
+
+        If a sub-selector is specified (e.g. ``//a/@href``), a simple
+        list of results is returned.
+
+        See W3School's `XPath Examples
+        <https://www.w3schools.com/xml/xpath_examples.asp>`_
+        for more details.
+
+        If ``first`` is ``True``, only returns the first
+        :class:`Element <Element>` found.
+        """
+        selected = self.lxml.xpath(selector)
+
+        elements = [
+            Element(element=selection, url=self.url, default_encoding=_encoding or self.encoding)
+            if not isinstance(selection, etree._ElementUnicodeResult) else str(selection)
+            for selection in selected
+        ]
+
+        # Sanitize the found HTML.
+        if clean:
+            elements_copy = elements.copy()
+            elements = []
+
+            for element in elements_copy:
+                element.raw_html = lxml_html_tostring(cleaner.clean_html(element.lxml))
+                elements.append(element)
+
+        return _get_first_or_list(elements, first)
+
+    def search(self, template: str) -> Result:
+        """Search the :class:`Element <Element>` for the given Parse template.
+
+        :param template: The Parse template to use.
+        """
+
+        return parse_search(template, self.html)
+
+    def search_all(self, template: str) -> _Result:
+        """Search the :class:`Element <Element>` (multiple times) for the given parse
+        template.
+
+        :param template: The Parse template to use.
+        """
+        return [r for r in findall(template, self.html)]
+
+    @property
+    def links(self) -> _Links:
+        """All found links on page, in as–is form."""
+
+        def gen():
+            for link in self.find('a'):
+
+                try:
+                    href = link.attrs['href'].strip()
+                    if href and not (href.startswith('#') and self.skip_anchors) and not href.startswith(('javascript:', 'mailto:')):
+                        yield href
+                except KeyError:
+                    pass
+
+        return set(gen())
+
+    def _make_absolute(self, link):
+        """Makes a given link absolute."""
+
+        # Parse the link with stdlib.
+        parsed = urlparse(link)._asdict()
+
+        # If link is relative, then join it with base_url.
+        if not parsed['netloc']:
+            return urljoin(self.base_url, link)
+
+        # Link is absolute; if it lacks a scheme, add one from base_url.
+        if not parsed['scheme']:
+            parsed['scheme'] = urlparse(self.base_url).scheme
+
+            # Reconstruct the URL to incorporate the new scheme.
+            parsed = (v for v in parsed.values())
+            return urlunparse(parsed)
+
+        # Link is absolute and complete with scheme; nothing to be done here.
+        return link
+
+
+    @property
+    def absolute_links(self) -> _Links:
+        """All found links on page, in absolute form
+        (`learn more <https://www.navegabem.com/absolute-or-relative-links.html>`_).
+        """
+
+        def gen():
+            for link in self.links:
+                yield self._make_absolute(link)
+
+        return set(gen())
+
+    @property
+    def base_url(self) -> _URL:
+        """The base URL for the page. Supports the ``<base>`` tag
+        (`learn more <https://www.w3schools.com/tags/tag_base.asp>`_)."""
+
+        # Support for <base> tag.
+        base = self.find('base', first=True)
+        if base:
+            result = base.attrs.get('href', '').strip()
+            if result:
+                return result
+
+        # Parse the url to separate out the path
+        parsed = urlparse(self.url)._asdict()
+
+        # Remove any part of the path after the last '/'
+        parsed['path'] = '/'.join(parsed['path'].split('/')[:-1]) + '/'
+
+        # Reconstruct the url with the modified path
+        parsed = (v for v in parsed.values())
+        url = urlunparse(parsed)
+
+        return url
+
+
+class Element(BaseParser):
+    """An element of HTML.
+
+    :param element: The element from which to base the parsing upon.
+    :param url: The URL from which the HTML originated, used for ``absolute_links``.
+    :param default_encoding: Which encoding to default to.
+    """
+
+    __slots__ = [
+        'element', 'url', 'skip_anchors', 'default_encoding', '_encoding',
+        '_html', '_lxml', '_pq', '_attrs', 'session'
+    ]
+
+    def __init__(self, *, element, url: _URL, default_encoding: _DefaultEncoding = None) -> None:
+        super(Element, self).__init__(element=element, url=url, default_encoding=default_encoding)
+        self.element = element
+        self.tag = element.tag
+        self.lineno = element.sourceline
+        self._attrs = None
+
+    def __repr__(self) -> str:
+        attrs = ['{}={}'.format(attr, repr(self.attrs[attr])) for attr in self.attrs]
+        return "<Element {} {}>".format(repr(self.element.tag), ' '.join(attrs))
+
+    @property
+    def attrs(self) -> _Attrs:
+        """Returns a dictionary of the attributes of the :class:`Element <Element>`
+        (`learn more <https://www.w3schools.com/tags/ref_attributes.asp>`_).
+        """
+        if self._attrs is None:
+            self._attrs = {k: v for k, v in self.element.items()}
+
+            # Split class and rel up, as there are ussually many of them:
+            for attr in ['class', 'rel']:
+                if attr in self._attrs:
+                    self._attrs[attr] = tuple(self._attrs[attr].split())
+
+        return self._attrs
+
+
+class HTML(BaseParser):
+    """An HTML document, ready for parsing.
+
+    :param url: The URL from which the HTML originated, used for ``absolute_links``.
+    :param html: HTML from which to base the parsing upon (optional).
+    :param default_encoding: Which encoding to default to.
+    """
+
+    def __init__(self, *, session: Union['HTMLSession', 'AsyncHTMLSession'] = None, url: str = DEFAULT_URL, html: _HTML, default_encoding: str = DEFAULT_ENCODING, async_: bool = False) -> None:
+
+        # Convert incoming unicode HTML into bytes.
+        if isinstance(html, str):
+            html = html.encode(DEFAULT_ENCODING)
+
+        super(HTML, self).__init__(
+            # Convert unicode HTML to bytes.
+            element=PyQuery(html)('html') or PyQuery(f'<html>{html}</html>')('html'),
+            html=html,
+            url=url,
+            default_encoding=default_encoding
+        )
+        self.session = session or async_ and AsyncHTMLSession() or HTMLSession()
+        self.page = None
+        self.next_symbol = DEFAULT_NEXT_SYMBOL
+
+    def __repr__(self) -> str:
+        return f"<HTML url={self.url!r}>"
+
+    def next(self, fetch: bool = False, next_symbol: _NextSymbol = DEFAULT_NEXT_SYMBOL) -> _Next:
+        """Attempts to find the next page, if there is one. If ``fetch``
+        is ``True`` (default), returns :class:`HTML <HTML>` object of
+        next page. If ``fetch`` is ``False``, simply returns the next URL.
+
+        """
+
+        def get_next():
+            candidates = self.find('a', containing=next_symbol)
+
+            for candidate in candidates:
+                if candidate.attrs.get('href'):
+                    # Support 'next' rel (e.g. reddit).
+                    if 'next' in candidate.attrs.get('rel', []):
+                        return candidate.attrs['href']
+
+                    # Support 'next' in classnames.
+                    for _class in candidate.attrs.get('class', []):
+                        if 'next' in _class:
+                            return candidate.attrs['href']
+
+                    if 'page' in candidate.attrs['href']:
+                        return candidate.attrs['href']
+
+            try:
+                # Resort to the last candidate.
+                return candidates[-1].attrs['href']
+            except IndexError:
+                return None
+
+        __next = get_next()
+        if __next:
+            url = self._make_absolute(__next)
+        else:
+            return None
+
+        if fetch:
+            return self.session.get(url)
+        else:
+            return url
+
+    def __iter__(self):
+
+        next = self
+
+        while True:
+            yield next
+            try:
+                next = next.next(fetch=True, next_symbol=self.next_symbol).html
+            except AttributeError:
+                break
+
+    def __next__(self):
+        return self.next(fetch=True, next_symbol=self.next_symbol).html
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        while True:
+            url = self.next(fetch=False, next_symbol=self.next_symbol)
+            if not url:
+                break
+            response = await self.session.get(url)
+            return response.html
+
+    def add_next_symbol(self, next_symbol):
+        self.next_symbol.append(next_symbol)
+
+    async def _async_render(self, *, url: str, script: str = None, scrolldown, sleep: int, wait: float, reload, content: Optional[str], timeout: Union[float, int], keep_page: bool):
+        """ Handle page creation and js rendering. Internal use for render/arender methods. """
+        try:
+            page = await self.browser.newPage()
+
+            # Wait before rendering the page, to prevent timeouts.
+            await asyncio.sleep(wait)
+
+            # Load the given page (GET request, obviously.)
+            if reload:
+                await page.goto(url, options={'timeout': int(timeout * 1000)})
+            else:
+                await page.goto(f'data:text/html,{self.html}', options={'timeout': int(timeout * 1000)})
+
+            result = None
+            if script:
+                result = await page.evaluate(script)
+
+            if scrolldown:
+                for _ in range(scrolldown):
+                    await page._keyboard.down('PageDown')
+                    await asyncio.sleep(sleep)
+            else:
+                await asyncio.sleep(sleep)
+
+            if scrolldown:
+                await page._keyboard.up('PageDown')
+
+            # Return the content of the page, JavaScript evaluated.
+            content = await page.content()
+            if not keep_page:
+                await page.close()
+                page = None
+            return content, result, page
+        except TimeoutError:
+            await page.close()
+            page = None
+            return None
+
+    def render(self, retries: int = 8, script: str = None, wait: float = 0.2, scrolldown=False, sleep: int = 0, reload: bool = True, timeout: Union[float, int] = 8.0, keep_page: bool = False):
+        """Reloads the response in Chromium, and replaces HTML content
+        with an updated version, with JavaScript executed.
+
+        :param retries: The number of times to retry loading the page in Chromium.
+        :param script: JavaScript to execute upon page load (optional).
+        :param wait: The number of seconds to wait before loading the page, preventing timeouts (optional).
+        :param scrolldown: Integer, if provided, of how many times to page down.
+        :param sleep: Integer, if provided, of how many long to sleep after initial render.
+        :param reload: If ``False``, content will not be loaded from the browser, but will be provided from memory.
+        :param keep_page: If ``True`` will allow you to interact with the browser page through ``r.html.page``.
+
+        If ``scrolldown`` is specified, the page will scrolldown the specified
+        number of times, after sleeping the specified amount of time
+        (e.g. ``scrolldown=10, sleep=1``).
+
+        If just ``sleep`` is provided, the rendering will wait *n* seconds, before
+        returning.
+
+        If ``script`` is specified, it will execute the provided JavaScript at
+        runtime. Example:
+
+        .. code-block:: python
+
+            script = \"\"\"
+                () => {
+                    return {
+                        width: document.documentElement.clientWidth,
+                        height: document.documentElement.clientHeight,
+                        deviceScaleFactor: window.devicePixelRatio,
+                    }
+                }
+            \"\"\"
+
+        Returns the return value of the executed  ``script``, if any is provided:
+
+        .. code-block:: python
+
+            >>> r.html.render(script=script)
+            {'width': 800, 'height': 600, 'deviceScaleFactor': 1}
+
+        Warning: the first time you run this method, it will download
+        Chromium into your home directory (``~/.pyppeteer``).
+        """
+
+        self.browser = self.session.browser  # Automatically create a event loop and browser
+        content = None
+
+        # Automatically set Reload to False, if example URL is being used.
+        if self.url == DEFAULT_URL:
+            reload = False
+
+
+        for i in range(retries):
+            if not content:
+                try:
+
+                    content, result, page = self.session.loop.run_until_complete(self._async_render(url=self.url, script=script, sleep=sleep, wait=wait, content=self.html, reload=reload, scrolldown=scrolldown, timeout=timeout, keep_page=keep_page))
+                except TypeError:
+                    pass
+            else:
+                break
+
+        if not content:
+            raise MaxRetries("Unable to render the page. Try increasing timeout")
+
+        html = HTML(url=self.url, html=content.encode(DEFAULT_ENCODING), default_encoding=DEFAULT_ENCODING)
+        self.__dict__.update(html.__dict__)
+        self.page = page
+        return result
+
+    async def arender(self, retries: int = 8, script: str = None, wait: float = 0.2, scrolldown=False, sleep: int = 0, reload: bool = True, timeout: Union[float, int] = 8.0, keep_page: bool = False):
+        """ Async version of render. Takes same parameters. """
+
+        self.browser = await self.session.browser
+        content = None
+
+        # Automatically set Reload to False, if example URL is being used.
+        if self.url == DEFAULT_URL:
+            reload = False
+
+        for _ in range(retries):
+            if not content:
+                try:
+
+                    content, result, page = await self._async_render(url=self.url, script=script, sleep=sleep, wait=wait, content=self.html, reload=reload, scrolldown=scrolldown, timeout=timeout, keep_page=keep_page)
+                except TypeError:
+                    pass
+            else:
+                break
+
+        if not content:
+            raise MaxRetries("Unable to render the page. Try increasing timeout")
+
+        html = HTML(url=self.url, html=content.encode(DEFAULT_ENCODING), default_encoding=DEFAULT_ENCODING)
+        self.__dict__.update(html.__dict__)
+        self.page = page
+        return result
+
+
+class HTMLResponse(requests.Response):
+    """An HTML-enabled :class:`requests.Response <requests.Response>` object.
+    Effectively the same, but with an intelligent ``.html`` property added.
+    """
+
+    def __init__(self, session: Union['HTMLSession', 'AsyncHTMLSession']) -> None:
+        super(HTMLResponse, self).__init__()
+        self._html = None  # type: HTML
+        self.session = session
+
+    @property
+    def html(self) -> HTML:
+        if not self._html:
+            self._html = HTML(session=self.session, url=self.url, html=self.content, default_encoding=self.encoding)
+
+        return self._html
+
+    @classmethod
+    def _from_response(cls, response, session: Union['HTMLSession', 'AsyncHTMLSession']):
+        html_r = cls(session=session)
+        html_r.__dict__.update(response.__dict__)
+        return html_r
+
+
+def user_agent(style=None) -> _UserAgent:
+    """Returns an apparently legit user-agent, if not requested one of a specific
+    style. Defaults to a Chrome-style User-Agent.
+    """
+    global useragent
+    if (not useragent) and style:
+        useragent = UserAgent()
+
+    return useragent[style] if style else DEFAULT_USER_AGENT
+
+
+def _get_first_or_list(l, first=False):
+    if first:
+        try:
+            return l[0]
+        except IndexError:
+            return None
+    else:
+        return l
+
+
+class BaseSession(requests.Session):
+    """ A consumable session, for cookie persistence and connection pooling,
+    amongst other things.
+    """
+
+    def __init__(self, mock_browser : bool = True, verify : bool = True,
+                 browser_args : list = ['--no-sandbox']):
+        super().__init__()
+
+        # Mock a web browser's user agent.
+        if mock_browser:
+            self.headers['User-Agent'] = user_agent()
+
+        self.hooks['response'].append(self.response_hook)
+        self.verify = verify
+
+        self.__browser_args = browser_args
+
+
+    def response_hook(self, response, **kwargs) -> HTMLResponse:
+        """ Change response enconding and replace it by a HTMLResponse. """
+        if not response.encoding:
+            response.encoding = DEFAULT_ENCODING
+        return HTMLResponse._from_response(response, self)
+
+    @property
+    async def browser(self):
+        if not hasattr(self, "_browser"):
+            self._browser = await pyppeteer.launch(ignoreHTTPSErrors=not(self.verify), headless=True, args=self.__browser_args)
+
+        return self._browser
+
+
+class HTMLSession(BaseSession):
+
+    def __init__(self, **kwargs):
+        super(HTMLSession, self).__init__(**kwargs)
+
+    @property
+    def browser(self):
+        if not hasattr(self, "_browser"):
+            self.loop = asyncio.get_event_loop()
+            if self.loop.is_running():
+                raise RuntimeError("Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.")
+            self._browser = self.loop.run_until_complete(super().browser)
+        return self._browser
+
+    def close(self):
+        """ If a browser was created close it first. """
+        if hasattr(self, "_browser"):
+            self.loop.run_until_complete(self._browser.close())
+        super().close()
+
+
+class AsyncHTMLSession(BaseSession):
+    """ An async consumable session. """
+
+    def __init__(self, loop=None, workers=None,
+                 mock_browser: bool = True, *args, **kwargs):
+        """ Set or create an event loop and a thread pool.
+
+            :param loop: Asyncio loop to use.
+            :param workers: Amount of threads to use for executing async calls.
+                If not pass it will default to the number of processors on the
+                machine, multiplied by 5. """
+        super().__init__(*args, **kwargs)
+
+        self.loop = loop or asyncio.get_event_loop()
+        self.thread_pool = ThreadPoolExecutor(max_workers=workers)
+
+    def request(self, *args, **kwargs):
+        """ Partial original request func and run it in a thread. """
+        func = partial(super().request, *args, **kwargs)
+        return self.loop.run_in_executor(self.thread_pool, func)
+
+    async def close(self):
+        """ If a browser was created close it first. """
+        if hasattr(self, "_browser"):
+            await self._browser.close()
+        super().close()
+
+    def run(self, *coros):
+        """ Pass in all the coroutines you want to run, it will wrap each one
+            in a task, run it and wait for the result. Return a list with all
+            results, this is returned in the same order coros are passed in. """
+        tasks = [
+            asyncio.ensure_future(coro()) for coro in coros
+        ]
+        done, _ = self.loop.run_until_complete(asyncio.wait(tasks))
+        return [t.result() for t in done]

webscout/version.py

@@ -1,2 +1,2 @@
-__version__ = "5.6"
+__version__ = "5.7"
 __prog__ = "webscout"

{webscout-5.6.dist-info → webscout-5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 5.6
+Version: 5.7
 Summary: Search for anything using Google, DuckDuckGo, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
 Author: OEvortex
 Author-email: helpingai5@gmail.com
@@ -1382,7 +1382,7 @@ if "error" not in function_call_data:
 else:
     print(f"Error: {function_call_data['error']}")
 ```
-###  LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, Editee, AI21, Chatify, Cerebras, X0GPT, Lepton, GEMINIAPI, Cleeai, Elmo, Genspark, Upstage, Free2GPT, Bing, DiscordRocks
+###  LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, Editee, AI21, Chatify, Cerebras, X0GPT, Lepton, GEMINIAPI, Cleeai, Elmo, Genspark, Upstage, Free2GPT, Bing, DiscordRocks,  GPTWeb, AIGameIO
 code similar to other provider
 ### `LLM` 
 ```python

{webscout-5.6.dist-info → webscout-5.7.dist-info}/RECORD

@@ -6,16 +6,17 @@ webscout/Bing_search.py,sha256=8pW3ZxFDfVsrtayEoAsAAoXoOCAGv2Jk_-HvOBNfCV4,6228
 webscout/DWEBS.py,sha256=xB3L_u47vu9KZCFNUb_H6WX5OLeL068Er9t9Mbss9YU,7895
 webscout/LLM.py,sha256=LbGCZdJf8A5dwfoGS4tyy39tAh5BDdhMZP0ScKaaQfU,4184
 webscout/YTdownloader.py,sha256=qVROwMmRpHy95NsbzZvpidEKuqMwFTCi7bW4jJgbl44,39152
-webscout/__init__.py,sha256=A4VcaiWmm02GvbDKCmf6F0-jRW_uT_BJkZhN6m6O108,1294
+webscout/__init__.py,sha256=mQpneQRVaRWoSzFIBcdQ6utnlOtVT6bZ0lyfehkQG-4,1326
 webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
 webscout/cli.py,sha256=RlBKeS9CSIsiBMqlzxevWtKjbY9htkZvA7J0bM_hHE8,14999
 webscout/exceptions.py,sha256=jqrZvxV2JnZFqvmCsKG1lQRd6fYQh7qk3ceA7m7-k9k,611
 webscout/g4f.py,sha256=NNcnlOtIWV9R93UsBN4jBGBEJ9sJ-Np1WbgjkGVDcYc,24487
 webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
+webscout/requestsHTMLfix.py,sha256=LZ9uynkwpGqVS2uFB5bMjdh6gLjSMerHPEqzVCvICIs,27961
 webscout/tempid.py,sha256=2a_YDFUCZy3TJVfqEPSLyMvHBjDk9gp5FiFCvJwsOaw,7984
 webscout/transcriber.py,sha256=kRymTd69zCjXdduCf7Gabt93Kz1l5ubsCwfwqs-pHl8,22334
 webscout/utils.py,sha256=2O8_lftBKsv5OEvVaXCN-h0sipup0m3jxzhFdWQrdY8,2873
-webscout/version.py,sha256=KI08BdmiHfkATxodx2s7Sg3CRosdzWRVqGfRbT9rh7c,44
+webscout/version.py,sha256=ux9lFFWTvtNI7NqdQnLgTJDCSAxKDe3rr2msA8MXkno,44
 webscout/webai.py,sha256=MzDWMFv8oI3wmeL3SwvxUrZdBRyh9MMn97kNTvgL5T4,87744
 webscout/webscout_search.py,sha256=AOHkaMLmFvM1mS--wVyLiPrDAb5BPLaURBHjleWOi4w,43743
 webscout/webscout_search_async.py,sha256=2-RCa9Deahhw3Bti78kXfVaX8y3Aygy4L7HeCaITk9M,14519
@@ -80,7 +81,7 @@ webscout/Provider/genspark.py,sha256=X3l0Kuh1fwn7XHFDzKlV4kqc35iRouUC54UOi0snNjk
 webscout/Provider/julius.py,sha256=DS2jsEz_McEb_Ma57ji6w9hRMoYycOGxfjFkh9ZUXLw,9883
 webscout/Provider/koala.py,sha256=qBtqjTvhoMQdDE8qUH0XuNa_x2bic77d7CUjIVboask,10106
 webscout/Provider/lepton.py,sha256=4RiQ4YNJljX558yhSUqws6pf1Yhf7pWIesa4SRQCry8,7590
-webscout/Provider/meta.py,sha256=PbqijyaLC7BFyI-T6gYgOpue_nMASgAUwM21SLHVkQs,30706
+webscout/Provider/meta.py,sha256=TpoStt2nTUTCYxx4GI3IOQ7gX_Mi36pL7HhtjPvLtZQ,30717
 webscout/Provider/turboseek.py,sha256=msIDtdi7-7UbjdTyzXFZgpOt0_MAWRpEac8O7N0SsB8,8539
 webscout/Provider/upstage.py,sha256=D2QSj7LcBsKyjp230LYdGwY6eO2j__wtK9d-8swjSW0,9204
 webscout/Provider/x0gpt.py,sha256=jxQWGE0E1KjJCZas2BkRmNRWrijXbFAsE2amkPExcVw,6504
@@ -95,9 +96,9 @@ webscout/Provider/TTI/deepinfra.py,sha256=4x3knEeQnjhan5Y6uogK3dXFVa3r8UjN51n91W
 webscout/Provider/TTS/__init__.py,sha256=g19AOO1X9Qb-MNFpwhx5lODDWQiG7HxZCO9TmOYOHGc,54
 webscout/Provider/TTS/streamElements.py,sha256=aaE55W_bAzvL3-pl7tHs_xU7dSZB5_V7ix-usoe-4aM,7398
 webscout/Provider/TTS/voicepod.py,sha256=wAAnpcpDyej72UjIRhEnEmcoJnUqp1lPCLiiwJMYJic,4343
-webscout-5.6.dist-info/LICENSE.md,sha256=9P0imsudI7MEvZe2pOcg8rKBn6E5FGHQ-riYozZI-Bk,2942
-webscout-5.6.dist-info/METADATA,sha256=f9-yjy9WcVYYnrr8L-zve1dbnXH1eVnW5DKfDP0BszQ,48118
-webscout-5.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-webscout-5.6.dist-info/entry_points.txt,sha256=Hh4YIIjvkqB9SVxZ2ri4DZUkgEu_WF_5_r_nZDIvfG8,73
-webscout-5.6.dist-info/top_level.txt,sha256=nYIw7OKBQDr_Z33IzZUKidRD3zQEo8jOJYkMVMeN334,9
-webscout-5.6.dist-info/RECORD,,
+webscout-5.7.dist-info/LICENSE.md,sha256=9P0imsudI7MEvZe2pOcg8rKBn6E5FGHQ-riYozZI-Bk,2942
+webscout-5.7.dist-info/METADATA,sha256=vpcKSqgYoSDcFFIrd3x0w6VWRHu8F5JpnQ4EMMvVPhg,48137
+webscout-5.7.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+webscout-5.7.dist-info/entry_points.txt,sha256=Hh4YIIjvkqB9SVxZ2ri4DZUkgEu_WF_5_r_nZDIvfG8,73
+webscout-5.7.dist-info/top_level.txt,sha256=nYIw7OKBQDr_Z33IzZUKidRD3zQEo8jOJYkMVMeN334,9
+webscout-5.7.dist-info/RECORD,,