webscout 4.8__py3-none-any.whl → 4.9__py3-none-any.whl

webscout/Provider/xdash.py

@@ -0,0 +1,202 @@
+import time
+import uuid
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+import click
+import requests
+from requests import get
+from uuid import uuid4
+from re import findall
+from requests.exceptions import RequestException
+from curl_cffi.requests import get, RequestsError
+import g4f
+from random import randint
+from PIL import Image
+import io
+import re
+import json
+import yaml
+from webscout.AIutel import Optimizers
+from webscout.AIutel import Conversation
+from webscout.AIutel import AwesomePrompts, sanitize_stream
+from webscout.AIbase import Provider, AsyncProvider
+from webscout import exceptions
+from typing import Any, AsyncGenerator, Dict
+import logging
+import httpx
+
+class XDASH(Provider):
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 600,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+    ):
+        """Instantiates XDASH
+
+        Args:
+            is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
+            max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
+            timeout (int, optional): Http request timeout. Defaults to 30.
+            intro (str, optional): Conversation introductory prompt. Defaults to None.
+            filepath (str, optional): Path to file containing conversation history. Defaults to None.
+            update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
+            proxies (dict, optional): Http request proxies. Defaults to {}.
+            history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
+            act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
+        """
+        self.session = requests.Session()
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.chat_endpoint = "https://www.xdash.ai/api/query"
+        self.stream_chunk_size = 64
+        self.timeout = timeout
+        self.last_response = {}
+        self.headers = {
+            "accept": "*/*",
+            "accept-encoding": "gzip, deflate, br, zstd",
+            "accept-language": "en-US,en;q=0.9,en-IN;q=0.8",
+            "content-type": "application/json",
+            "cookie": "cf_clearance=73aup_8JU0LU.tRr7D4qd4Kt7gapKFi3RVW8jLzQoP0-1723549451-1.0.1.1-HTRrjMvM5GRLsfCTB0v3N_UxQzQMfA1fvOSf0dsZJ73HR6.IUTH8BH.G1dpx3s_IxVHCBCHMXOCt0K7vyIwMgw",
+            "dnt": "1",
+            "origin": "https://www.xdash.ai",
+            "priority": "u=1, i",
+            "referer": "https://www.xdash.ai/search",
+            "sec-ch-ua": '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+            "sec-fetch-dest": "empty",
+            "sec-fetch-mode": "cors",
+            "sec-fetch-site": "same-origin",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0"
+        }
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        self.session.headers.update(self.headers)
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> dict:
+        """Chat with AI
+
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            raw (bool, optional): Stream back raw response as received. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+           dict : {}
+        ```json
+        {
+           "text" : "How may I assist you today?"
+        }
+        ```
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        self.session.headers.update(self.headers)
+        payload = {
+            "query": conversation_prompt,
+            "search_uuid": uuid.uuid4().hex,
+            "visitor_uuid": uuid.uuid4().hex,
+            "token": uuid.uuid4().hex
+        }
+
+        response = self.session.post(
+            self.chat_endpoint, json=payload, timeout=self.timeout
+        )
+        if not response.ok:
+            raise exceptions.FailedToGenerateResponseError(
+                f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
+            )
+
+        # Extract the LLM response
+        llm_response = response.text.split("__LLM_RESPONSE__")[1].split("__RELATED_QUESTIONS__")[0].strip()
+        resp = dict(text=llm_response)
+        self.last_response.update(resp)
+        self.conversation.update_chat_history(
+            prompt, self.get_message(self.last_response)
+        )
+        return self.last_response
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """Generate response `str`
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+            str: Response generated
+        """
+
+        return self.get_message(
+            self.ask(
+                prompt,
+                optimizer=optimizer,
+                conversationally=conversationally,
+            )
+        )
+
+    def get_message(self, response: dict) -> str:
+        """Retrieves message only from response
+
+        Args:
+            response (dict): Response generated by `self.ask`
+
+        Returns:
+            str: Message extracted
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response["text"]
+if __name__ == '__main__':
+    from rich import print
+    ai = XDASH()
+    response = ai.chat(input(">>> "))
+    print(response)

webscout/Provider/yep.py

@@ -0,0 +1,258 @@
+import time
+import uuid
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+import click
+import requests
+from requests import get
+from uuid import uuid4
+from re import findall
+from requests.exceptions import RequestException
+from curl_cffi.requests import get, RequestsError
+import g4f
+from random import randint
+from PIL import Image
+import io
+import re
+import json
+import yaml
+from webscout.AIutel import Optimizers
+from webscout.AIutel import Conversation
+from webscout.AIutel import AwesomePrompts, sanitize_stream
+from webscout.AIbase import  Provider, AsyncProvider
+from webscout import exceptions
+from typing import Any, AsyncGenerator, Dict
+import logging
+import httpx
+import cloudscraper
+
+class YEPCHAT(Provider):
+    """
+    This class provides methods for interacting with the Yep.com chat API in a consistent provider structure.
+    """
+
+    AVAILABLE_MODELS = [
+        "Mixtral-8x7B-Instruct-v0.1"  # Add other available models here
+    ]
+
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 1280,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+        model: str = "Mixtral-8x7B-Instruct-v0.1",
+        temperature: float = 0.6,
+        top_p: float = 0.7,
+    ):
+        """Instantiates YEPCHAT
+
+        Args:
+            is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
+            max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 1280.
+            timeout (int, optional): Http request timeout. Defaults to 30.
+            intro (str, optional): Conversation introductory prompt. Defaults to None.
+            filepath (str, optional): Path to file containing conversation history. Defaults to None.
+            update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
+            proxies (dict, optional): Http request proxies. Defaults to {}.
+            history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
+            act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
+            model (str, optional): Model to use for generating text. Defaults to "Mixtral-8x7B-Instruct-v0.1".
+            temperature (float, optional): Temperature parameter for the model. Defaults to 0.6.
+            top_p (float, optional): Top_p parameter for the model. Defaults to 0.7.
+        """
+
+        if model not in self.AVAILABLE_MODELS:
+            raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
+
+        self.session = cloudscraper.create_scraper()  # Create a Cloudscraper session
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.chat_endpoint = "https://api.yep.com/v1/chat/completions"
+        self.stream_chunk_size = 64
+        self.timeout = timeout
+        self.last_response = {}
+        self.model = model
+        self.temperature = temperature
+        self.top_p = top_p
+        self.headers = {
+            'Accept': '*/*',
+            'Accept-Encoding': 'gzip, deflate, br, zstd',
+            'Accept-Language': 'en-US,en;q=0.9,en-IN;q=0.8',
+            'Content-Type': 'application/json; charset=utf-8',
+            'DNT': '1',
+            'Origin': 'https://yep.com',
+            'Referer': 'https://yep.com/',
+            'Sec-CH-UA': '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
+            'Sec-CH-UA-Mobile': '?0',
+            'Sec-CH-UA-Platform': '"Windows"',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'
+        }
+        self.cookies = {
+            '__Host-session': uuid4().hex,
+        }
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> dict:
+        """Chat with AI
+
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Whether to stream the response. Defaults to False.
+            raw (bool, optional): Whether to return the raw response. Defaults to False.
+            optimizer (str, optional): The name of the optimizer to use. Defaults to None.
+            conversationally (bool, optional): Whether to chat conversationally. Defaults to False.
+
+        Returns:
+            The response from the API.
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        data = {
+            "stream": stream,
+            "max_tokens": self.max_tokens_to_sample,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "messages": [
+                {"content": conversation_prompt, "role": "user"}
+            ],
+            "model": self.model
+        }
+
+        def for_stream():
+            response = self.session.post(
+                self.chat_endpoint, headers=self.headers, cookies=self.cookies, json=data, stream=True, timeout=self.timeout
+            )
+            if not response.ok:
+                raise exceptions.FailedToGenerateResponseError(
+                    f"Failed to generate response - ({response.status_code}, {response.reason})"
+                )
+            streaming_response = ""
+            for line in response.iter_lines(decode_unicode=True):
+                if line:
+                    if line.startswith("data: "):
+                        json_data = line[6:]
+                        if json_data == "[DONE]":
+                            break
+                        try:
+                            data = json.loads(json_data)
+                            content = data["choices"][0]["delta"].get("content", "")
+                            streaming_response += content
+                            yield content if raw else dict(text=streaming_response)
+                        except json.decoder.JSONDecodeError:
+                            continue
+            self.last_response.update(dict(text=streaming_response))
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+
+        def for_non_stream():
+            response = self.session.post(
+                self.chat_endpoint, headers=self.headers, cookies=self.cookies, json=data, timeout=self.timeout
+            )
+            if not response.ok:
+                raise exceptions.FailedToGenerateResponseError(
+                    f"Failed to generate response - ({response.status_code}, {response.reason})"
+                )
+            resp = response.json()
+            self.last_response.update(dict(text=resp['choices'][0]['message']['content']))
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+            return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """Generate response `str`
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+            str: Response generated
+        """
+
+        def for_stream():
+            for response in self.ask(
+                prompt, True, optimizer=optimizer, conversationally=conversationally
+            ):
+                yield self.get_message(response)
+
+        def for_non_stream():
+            return self.get_message(
+                self.ask(
+                    prompt,
+                    False,
+                    optimizer=optimizer,
+                    conversationally=conversationally,
+                )
+            )
+
+        return for_stream() if stream else for_non_stream()
+
+    def get_message(self, response: dict) -> str:
+        """Retrieves message only from response
+
+        Args:
+            response (dict): Response generated by `self.ask`
+
+        Returns:
+            str: Message extracted
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response["text"]
+if __name__ == '__main__':
+    from rich import print
+    ai = YEPCHAT()
+    response = ai.chat("tell me about india")
+    for chunk in response:
+        print(chunk, end="", flush=True)

webscout/__init__.py

@@ -7,7 +7,7 @@ from .voice import play_audio
 from .websx_search import WEBSX
 from .LLM import VLM, LLM
 from .YTdownloader import *
-
+from .Bing_search import *
 import g4f
 from .YTdownloader import *
 from .Provider import *
@@ -55,64 +55,6 @@ gpt4free_providers = [
 
 available_providers = webai + gpt4free_providers
 
-# Add all the provider classes, Localai models, Thread, and Model to __all__
-__all__ = [
-   "WEBS",
-   "AsyncWEBS",
-   "__version__",
-   "DWEBS",
-   "transcriber",
-   "play_audio",
-   "TempMailClient", 
-   "TemporaryPhoneNumber",
-   "LLM",
-   "YTdownloader",
-   "WEBSX",
-   "VLM",
-   # Localai models and utilities 
-   # "Model",
-   # "Thread",
-   # "formats", 
-
-   # AI Providers
-    'ThinkAnyAI',
-    'Xjai',
-    'LLAMA2', 
-    'AsyncLLAMA2',
-    'Cohere',
-    'REKA',
-    'GROQ',
-    'AsyncGROQ',
-    'OPENAI',
-    'AsyncOPENAI',
-    'LEO',
-    'AsyncLEO',
-    'KOBOLDAI',
-    'AsyncKOBOLDAI',
-    'OPENGPT', 
-    'AsyncOPENGPT',
-    'PERPLEXITY',
-    'BLACKBOXAI', 
-    'AsyncBLACKBOXAI',
-    'PhindSearch', 
-    'AsyncPhindSearch',
-    'YEPCHAT',
-    'AsyncYEPCHAT',
-    'YouChat',
-    'GEMINI',
-    'Berlin4h',
-    'ChatGPTUK',
-    'POE',
-    'BasedGPT',
-    'DeepSeek',
-    'DeepInfra',
-    'VLM',
-    'AsyncDeepInfra',
-    'VTLchat',
-    'AsyncPhindv2',
-    'Phindv2',
-    'OPENGPTv2',
-]
 
 import logging
 logging.getLogger("webscout").addHandler(logging.NullHandler())

{webscout-4.8.dist-info → webscout-4.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 4.8
+Version: 4.9
 Summary: Search for anything using Google, DuckDuckGo, brave, qwant,  phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
 Author: OEvortex
 Author-email: helpingai5@gmail.com
@@ -64,6 +64,8 @@ Requires-Dist: yaspin
 Requires-Dist: pillow
 Requires-Dist: requests-html
 Requires-Dist: bson
+Requires-Dist: cloudscraper
+Requires-Dist: emoji
 Provides-Extra: dev
 Requires-Dist: ruff >=0.1.6 ; extra == 'dev'
 Requires-Dist: pytest >=7.4.2 ; extra == 'dev'
@@ -208,7 +210,7 @@ python -m webscout --help
 
 
 [Go To TOP](#TOP)
-## YTdownloader -webscout can now download yt videos
+## YTdownloader
 
 ```python
 from os import rename, getcwd
@@ -232,7 +234,7 @@ if __name__ == "__main__":
     download_video("https://www.youtube.com/watch?v=c0tMvzB0OKw")
 ```
 
-## Weather - webscout can now forcast weather
+## Weather
 1. weather
 ```python
 from webscout import weather as w
@@ -331,6 +333,7 @@ async def main() -> None:
 if __name__ == '__main__':
     asyncio.run(main())
 ```
+
 ## Transcriber
 The transcriber function in webscout is a handy tool that transcribes YouTube videos. Here's an example code demonstrating its usage:
 ```python
@@ -387,77 +390,25 @@ if __name__ == "__main__":
     main()
 ```
 
-## DWEBS: Advanced Web Searches
-
-`DWEBS` is a standalone feature designed to perform advanced web searches with enhanced capabilities. It is particularly powerful in extracting relevant information directly from webpages and Search engine, focusing exclusively on text (web) searches. Unlike the `WEBS` , which provides a broader range of search functionalities, `DWEBS` is specifically tailored for in-depth web searches.
-
-### Activating DWEBS
-
-To utilize the `DWEBS` feature, you must first create an instance of the `DWEBS` . This is designed to be used independently of the `WEBS` , offering a focused approach to web searches.
-
-### Point to remember before using `DWEBS`
-As `DWEBS` is designed to extract relevant information directly from webpages and Search engine, It extracts html from webpages and saves them to folder named files 
-
-### Usage Example
-
-Here's a basic example of how to use the `DWEBS` :
+## GoogleS -- formerly DWEBS
 ```python
-from webscout import DWEBS
-
-def finalextractor(extract_webpage=True):
-    print('---------------Here Running for GoogleSearch--------------------')
-    # 1. Google Search
-    google_searcher = DWEBS.GoogleSearcher()
-    query_html_path = google_searcher.search(
-        query='HelpingAI-9B',
-        result_num=10,
-        safe=False,
-        overwrite=False,
-    )
-
-    # 2. Search Result Extraction
-    query_results_extractor = DWEBS.QueryResultsExtractor()
-    query_search_results = query_results_extractor.extract(query_html_path)
-
-    if extract_webpage:
-        print('---------------Batch Webpage Fetcher--------------------')
-        # 3. Batch Webpage Fetching
-        batch_webpage_fetcher = DWEBS.BatchWebpageFetcher()
-        urls = [query_extracts['url'] for query_extracts in query_search_results['query_results']]
-        url_and_html_path_list = batch_webpage_fetcher.fetch(
-            urls,
-            overwrite=False,
-            output_parent=query_search_results["query"],
-        )
-
-        print('---------------Batch Webpage Extractor--------------------')
-        # 4. Batch Webpage Content Extraction
-        batch_webpage_content_extractor = DWEBS.BatchWebpageContentExtractor()
-        webpageurls = [url_and_html['html_path'] for url_and_html in url_and_html_path_list]
-        html_path_and_extracted_content_list = batch_webpage_content_extractor.extract(webpageurls)
-
-        # 5. Printing Extracted Content
-        for html_path_and_extracted_content in html_path_and_extracted_content_list:
-            print(html_path_and_extracted_content['extracted_content'])
-    else:
-        # Print only search results if extract_webpage is False
-        for result in query_search_results['query_results']:
-            DWEBS.logger.mesg(
-                f"{result['title']}\n"
-                f" - {result['site']}\n"
-                f" - {result['url']}\n"
-                f" - {result['abstract']}\n"
-                f"\n"
-            )
-
-        DWEBS.logger.success(f"- {len(query_search_results['query_results'])} query results")
-        DWEBS.logger.success(f"- {len(query_search_results['related_questions'])} related questions")
-
-# Example usage:
-finalextractor(extract_webpage=True)  # Extract webpage content
-finalextractor(extract_webpage=False) # Skip webpage extraction and print search results only
-
+from webscout import GoogleS
+from rich import print
+searcher = GoogleS()
+results = searcher.search("HelpingAI-9B", max_results=20, extract_webpage_text=False, max_extract_characters=100)
+for result in results:
+    print(result)
 ```
+### BingS
+```python
+from webscout import BingS
+from rich import print
+searcher = BingS()
+results = searcher.search("Python development tools", max_results=30)
+for result in results:
+    print(result)
+```
+
 ## Text-to-Speech:
 ```python
 from webscout import play_audio
@@ -1406,7 +1357,7 @@ from webscout import AndiSearch
 a = AndiSearch()
 print(a.chat("HelpingAI-9B"))
 ```
-### Function calling-bete
+### Function calling-beta
 ```python
 import json
 import logging
@@ -1519,7 +1470,7 @@ if "error" not in function_call_data:
 else:
     print(f"Error: {function_call_data['error']}")
 ```
-###  LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle
+###  LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek,
 code similar to other provider
 ### `LLM` 
 ```python