webscout 5.7__py3-none-any.whl → 5.8__py3-none-any.whl

webscout/AIutel.py

@@ -3,6 +3,8 @@ import json
 import platform
 import subprocess
 import logging
+import threading
+import time
 import appdirs
 import datetime
 import re
@@ -11,7 +13,7 @@ import click
 from rich.markdown import Markdown
 from rich.console import Console
 import g4f
-from typing import Union
+from typing import List, Tuple, Union
 from typing import NoReturn
 import requests
 from pathlib import Path
@@ -1053,4 +1055,76 @@ class Audio:
         """
         if not Path(path_to_audio_file).is_file():
             raise FileNotFoundError(f"File does not exist - '{path_to_audio_file}'")
-        playsound(path_to_audio_file)
+        playsound(path_to_audio_file)
+class ProxyManager:
+    def __init__(self, refresh_interval=60):
+        self.proxies: List[Tuple[str, float]] = []  # Store proxy and its latency
+        self.last_refresh: float = 0
+        self.refresh_interval = refresh_interval
+        self.lock = threading.Lock()  # Add a lock for thread safety
+        # Start auto-refresh in a separate thread
+        threading.Thread(target=self.auto_refresh_proxies, daemon=True).start()
+
+    def fetch_proxies(self, max_proxies=50) -> List[str]:
+        try:
+            url = "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all"
+            response = requests.get(url)
+            proxies = response.text.split('\r\n')[:max_proxies]  # Extract up to max_proxies
+            return [proxy for proxy in proxies if proxy]
+        except requests.RequestException as e:
+            print(f"Error fetching proxies: {e}")
+            return []
+
+    def test_proxy(self, proxy: str) -> Tuple[str, float] | None:
+        # Test both HTTP and HTTPS
+        for protocol in ['http', 'https']:
+            try:
+                start_time = time.time()
+                response = requests.get('http://httpbin.org/ip', proxies={protocol: f"{protocol}://{proxy}"}, timeout=5)
+                if response.status_code == 200:
+                    end_time = time.time()
+                    return proxy, end_time - start_time
+            except requests.RequestException:
+                pass
+        return None
+
+    def refresh_proxies(self) -> int:
+        new_proxies = self.fetch_proxies()
+        threads = []
+        working_proxies = []
+
+        # Use threading for faster proxy testing
+        for proxy in new_proxies:
+            thread = threading.Thread(target=self.test_proxy_and_append, args=(proxy, working_proxies))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        with self.lock:  # Acquire lock before updating proxies list
+            self.proxies = working_proxies
+            self.last_refresh = time.time()
+
+        # print(f"Refreshed proxies at {datetime.now()}. Total working proxies: {len(self.proxies)}")
+        return len(self.proxies)
+
+    def test_proxy_and_append(self, proxy: str, working_proxies: list):
+        result = self.test_proxy(proxy)
+        if result:
+            with self.lock:  # Acquire lock before appending to shared list
+                working_proxies.append(result)  # Append the proxy and its latency
+
+    def auto_refresh_proxies(self):
+        while True:
+            time.sleep(self.refresh_interval)
+            self.refresh_proxies()
+
+    def get_fastest_proxy(self) -> str | None:
+        with self.lock:  # Acquire lock before accessing proxies list
+            if self.proxies:
+                # Sort proxies by latency and return the fastest
+                self.proxies.sort(key=lambda x: x[1])  # Sort by latency
+                return self.proxies[0][0]  # Return the fastest proxy
+        return None

webscout/Agents/Onlinesearcher.py

@@ -1,162 +1,170 @@
 import json
-import colorlog
-from webscout import WEBS
-from webscout import DeepInfra
 import httpx
 from bs4 import BeautifulSoup
 from typing import List, Dict
-import logging
+from webscout import WEBS, GEMINIAPI
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 
 class WebSearchAgent:
     def __init__(self):
         self.webs = WEBS()
-        self.ai = DeepInfra(is_conversation=False)
-
-    def generate_search_queries(self, information, num_queries=3):
-        prompt = f"""
-        Task: Generate {num_queries} optimal search queries based on the given information.
-
-        Instructions:
-        1. Analyze the provided information carefully.
-        2. Identify key concepts, entities, and relationships.
-        3. Formulate {num_queries} concise and specific search queries that will yield relevant results.
-        4. Each query should focus on a different aspect or angle of the information.
-        5. The queries should be in natural language, not in the form of keywords.
-        6. Avoid unnecessary words or phrases that might limit the search results.
-
-        Your response must be in the following JSON format:
-        {{
-            "search_queries": [
-                "Your first search query here",
-                "Your second search query here",
-                "Your third search query here"
-            ]
-        }}
-
-        Ensure that:
-        - You provide exactly {num_queries} search queries.
-        - Each query is unique and focuses on a different aspect of the information.
-        - The queries are in plain text, suitable for a web search engine.
-
-        Information to base the search queries on:
-        {information}
-
-        Now, generate the optimal search queries:
-        """
+        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
+
+    def generate_search_queries(self, information: str, num_queries: int = 10) -> List[str]:
+        prompt = f""" Task: Generate exactly {num_queries} optimal search queries based on the given information.  
+Instructions: 
+1. Analyze the provided information thoroughly. 
+2. Identify key concepts, entities, and relationships. 
+3. Formulate {num_queries} concise and specific search queries that will yield relevant and diverse results. 
+4. Each query should focus on a different aspect or angle of the information. 
+5. The queries should be in natural language, not in the form of keywords. 
+6. Avoid unnecessary words or phrases that might limit the search results. 
+7. **Important**: Return the response **ONLY** in JSON format without any additional text or code blocks.  
+Your response must be in the following JSON format: {{
+    "search_queries": [
+        "Your first search query here",
+        "Your second search query here",
+        "...",
+        "Your last search query here"
+    ]
+}}  
+Ensure that: 
+- You provide exactly {num_queries} search queries. 
+- Each query is unique and focuses on a different aspect of the information. 
+- The queries are in plain text, suitable for a web search engine.  
+
+Information to base the search queries on: 
+{information}  
+
+Now, generate the optimal search queries: """
 
         response = ""
         for chunk in self.ai.chat(prompt):
             response += chunk
 
-        try:
-            json_response = json.loads(response)
-            return json_response["search_queries"]
-        except json.JSONDecodeError:
-            print(f"Warning: Failed to parse JSON. Raw response: {response}")
-            # Fallback: try to extract queries manually
-            queries = []
-            for line in response.split('\n'):
-                if line.strip().startswith('"') and line.strip().endswith('"'):
-                    queries.append(line.strip(' "'))
-            if queries:
-                return queries[:num_queries]
-            else:
-                print(f"Warning: Using original information as search query.")
-                return [information]
-
-    def search(self, information, region='wt-wt', safesearch='off', timelimit='y', max_results=5):
-        search_queries = self.generate_search_queries(information)
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(0)
+            try:
+                json_response = json.loads(json_str)
+                print(json_response['search_queries'])
+                return json_response["search_queries"]
+            except json.JSONDecodeError:
+                pass
+
+        queries = re.findall(r'"([^"]+)"', response)
+        if len(queries) >= num_queries:
+            return queries[:num_queries]
+        elif queries:
+            return queries
+        else:
+            return [information]
+
+    def search(self, information: str, region: str = 'wt-wt', safesearch: str = 'off',
+               timelimit: str = 'y', max_results: int = 10) -> List[Dict]:
+        search_queries = self.generate_search_queries(information, num_queries=10)
         all_results = []
-        
+
         for query in search_queries:
             results = []
             with self.webs as webs:
-                for result in webs.text(query, region=region, safesearch=safesearch, timelimit=timelimit, max_results=max_results):
+                for result in webs.text(query, region=region, safesearch=safesearch,
+                                       timelimit=timelimit, max_results=max_results):
                     results.append(result)
             all_results.extend(results)
-        
+
         return all_results
 
-    def extract_urls(self, results):
-        urls = []
-        for result in results:
-            url = result.get('href')
-            if url:
-                urls.append(url)
-        return list(set(urls))
+    def extract_urls(self, results: List[Dict]) -> List[str]:
+        urls = [result.get('href') for result in results if result.get('href')]
+        unique_urls = list(set(urls))
+        return unique_urls
 
-    def fetch_webpage(self, url: str) -> str:
+    def fetch_webpage(self, url: str) -> Dict[str, str]:
         try:
-            response = httpx.get(url, timeout=120)
-            if response.status_code == 200:
-                html = response.text
-                soup = BeautifulSoup(html, 'html.parser')
-                paragraphs = soup.find_all('p')
-                text = ' '.join([p.get_text() for p in paragraphs])
-                words = text.split()
-                if len(words) > 600:
-                    text = ' '.join(words[:600]) + '...'
-                return text
-            else:
-                return f"Failed to fetch {url}: HTTP {response.status}"
+            with httpx.Client(timeout=120) as client:
+                response = client.get(url)
+                if response.status_code == 200:
+                    html = response.text
+                    soup = BeautifulSoup(html, 'html.parser')
+                    paragraphs = soup.find_all('p')
+                    text = ' '.join([p.get_text() for p in paragraphs])
+                    words = text.split()
+                    if len(words) > 600:
+                        text = ' '.join(words[:600]) + '...'
+                    return {"url": url, "content": text}
+                else:
+                    return {"url": url, "content": f"Failed to fetch {url}: HTTP {response.status_code}"}
         except Exception as e:
-            return f"Error fetching {url}: {str(e)}"
+            return {"url": url, "content": f"Error fetching {url}: {str(e)}"}
 
-    def fetch_all_webpages(self, urls: List[str]) -> List[Dict[str, str]]:
+    def fetch_all_webpages(self, urls: List[str], max_workers: int = 10) -> List[Dict[str, str]]:
         contents = []
-        for url in urls:
-            content = self.fetch_webpage(url)
-            contents.append({"url": url, "content": content})
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_url = {executor.submit(self.fetch_webpage, url): url for url in urls}
+            for future in as_completed(future_to_url):
+                result = future.result()
+                contents.append(result)
         return contents
 
+
 class OnlineSearcher:
     def __init__(self):
         self.agent = WebSearchAgent()
-        self.ai = DeepInfra(is_conversation=False)
+        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
 
-    def answer_question(self, question: str):
-        search_results = self.agent.search(question)
+    def answer_question(self, question: str) -> None:
+        search_results = self.agent.search(question, max_results=10)
         urls = self.agent.extract_urls(search_results)
         webpage_contents = self.agent.fetch_all_webpages(urls)
 
         context = "Web search results and extracted content:\n\n"
         for i, result in enumerate(search_results, 1):
-            context += f"{i}. Title: {result['title']}\n   URL: {result['href']}\n   Snippet: {result['body']}\n\n"
+            title = result.get('title', 'No Title')
+            href = result.get('href', 'No URL')
+            snippet = result.get('body', 'No Snippet')
+            context += f"{i}. **Title:** {title}\n   **URL:** {href}\n   **Snippet:** {snippet}\n\n"
 
         context += "Extracted webpage contents:\n"
-        for i, webpage in enumerate(webpage_contents):
-            context += f"{i}. URL: {webpage['url']}\n   Content: {webpage['content'][:600]}...\n\n"
-
-        prompt = f"""
-        Task: Provide a comprehensive and accurate answer to the given question based on the provided web search results and your general knowledge.
+        for i, webpage in enumerate(webpage_contents, 1):
+            content = webpage['content']
+            content_preview = content[:600] + '...' if len(content) > 600 else content
+            context += f"{i}. **URL:** {webpage['url']}\n   **Content:** {content_preview}\n\n"
+
+        prompt = f""" Task: Provide a comprehensive, insightful, and well-structured answer to the given question based on the provided web search results and your general knowledge.  
+Question: {question}  
+Context: {context}  
+Instructions:  
+1. Carefully analyze the provided web search results and extracted content.  
+2. Synthesize the information to form a coherent and comprehensive answer.  
+3. If the search results contain relevant information, incorporate it into your answer seamlessly.  
+4. Avoid providing irrelevant information, and do not reference "according to web page".  
+5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.  
+6. Ensure your answer is well-structured, factual, and directly addresses the question.  
+7. Use clear headings, bullet points, or other formatting tools to enhance readability where appropriate.  
+8. Strive for a tone and style similar to that of professional, authoritative sources like Perplexity, ensuring clarity and depth in your response.  
+Your response should be informative, accurate, and properly sourced when possible. Begin your answer now: """
+
+        for chunk in self.ai.chat(prompt, stream=True):
+            print(chunk, end='', flush=True)  # Print each chunk in real-time
 
-        Question: {question}
 
-        Context:
-        {context}
-
-        Instructions:
-        1. Carefully analyze the provided web search results and extracted content.
-        2. Synthesize the information to form a coherent and comprehensive answer.
-        3. If the search results contain relevant information, incorporate it into your answer.
-        4. Don't provide irrelevant information, and don't say "according to web page".
-        5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.
-        6. Ensure your answer is well-structured, factual, and directly addresses the question.
-        7. If appropriate, provide additional context or related information that might be helpful.
-
-        Your response should be informative, accurate, and properly sourced when possible. Begin your answer now:
-        """
-
-        return self.ai.chat(prompt)
 
 # Usage example
 if __name__ == "__main__":
     assistant = OnlineSearcher()
     while True:
-        question = input(">>> ")
-        if question.lower() == 'quit':
+        try:
+            question = input(">>> ")
+            if question.lower() == 'quit':
+                break
+            print("=" * 50)
+            assistant.answer_question(question)  # The answer is printed in real-time
+            print("=" * 50)
+        except KeyboardInterrupt:
+            print("\nExiting.")
             break
-        print("\n" + "="*50)
-        for chunk in assistant.answer_question(question):
-            print(chunk, end="", flush=True)
-        print("\n" + "="*50)
+        except Exception as e:
+            print(f"An error occurred: {e}")

webscout/Provider/ChatGPTES.py

@@ -0,0 +1,239 @@
+import requests
+import re
+import json
+import os
+from typing import List, Dict
+from webscout.AIutel import Optimizers, Conversation, AwesomePrompts
+from webscout.AIbase import Provider
+from webscout import exceptions
+from rich import print
+
+class ChatGPTES(Provider):
+    """
+    A class to interact with the ChatGPT.es API.
+    """
+
+    SUPPORTED_MODELS = {
+        'gpt-4o', 'gpt-4o-mini', 'chatgpt-4o-latest'
+    }
+
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 600,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+        model: str = "chatgpt-4o-latest",  # Default model
+        system_prompt: str = "You are a helpful assistant.",
+    ):
+        """
+        Initializes the ChatGPT.es API with given parameters.
+        """
+        if model not in self.SUPPORTED_MODELS:
+            raise ValueError(f"Unsupported model: {model}. Choose from: {self.SUPPORTED_MODELS}")
+
+        self.session = requests.Session()
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.api_endpoint = 'https://chatgpt.es/wp-admin/admin-ajax.php'
+        self.stream_chunk_size = 64
+        self.timeout = timeout
+        self.last_response = {}
+        self.system_prompt = system_prompt
+        self.model = model
+        self.initial_headers = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+                          'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+            'Referer': 'https://www.google.com/',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,'
+                      'image/avif,image/webp,image/apng,*/*;q=0.8,'
+                      'application/signed-exchange;v=b3;q=0.7',
+        }
+        self.post_headers = {
+            'User-Agent': self.initial_headers['User-Agent'],
+            'Referer': 'https://chatgpt.es/',
+            'Origin': 'https://chatgpt.es',
+            'Accept': '*/*',
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+        }
+        self.nonce = None
+        self.post_id = None
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+
+        # Conversation setup
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def get_nonce_and_post_id(self):
+        """
+        Retrieves the nonce and post ID from the ChatGPT.es website.
+        """
+        try:
+            response = self.session.get('https://chatgpt.es/', headers=self.initial_headers, timeout=self.timeout)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise ConnectionError(f"Failed to retrieve nonce and post_id: {e}")
+
+        nonce_match = re.search(r'data-nonce="(.+?)"', response.text)
+        post_id_match = re.search(r'data-post-id="(.+?)"', response.text)
+
+        if not nonce_match or not post_id_match:
+            raise ValueError("Failed to parse nonce or post_id from the response.")
+
+        self.nonce = nonce_match.group(1)
+        self.post_id = post_id_match.group(1)
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> dict:
+        """
+        Chat with ChatGPT.es
+
+        Args:
+            prompt (str): Prompt to be sent.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            raw (bool, optional): Stream back raw response as received. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+
+        Returns:
+            dict: Response dictionary.
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                optimizer_func = getattr(Optimizers, optimizer)
+                conversation_prompt = optimizer_func(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise ValueError(f"Optimizer '{optimizer}' is not supported. "
+                                 f"Available optimizers: {list(self.__available_optimizers)}")
+
+        # Retrieve nonce and post_id if they are not set
+        if not self.nonce or not self.post_id:
+            self.get_nonce_and_post_id()
+
+        messages = [
+            {"role": "user", "content": conversation_prompt},
+        ]
+
+        # Prepare conversation history
+        conversation = ["Human: strictly respond in the same language as my prompt, preferably English"]
+        for msg in messages:
+            role = "Human" if msg['role'] == "user" else "AI"
+            conversation.append(f"{role}: {msg['content']}")
+
+        payload = {
+            '_wpnonce': self.nonce,
+            'post_id': self.post_id,
+            'url': 'https://chatgpt.es',
+            'action': 'wpaicg_chat_shortcode_message',
+            'message': messages[-1]['content'],
+            'bot_id': '0',
+            'chatbot_identity': 'shortcode',
+            'wpaicg_chat_client_id': os.urandom(5).hex(),
+            'wpaicg_chat_history': json.dumps(conversation)
+        }
+
+        try:
+            response = self.session.post(
+                self.api_endpoint,
+                headers=self.post_headers,
+                data=payload,
+                timeout=self.timeout
+            )
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise ConnectionError(f"Failed to send request to ChatGPT.es: {e}")
+
+        try:
+            response_data = response.json()
+        except json.JSONDecodeError:
+            raise ValueError(f"Invalid JSON response: {response.text}")
+
+        if not isinstance(response_data, dict):
+            raise TypeError(f"Expected response_data to be a dict, got {type(response_data)}")
+
+        # Extract the message directly from the 'data' key
+        message = response_data.get('data')
+        if not isinstance(message, str):
+            raise KeyError("Missing 'data' key in response or 'data' is not a string")
+
+        self.last_response.update(dict(text=message))
+        self.conversation.update_chat_history(
+            prompt, self.get_message(self.last_response)
+        )
+        return self.last_response
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """
+        Generate response as a string.
+
+        Args:
+            prompt (str): Prompt to be sent.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+
+        Returns:
+            str: Response generated.
+        """
+        response = self.ask(
+            prompt,
+            stream=stream,
+            optimizer=optimizer,
+            conversationally=conversationally,
+        )
+        return self.get_message(response)
+
+    def get_message(self, response: dict) -> str:
+        """
+        Retrieves message only from response.
+
+        Args:
+            response (dict): Response generated by `self.ask`.
+
+        Returns:
+            str: Message extracted.
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response["text"]
+
+if __name__ == "__main__":
+    ai = ChatGPTES() 
+    response = ai.chat(input(">>> "))
+    for chunk in response:
+        print(chunk, end="", flush=True)
+

webscout/Provider/Deepinfra.py

@@ -21,7 +21,7 @@ class DeepInfra(Provider):
         proxies: dict = {},
         history_offset: int = 10250,
         act: str = None,
-        model: str = "Qwen/Qwen2-72B-Instruct",
+        model: str = "Qwen/Qwen2.5-72B-Instruct",
         system_prompt: str = "You are a Helpful AI."
     ):
         """Instantiates DeepInfra

webscout/Provider/TTI/__init__.py

@@ -2,4 +2,6 @@ from .deepinfra import *
 from .PollinationsAI import *
 from .aiforce import *
 from .blackboximage import *
-from .Nexra import *
+from .Nexra import *
+from .huggingface import *
+from .artbit import *