webscout 5.7__py3-none-any.whl → 5.9__py3-none-any.whl

webscout/AIutel.py

@@ -3,6 +3,8 @@ import json
 import platform
 import subprocess
 import logging
+import threading
+import time
 import appdirs
 import datetime
 import re
@@ -11,7 +13,7 @@ import click
 from rich.markdown import Markdown
 from rich.console import Console
 import g4f
-from typing import Union
+from typing import List, Tuple, Union
 from typing import NoReturn
 import requests
 from pathlib import Path
@@ -1053,4 +1055,76 @@ class Audio:
         """
         if not Path(path_to_audio_file).is_file():
             raise FileNotFoundError(f"File does not exist - '{path_to_audio_file}'")
-        playsound(path_to_audio_file)
+        playsound(path_to_audio_file)
+class ProxyManager:
+    def __init__(self, refresh_interval=60):
+        self.proxies: List[Tuple[str, float]] = []  # Store proxy and its latency
+        self.last_refresh: float = 0
+        self.refresh_interval = refresh_interval
+        self.lock = threading.Lock()  # Add a lock for thread safety
+        # Start auto-refresh in a separate thread
+        threading.Thread(target=self.auto_refresh_proxies, daemon=True).start()
+
+    def fetch_proxies(self, max_proxies=50) -> List[str]:
+        try:
+            url = "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all"
+            response = requests.get(url)
+            proxies = response.text.split('\r\n')[:max_proxies]  # Extract up to max_proxies
+            return [proxy for proxy in proxies if proxy]
+        except requests.RequestException as e:
+            print(f"Error fetching proxies: {e}")
+            return []
+
+    def test_proxy(self, proxy: str) -> Tuple[str, float] | None:
+        # Test both HTTP and HTTPS
+        for protocol in ['http', 'https']:
+            try:
+                start_time = time.time()
+                response = requests.get('http://httpbin.org/ip', proxies={protocol: f"{protocol}://{proxy}"}, timeout=5)
+                if response.status_code == 200:
+                    end_time = time.time()
+                    return proxy, end_time - start_time
+            except requests.RequestException:
+                pass
+        return None
+
+    def refresh_proxies(self) -> int:
+        new_proxies = self.fetch_proxies()
+        threads = []
+        working_proxies = []
+
+        # Use threading for faster proxy testing
+        for proxy in new_proxies:
+            thread = threading.Thread(target=self.test_proxy_and_append, args=(proxy, working_proxies))
+            threads.append(thread)
+            thread.start()
+
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
+
+        with self.lock:  # Acquire lock before updating proxies list
+            self.proxies = working_proxies
+            self.last_refresh = time.time()
+
+        # print(f"Refreshed proxies at {datetime.now()}. Total working proxies: {len(self.proxies)}")
+        return len(self.proxies)
+
+    def test_proxy_and_append(self, proxy: str, working_proxies: list):
+        result = self.test_proxy(proxy)
+        if result:
+            with self.lock:  # Acquire lock before appending to shared list
+                working_proxies.append(result)  # Append the proxy and its latency
+
+    def auto_refresh_proxies(self):
+        while True:
+            time.sleep(self.refresh_interval)
+            self.refresh_proxies()
+
+    def get_fastest_proxy(self) -> str | None:
+        with self.lock:  # Acquire lock before accessing proxies list
+            if self.proxies:
+                # Sort proxies by latency and return the fastest
+                self.proxies.sort(key=lambda x: x[1])  # Sort by latency
+                return self.proxies[0][0]  # Return the fastest proxy
+        return None

webscout/Agents/Onlinesearcher.py

@@ -1,162 +1,170 @@
 import json
-import colorlog
-from webscout import WEBS
-from webscout import DeepInfra
 import httpx
 from bs4 import BeautifulSoup
 from typing import List, Dict
-import logging
+from webscout import WEBS, GEMINIAPI
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 
 class WebSearchAgent:
     def __init__(self):
         self.webs = WEBS()
-        self.ai = DeepInfra(is_conversation=False)
-
-    def generate_search_queries(self, information, num_queries=3):
-        prompt = f"""
-        Task: Generate {num_queries} optimal search queries based on the given information.
-
-        Instructions:
-        1. Analyze the provided information carefully.
-        2. Identify key concepts, entities, and relationships.
-        3. Formulate {num_queries} concise and specific search queries that will yield relevant results.
-        4. Each query should focus on a different aspect or angle of the information.
-        5. The queries should be in natural language, not in the form of keywords.
-        6. Avoid unnecessary words or phrases that might limit the search results.
-
-        Your response must be in the following JSON format:
-        {{
-            "search_queries": [
-                "Your first search query here",
-                "Your second search query here",
-                "Your third search query here"
-            ]
-        }}
-
-        Ensure that:
-        - You provide exactly {num_queries} search queries.
-        - Each query is unique and focuses on a different aspect of the information.
-        - The queries are in plain text, suitable for a web search engine.
-
-        Information to base the search queries on:
-        {information}
-
-        Now, generate the optimal search queries:
-        """
+        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
+
+    def generate_search_queries(self, information: str, num_queries: int = 10) -> List[str]:
+        prompt = f""" Task: Generate exactly {num_queries} optimal search queries based on the given information.  
+Instructions: 
+1. Analyze the provided information thoroughly. 
+2. Identify key concepts, entities, and relationships. 
+3. Formulate {num_queries} concise and specific search queries that will yield relevant and diverse results. 
+4. Each query should focus on a different aspect or angle of the information. 
+5. The queries should be in natural language, not in the form of keywords. 
+6. Avoid unnecessary words or phrases that might limit the search results. 
+7. **Important**: Return the response **ONLY** in JSON format without any additional text or code blocks.  
+Your response must be in the following JSON format: {{
+    "search_queries": [
+        "Your first search query here",
+        "Your second search query here",
+        "...",
+        "Your last search query here"
+    ]
+}}  
+Ensure that: 
+- You provide exactly {num_queries} search queries. 
+- Each query is unique and focuses on a different aspect of the information. 
+- The queries are in plain text, suitable for a web search engine.  
+
+Information to base the search queries on: 
+{information}  
+
+Now, generate the optimal search queries: """
 
         response = ""
         for chunk in self.ai.chat(prompt):
             response += chunk
 
-        try:
-            json_response = json.loads(response)
-            return json_response["search_queries"]
-        except json.JSONDecodeError:
-            print(f"Warning: Failed to parse JSON. Raw response: {response}")
-            # Fallback: try to extract queries manually
-            queries = []
-            for line in response.split('\n'):
-                if line.strip().startswith('"') and line.strip().endswith('"'):
-                    queries.append(line.strip(' "'))
-            if queries:
-                return queries[:num_queries]
-            else:
-                print(f"Warning: Using original information as search query.")
-                return [information]
-
-    def search(self, information, region='wt-wt', safesearch='off', timelimit='y', max_results=5):
-        search_queries = self.generate_search_queries(information)
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(0)
+            try:
+                json_response = json.loads(json_str)
+                print(json_response['search_queries'])
+                return json_response["search_queries"]
+            except json.JSONDecodeError:
+                pass
+
+        queries = re.findall(r'"([^"]+)"', response)
+        if len(queries) >= num_queries:
+            return queries[:num_queries]
+        elif queries:
+            return queries
+        else:
+            return [information]
+
+    def search(self, information: str, region: str = 'wt-wt', safesearch: str = 'off',
+               timelimit: str = 'y', max_results: int = 10) -> List[Dict]:
+        search_queries = self.generate_search_queries(information, num_queries=10)
         all_results = []
-        
+
         for query in search_queries:
             results = []
             with self.webs as webs:
-                for result in webs.text(query, region=region, safesearch=safesearch, timelimit=timelimit, max_results=max_results):
+                for result in webs.text(query, region=region, safesearch=safesearch,
+                                       timelimit=timelimit, max_results=max_results):
                     results.append(result)
             all_results.extend(results)
-        
+
         return all_results
 
-    def extract_urls(self, results):
-        urls = []
-        for result in results:
-            url = result.get('href')
-            if url:
-                urls.append(url)
-        return list(set(urls))
+    def extract_urls(self, results: List[Dict]) -> List[str]:
+        urls = [result.get('href') for result in results if result.get('href')]
+        unique_urls = list(set(urls))
+        return unique_urls
 
-    def fetch_webpage(self, url: str) -> str:
+    def fetch_webpage(self, url: str) -> Dict[str, str]:
         try:
-            response = httpx.get(url, timeout=120)
-            if response.status_code == 200:
-                html = response.text
-                soup = BeautifulSoup(html, 'html.parser')
-                paragraphs = soup.find_all('p')
-                text = ' '.join([p.get_text() for p in paragraphs])
-                words = text.split()
-                if len(words) > 600:
-                    text = ' '.join(words[:600]) + '...'
-                return text
-            else:
-                return f"Failed to fetch {url}: HTTP {response.status}"
+            with httpx.Client(timeout=120) as client:
+                response = client.get(url)
+                if response.status_code == 200:
+                    html = response.text
+                    soup = BeautifulSoup(html, 'html.parser')
+                    paragraphs = soup.find_all('p')
+                    text = ' '.join([p.get_text() for p in paragraphs])
+                    words = text.split()
+                    if len(words) > 600:
+                        text = ' '.join(words[:600]) + '...'
+                    return {"url": url, "content": text}
+                else:
+                    return {"url": url, "content": f"Failed to fetch {url}: HTTP {response.status_code}"}
         except Exception as e:
-            return f"Error fetching {url}: {str(e)}"
+            return {"url": url, "content": f"Error fetching {url}: {str(e)}"}
 
-    def fetch_all_webpages(self, urls: List[str]) -> List[Dict[str, str]]:
+    def fetch_all_webpages(self, urls: List[str], max_workers: int = 10) -> List[Dict[str, str]]:
         contents = []
-        for url in urls:
-            content = self.fetch_webpage(url)
-            contents.append({"url": url, "content": content})
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_url = {executor.submit(self.fetch_webpage, url): url for url in urls}
+            for future in as_completed(future_to_url):
+                result = future.result()
+                contents.append(result)
         return contents
 
+
 class OnlineSearcher:
     def __init__(self):
         self.agent = WebSearchAgent()
-        self.ai = DeepInfra(is_conversation=False)
+        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
 
-    def answer_question(self, question: str):
-        search_results = self.agent.search(question)
+    def answer_question(self, question: str) -> None:
+        search_results = self.agent.search(question, max_results=10)
         urls = self.agent.extract_urls(search_results)
         webpage_contents = self.agent.fetch_all_webpages(urls)
 
         context = "Web search results and extracted content:\n\n"
         for i, result in enumerate(search_results, 1):
-            context += f"{i}. Title: {result['title']}\n   URL: {result['href']}\n   Snippet: {result['body']}\n\n"
+            title = result.get('title', 'No Title')
+            href = result.get('href', 'No URL')
+            snippet = result.get('body', 'No Snippet')
+            context += f"{i}. **Title:** {title}\n   **URL:** {href}\n   **Snippet:** {snippet}\n\n"
 
         context += "Extracted webpage contents:\n"
-        for i, webpage in enumerate(webpage_contents):
-            context += f"{i}. URL: {webpage['url']}\n   Content: {webpage['content'][:600]}...\n\n"
-
-        prompt = f"""
-        Task: Provide a comprehensive and accurate answer to the given question based on the provided web search results and your general knowledge.
+        for i, webpage in enumerate(webpage_contents, 1):
+            content = webpage['content']
+            content_preview = content[:600] + '...' if len(content) > 600 else content
+            context += f"{i}. **URL:** {webpage['url']}\n   **Content:** {content_preview}\n\n"
+
+        prompt = f""" Task: Provide a comprehensive, insightful, and well-structured answer to the given question based on the provided web search results and your general knowledge.  
+Question: {question}  
+Context: {context}  
+Instructions:  
+1. Carefully analyze the provided web search results and extracted content.  
+2. Synthesize the information to form a coherent and comprehensive answer.  
+3. If the search results contain relevant information, incorporate it into your answer seamlessly.  
+4. Avoid providing irrelevant information, and do not reference "according to web page".  
+5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.  
+6. Ensure your answer is well-structured, factual, and directly addresses the question.  
+7. Use clear headings, bullet points, or other formatting tools to enhance readability where appropriate.  
+8. Strive for a tone and style similar to that of professional, authoritative sources like Perplexity, ensuring clarity and depth in your response.  
+Your response should be informative, accurate, and properly sourced when possible. Begin your answer now: """
+
+        for chunk in self.ai.chat(prompt, stream=True):
+            print(chunk, end='', flush=True)  # Print each chunk in real-time
 
-        Question: {question}
 
-        Context:
-        {context}
-
-        Instructions:
-        1. Carefully analyze the provided web search results and extracted content.
-        2. Synthesize the information to form a coherent and comprehensive answer.
-        3. If the search results contain relevant information, incorporate it into your answer.
-        4. Don't provide irrelevant information, and don't say "according to web page".
-        5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.
-        6. Ensure your answer is well-structured, factual, and directly addresses the question.
-        7. If appropriate, provide additional context or related information that might be helpful.
-
-        Your response should be informative, accurate, and properly sourced when possible. Begin your answer now:
-        """
-
-        return self.ai.chat(prompt)
 
 # Usage example
 if __name__ == "__main__":
     assistant = OnlineSearcher()
     while True:
-        question = input(">>> ")
-        if question.lower() == 'quit':
+        try:
+            question = input(">>> ")
+            if question.lower() == 'quit':
+                break
+            print("=" * 50)
+            assistant.answer_question(question)  # The answer is printed in real-time
+            print("=" * 50)
+        except KeyboardInterrupt:
+            print("\nExiting.")
             break
-        print("\n" + "="*50)
-        for chunk in assistant.answer_question(question):
-            print(chunk, end="", flush=True)
-        print("\n" + "="*50)
+        except Exception as e:
+            print(f"An error occurred: {e}")

webscout/Provider/Amigo.py

@@ -0,0 +1,265 @@
+import requests
+import json
+import uuid
+import os
+from typing import Any, Dict, Optional, Generator
+
+from webscout.AIutel import Optimizers
+from webscout.AIutel import Conversation
+from webscout.AIutel import AwesomePrompts, sanitize_stream
+from webscout.AIbase import Provider, AsyncProvider
+from webscout import exceptions
+
+class AmigoChat(Provider):
+    """
+    A class to interact with the AmigoChat.io API.
+    """
+
+    AVAILABLE_MODELS = [
+        "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",  # Llama 3
+        "o1-mini",                                   # OpenAI O1 Mini
+        "claude-3-sonnet-20240229",                  # Claude Sonnet
+        "gemini-1.5-pro",                             # Gemini Pro
+        "gemini-1-5-flash",                            # Gemini Flash
+        "o1-preview",                                 # OpenAI O1 Preview
+    ]
+
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 600,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+        model: str = "o1-preview",  # Default model
+    ):
+        """
+        Initializes the AmigoChat.io API with given parameters.
+
+        Args:
+            is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
+            max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
+            timeout (int, optional): Http request timeout. Defaults to 30.
+            intro (str, optional): Conversation introductory prompt. Defaults to None.
+            filepath (str, optional): Path to file containing conversation history. Defaults to None.
+            update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
+            proxies (dict, optional): Http request proxies. Defaults to {}.
+            history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
+            act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
+            model (str, optional): The AI model to use for text generation. Defaults to "o1-preview". 
+                                    Options: "llama-three-point-one", "openai-o-one-mini", "claude", 
+                                             "gemini-1.5-pro", "gemini-1.5-flash", "openai-o-one".
+        """
+        if model not in self.AVAILABLE_MODELS:
+            raise ValueError(f"Invalid model: {model}. Choose from: {self.AVAILABLE_MODELS}")
+
+        self.session = requests.Session()
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.api_endpoint = "https://api.amigochat.io/v1/chat/completions"
+        self.stream_chunk_size = 64
+        self.timeout = timeout
+        self.last_response = {}
+        self.model = model
+        self.headers = {
+            "Accept": "*/*",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "en-US,en;q=0.9,en-IN;q=0.8",
+            "Authorization": "Bearer ",  # empty
+            "Content-Type": "application/json",
+            "DNT": "1",
+            "Origin": "https://amigochat.io",
+            "Priority": "u=1, i",
+            "Referer": "https://amigochat.io/",
+            "Sec-CH-UA": '"Microsoft Edge";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
+            "Sec-CH-UA-Mobile": "?0",
+            "Sec-CH-UA-Platform": '"Windows"',
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                          "AppleWebKit/537.36 (KHTML, like Gecko) "
+                          "Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0",
+            "X-Device-Language": "en-US",
+            "X-Device-Platform": "web",
+            "X-Device-UUID": str(uuid.uuid4()),
+            "X-Device-Version": "1.0.22"
+        }
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        self.session.headers.update(self.headers)
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> Dict[str, Any]:
+        """Chat with AI
+
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            raw (bool, optional): Stream back raw response as received. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+           dict : {}
+        ```json
+        {
+           "text" : "How may I assist you today?"
+        }
+        ```
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        # Define the payload
+        payload = {
+            "messages": [
+                {"role": "system", "content": "Mai hu ba khabr"},
+                {"role": "user", "content": conversation_prompt}
+            ],
+            "model": self.model,
+            "frequency_penalty": 0,
+            "max_tokens": 4000,
+            "presence_penalty": 0,
+            "stream": stream,  # Enable streaming
+            "temperature": 0.5,
+            "top_p": 0.95
+        }
+
+        def for_stream():
+            try:
+                # Make the POST request with streaming enabled
+                with requests.post(self.api_endpoint, headers=self.headers, json=payload, stream=True) as response:
+                    # Check if the request was successful
+                    if response.status_code == 201:
+                        # Iterate over the streamed response line by line
+                        for line in response.iter_lines():
+                            if line:
+                                # Decode the line from bytes to string
+                                decoded_line = line.decode('utf-8').strip()
+                                if decoded_line.startswith("data: "):
+                                    data_str = decoded_line[6:]
+                                    if data_str == "[DONE]":
+                                        break
+                                    try:
+                                        # Load the JSON data
+                                        data_json = json.loads(data_str)
+                                        
+                                        # Extract the content from the response
+                                        choices = data_json.get("choices", [])
+                                        if choices:
+                                            delta = choices[0].get("delta", {})
+                                            content = delta.get("content", "")
+                                            if content:
+                                                yield content if raw else dict(text=content)
+                                    except json.JSONDecodeError:
+                                        print(f"Received non-JSON data: {data_str}")
+                    else:
+                        print(f"Request failed with status code {response.status_code}")
+                        print("Response:", response.text)
+
+            except requests.exceptions.RequestException as e:
+                print("An error occurred while making the request:", e)
+
+        def for_non_stream():
+            # Accumulate the streaming response
+            full_response = ""
+            for chunk in for_stream():
+                if not raw:  # If not raw, chunk is a dictionary 
+                    full_response += chunk["text"]
+
+            # Update self.last_response with the full text
+            self.last_response.update(dict(text=full_response))
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+            return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """Generate response `str`
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+            str: Response generated
+        """
+
+        def for_stream():
+            for response in self.ask(
+                prompt, True, optimizer=optimizer, conversationally=conversationally
+            ):
+                yield self.get_message(response)
+
+        def for_non_stream():
+            return self.get_message(
+                self.ask(
+                    prompt,
+                    False,
+                    optimizer=optimizer,
+                    conversationally=conversationally,
+                )
+            )
+
+        return for_stream() if stream else for_non_stream()
+
+    def get_message(self, response: dict) -> str:
+        """Retrieves message only from response
+
+        Args:
+            response (dict): Response generated by `self.ask`
+
+        Returns:
+            str: Message extracted
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response["text"]
+    
+if __name__ == '__main__':
+    from rich import print
+    ai = AmigoChat(model="o1-preview")
+    response = ai.chat(input(">>> "))
+    for chunk in response:
+        print(chunk, end="", flush=True)