webscout 4.4__tar.gz → 4.5__tar.gz

{webscout-4.4/webscout.egg-info → webscout-4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 4.4
+Version: 4.5
 Summary: Search for anything using Google, DuckDuckGo, brave, qwant,  phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
 Author: OEvortex
 Author-email: helpingai5@gmail.com
@@ -742,30 +742,11 @@ with WEBS() as WEBS:
 ## usage of WEBSX -- Another Websearch thing
 ```python
 from webscout import WEBSX
+s = "Python development tools"
 
-def main():
-    # Initialize the WEBSX client
-    search = WEBSX(
-        k=10,  
-    )
+result = WEBSX(s)
 
-    # Example using `run` method - Get a summary
-    query = "What is the capital of France?"
-    answer = search.run(query)
-    print(f"Answer: {answer}\n")
-
-    # Example using `results` method - Get detailed results with metadata
-    query = "What is the capital of France?"
-    results = search.results(query, num_results=3)
-    print("Search Results:")
-    for result in results:
-        print(f"Title: {result['title']}")
-        print(f"Snippet: {result['snippet']}")
-        print(f"Link: {result['link']}\n")
-        print(f'Engines: {result["engines"]}')
-
-if __name__ == "__main__":
-    main()
+print(result)
 ```
 ## ALL acts
 <details>
@@ -1028,7 +1009,7 @@ ___
 ### 0. `Duckchat` - chat with LLM
 ```python
 from webscout import WEBS as w
-R = w().chat("hello", model='claude-3-haiku') # GPT-3.5 Turbo, mixtral-8x7b, llama-3-70b, claude-3-haiku
+R = w().chat("Who are you", model='gpt-4o-mini') # GPT-3.5 Turbo, mixtral-8x7b, llama-3-70b, claude-3-haiku, gpt-4o-mini
 print(R)
 ```
 ### 1. `PhindSearch` - Search using Phind.com 
@@ -1505,7 +1486,12 @@ llama = LLAMA()
 r = llama.chat("What is the meaning of life?")
 print(r)
 ```
-
+### 25. AndiSearch
+```python
+from webscout import AndiSearch
+a = AndiSearch()
+print(a.chat("HelpingAI-9B"))
+```
 ### `LLM` 
 ```python
 from webscout.LLM import LLM

{webscout-4.4 → webscout-4.5}/README.md

@@ -670,30 +670,11 @@ with WEBS() as WEBS:
 ## usage of WEBSX -- Another Websearch thing
 ```python
 from webscout import WEBSX
+s = "Python development tools"
 
-def main():
-    # Initialize the WEBSX client
-    search = WEBSX(
-        k=10,  
-    )
+result = WEBSX(s)
 
-    # Example using `run` method - Get a summary
-    query = "What is the capital of France?"
-    answer = search.run(query)
-    print(f"Answer: {answer}\n")
-
-    # Example using `results` method - Get detailed results with metadata
-    query = "What is the capital of France?"
-    results = search.results(query, num_results=3)
-    print("Search Results:")
-    for result in results:
-        print(f"Title: {result['title']}")
-        print(f"Snippet: {result['snippet']}")
-        print(f"Link: {result['link']}\n")
-        print(f'Engines: {result["engines"]}')
-
-if __name__ == "__main__":
-    main()
+print(result)
 ```
 ## ALL acts
 <details>
@@ -956,7 +937,7 @@ ___
 ### 0. `Duckchat` - chat with LLM
 ```python
 from webscout import WEBS as w
-R = w().chat("hello", model='claude-3-haiku') # GPT-3.5 Turbo, mixtral-8x7b, llama-3-70b, claude-3-haiku
+R = w().chat("Who are you", model='gpt-4o-mini') # GPT-3.5 Turbo, mixtral-8x7b, llama-3-70b, claude-3-haiku, gpt-4o-mini
 print(R)
 ```
 ### 1. `PhindSearch` - Search using Phind.com 
@@ -1433,7 +1414,12 @@ llama = LLAMA()
 r = llama.chat("What is the meaning of life?")
 print(r)
 ```
-
+### 25. AndiSearch
+```python
+from webscout import AndiSearch
+a = AndiSearch()
+print(a.chat("HelpingAI-9B"))
+```
 ### `LLM` 
 ```python
 from webscout.LLM import LLM

{webscout-4.4 → webscout-4.5}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", encoding="utf-8") as f:
 
 setup(
     name="webscout",
-    version="4.4",
+    version="4.5",
     description="Search for anything using Google, DuckDuckGo, brave, qwant,  phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more",
     long_description=README,
     long_description_content_type="text/markdown",

{webscout-4.4 → webscout-4.5}/webscout/AIutel.py

@@ -52,7 +52,8 @@ webai = [
    "vtlchat",
    "geminiflash",
    "geminipro",
-   "ollama"
+   "ollama",
+   "andi",
 ]
 
 gpt4free_providers = [

webscout-4.5/webscout/Agents/Onlinesearcher.py

@@ -0,0 +1,175 @@
+import json
+from webscout import WEBS
+import httpx
+from bs4 import BeautifulSoup
+from typing import List, Dict
+
+class DeepInfra:
+    def __init__(
+        self,
+        model: str = "meta-llama/Meta-Llama-3.1-70B-Instruct",
+        max_tokens: int = 8000,
+        timeout: int = 120,
+        system_prompt: str = "You are a helpful AI assistant.",
+        proxies: dict = {}
+    ):
+        self.model = model
+        self.max_tokens = max_tokens
+        self.timeout = timeout
+        self.system_prompt = system_prompt
+        self.chat_endpoint = "https://api.deepinfra.com/v1/openai/chat/completions"
+        
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+            'Accept-Language': 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3',
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+            'Content-Type': 'application/json',
+            'Origin': 'https://deepinfra.com',
+            'Pragma': 'no-cache',
+            'Referer': 'https://deepinfra.com/',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-site',
+            'X-Deepinfra-Source': 'web-embed',
+            'accept': 'text/event-stream',
+            'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"macOS"'
+        }
+        
+        self.client = httpx.Client(proxies=proxies, headers=self.headers)
+
+    def ask(self, prompt: str, system_prompt: str = None) -> str:
+        payload = {
+            'model': self.model,
+            'messages': [
+                {"role": "system", "content": system_prompt or self.system_prompt},
+                {"role": "user", "content": prompt},
+            ],
+            'temperature': 0.7,
+            'max_tokens': self.max_tokens,
+            'stop': []
+        }
+
+        response = self.client.post(self.chat_endpoint, json=payload, timeout=self.timeout)
+        if response.status_code != 200:
+            raise Exception(f"Failed to generate response - ({response.status_code}, {response.reason_phrase}) - {response.text}")
+
+        resp = response.json()
+        return resp["choices"][0]["message"]["content"]
+
+class WebSearchAgent:
+
+    def __init__(self, model="Qwen/Qwen2-72B-Instruct"):
+        self.webs = WEBS()
+        self.deepinfra = DeepInfra(model=model)
+
+    def generate_search_query(self, information):
+        prompt = f"""
+        Instructions:
+        You are a smart online searcher for a large language model.
+        Given information, you must create a search query to search the internet for relevant information.
+        Your search query must be in the form of a json response.
+        Exact json response format must be as follows:
+        
+        {{
+            "search_query": "your search query"
+        }}
+        - You must only provide ONE search query
+        - You must provide the BEST search query for the given information
+        - The search query must be normal text.
+
+        Information: {information}
+        """
+
+        response = self.deepinfra.ask(prompt)
+        return json.loads(response)["search_query"]
+
+    def search(self, information, region='wt-wt', safesearch='off', timelimit='y', max_results=5):
+        search_query = self.generate_search_query(information)
+        
+        results = []
+        with self.webs as webs:
+            for result in webs.text(search_query, region=region, safesearch=safesearch, timelimit=timelimit, max_results=max_results):
+                results.append(result)
+        
+        return results
+
+    def extract_urls(self, results):
+        urls = []
+        for result in results:
+            url = result.get('href')
+            if url:
+                urls.append(url)
+        return list(set(urls))  # Remove duplicates
+
+    def fetch_webpage(self, url: str) -> str:
+        try:
+            response = httpx.get(url, timeout=120)
+            if response.status_code == 200:
+                html = response.text
+                soup = BeautifulSoup(html, 'html.parser')
+                
+                # Extract text from <p> tags
+                paragraphs = soup.find_all('p')
+                text = ' '.join([p.get_text() for p in paragraphs])
+                
+                # Limit the text to around 4000 words
+                words = text.split()
+                if len(words) > 4000:
+                    text = ' '.join(words[:4000]) + '...'
+                
+                return text
+            else:
+                return f"Failed to fetch {url}: HTTP {response.status}"
+        except Exception as e:
+            return f"Error fetching {url}: {str(e)}"
+
+    def fetch_all_webpages(self, urls: List[str]) -> List[Dict[str, str]]:
+        contents = []
+        for url in urls:
+            content = self.fetch_webpage(url)
+            contents.append({"url": url, "content": content})
+        return contents
+
+class OnlineSearcher:
+    def __init__(self, model="meta-llama/Meta-Llama-3.1-405B-Instruct"):
+        self.agent = WebSearchAgent(model)
+        self.deepinfra = DeepInfra(model="model")
+
+    def answer_question(self, question: str) -> str:
+        # Perform web search
+        search_results = self.agent.search(question)
+
+        # Extract URLs
+        urls = self.agent.extract_urls(search_results)
+
+        # Fetch webpage contents
+        webpage_contents = self.agent.fetch_all_webpages(urls)
+
+        # Prepare context for AI
+        context = "Based on the following search results and webpage contents:\n\n"
+        for i, result in enumerate(search_results, 1):
+            context += f"{i}. Title: {result['title']}\n   URL: {result['href']}\n   Snippet: {result['body']}\n\n"
+
+        context += "Extracted webpage contents:\n"
+        for i, webpage in enumerate(webpage_contents):
+            context += f"{i}. URL: {webpage['url']}\n   Content: {webpage['content'][:4000]}...\n\n"
+
+        # Generate answer using AI
+        prompt = f"{context}\n\nQuestion: {question}\n\nPlease provide a comprehensive answer to the question based on the search results and webpage contents above. Include relevant webpage URLs in your answer when appropriate. If the search results and webpage contents don't contain relevant information, please state that and provide the best answer you can based on your general knowledge. [YOUR RESPONSE WITH SOURCE LINKS ([➊](URL))"
+
+        answer = self.deepinfra.ask(prompt)
+        return answer
+
+# Usage example
+if __name__ == "__main__":
+    assistant = OnlineSearcher()
+    while True:
+        question = input(">>> ")
+        if question.lower() == 'quit':
+            break
+        answer = assistant.answer_question(question)
+        print(answer)
+        print("\n" + "-"*50 + "\n")

webscout-4.5/webscout/Agents/__init__.py

@@ -0,0 +1,2 @@
+from .Onlinesearcher import *
+from .functioncall import *

webscout-4.5/webscout/Agents/functioncall.py

@@ -0,0 +1,126 @@
+import json
+import logging
+from webscout import DeepInfra, WEBS
+
+class FunctionCallingAgent:
+    def __init__(self, model: str = "Qwen/Qwen2-72B-Instruct", system_prompt: str = 'You are a helpful assistant that will always answer what user wants', tools: list = None):
+        self.deepinfra = DeepInfra(model=model, system_prompt=system_prompt)
+        self.tools = tools if tools is not None else []
+        # logging.basicConfig(level=logging.INFO)
+        # self.webs = WEBS()  # Initialize a WEBS object for web search
+
+    def function_call_handler(self, message_text: str):
+        """Handles function calls based on the provided message text
+
+        Args:
+            message_text (str): The input message text from the user.
+
+        Returns:
+            dict: The extracted function call and arguments.
+        """
+        system_message = f'[SYSTEM]You are a helpful assistant. You have access to the following functions: \n {str(self.tools)}\n\nTo use these functions respond with:\n<functioncall> {{ "name": "function_name", "arguments": {{ "arg_1": "value_1", "arg_2": "value_2", ... }} }}  </functioncall>  [USER] {message_text}'
+        
+        response = self.deepinfra.chat(system_message)
+        # logging.info(f"Raw response: {response}")
+
+        try:
+            # Extract the JSON-like part of the response
+            start_idx = response.find("{")
+            end_idx = response.rfind("}") + 1
+            if start_idx == -1 or end_idx == -1:
+                raise ValueError("JSON-like structure not found in the response")
+
+            response_json_str = response[start_idx:end_idx]
+            # Ensure the JSON string is properly formatted
+            response_json_str = response_json_str.replace("'", '"')  # Replace single quotes with double quotes
+            response_json_str = response_json_str.strip()
+            response_data = json.loads(response_json_str)
+        except (ValueError, json.JSONDecodeError) as e:
+            # logging.error(f"An error occurred while parsing response: {e}")
+            return {"error": str(e)}
+
+        return response_data
+
+    def execute_function(self, function_call_data: dict) -> str:
+        """Executes the specified function with the provided arguments.
+
+        Args:
+            function_call_data (dict): A dictionary containing the function name and arguments.
+
+        Returns:
+            str: The result of the function execution.
+        """
+        function_name = function_call_data.get("name")
+        arguments = function_call_data.get("arguments", "{}")  # Default to empty dict if not present
+
+        # Parse the arguments string into a dictionary
+        try:
+            arguments_dict = json.loads(arguments)
+        except json.JSONDecodeError:
+            # logging.error("Failed to parse arguments as JSON.")
+            return "Invalid arguments format."
+
+        # logging.info(f"Executing function: {function_name} with arguments: {arguments_dict}")
+
+        # if function_name == "web_search":
+        #     query = arguments_dict.get("query")
+        #     if query:
+        #         search_results = self.webs.text(query)
+        #         # You can process the search results here, e.g., extract URLs, summarize, etc.
+        #         return f"Here's what I found:\n\n{search_results}"
+        #     else:
+        #         return "Please provide a search query."
+        # else:
+        #     return f"Function '{function_name}' is not yet implemented."
+
+# Example usage
+if __name__ == "__main__":
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "UserDetail",
+                "parameters": {
+                    "type": "object",
+                    "title": "UserDetail",
+                    "properties": {
+                        "name": {
+                            "title": "Name",
+                            "type": "string"
+                        },
+                        "age": {
+                            "title": "Age",
+                            "type": "integer"
+                        }
+                    },
+                    "required": ["name", "age"]
+                }
+            }
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "web_search",
+                "description": "Search query on google",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "web search query"
+                        }
+                    },
+                    "required": ["query"]
+                }
+            }
+        }
+    ]
+
+    agent = FunctionCallingAgent(tools=tools)
+    message = "tell me about HelpingAI flash"
+    function_call_data = agent.function_call_handler(message)
+    print(f"Function Call Data: {function_call_data}")
+
+    if "error" not in function_call_data:
+        result = agent.execute_function(function_call_data)
+        # print(f"Function Execution Result: {result}")