webscout 4.4__py3-none-any.whl → 4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AIutel.py +2 -1
- webscout/Agents/Onlinesearcher.py +175 -0
- webscout/Agents/__init__.py +2 -0
- webscout/Agents/functioncall.py +126 -0
- webscout/Provider/Andi.py +275 -0
- webscout/Provider/__init__.py +3 -1
- webscout/__init__.py +4 -3
- webscout/voice.py +8 -1
- webscout/webai.py +13 -0
- webscout/webscout_search.py +1 -0
- webscout/websx_search.py +18 -369
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/METADATA +11 -25
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/RECORD +17 -13
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/LICENSE.md +0 -0
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/WHEEL +0 -0
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/entry_points.txt +0 -0
- {webscout-4.4.dist-info → webscout-4.5.dist-info}/top_level.txt +0 -0
webscout/AIutel.py
CHANGED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from webscout import WEBS
|
|
3
|
+
import httpx
|
|
4
|
+
from bs4 import BeautifulSoup
|
|
5
|
+
from typing import List, Dict
|
|
6
|
+
|
|
7
|
+
class DeepInfra:
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
model: str = "meta-llama/Meta-Llama-3.1-70B-Instruct",
|
|
11
|
+
max_tokens: int = 8000,
|
|
12
|
+
timeout: int = 120,
|
|
13
|
+
system_prompt: str = "You are a helpful AI assistant.",
|
|
14
|
+
proxies: dict = {}
|
|
15
|
+
):
|
|
16
|
+
self.model = model
|
|
17
|
+
self.max_tokens = max_tokens
|
|
18
|
+
self.timeout = timeout
|
|
19
|
+
self.system_prompt = system_prompt
|
|
20
|
+
self.chat_endpoint = "https://api.deepinfra.com/v1/openai/chat/completions"
|
|
21
|
+
|
|
22
|
+
self.headers = {
|
|
23
|
+
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
|
24
|
+
'Accept-Language': 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3',
|
|
25
|
+
'Cache-Control': 'no-cache',
|
|
26
|
+
'Connection': 'keep-alive',
|
|
27
|
+
'Content-Type': 'application/json',
|
|
28
|
+
'Origin': 'https://deepinfra.com',
|
|
29
|
+
'Pragma': 'no-cache',
|
|
30
|
+
'Referer': 'https://deepinfra.com/',
|
|
31
|
+
'Sec-Fetch-Dest': 'empty',
|
|
32
|
+
'Sec-Fetch-Mode': 'cors',
|
|
33
|
+
'Sec-Fetch-Site': 'same-site',
|
|
34
|
+
'X-Deepinfra-Source': 'web-embed',
|
|
35
|
+
'accept': 'text/event-stream',
|
|
36
|
+
'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
|
|
37
|
+
'sec-ch-ua-mobile': '?0',
|
|
38
|
+
'sec-ch-ua-platform': '"macOS"'
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
self.client = httpx.Client(proxies=proxies, headers=self.headers)
|
|
42
|
+
|
|
43
|
+
def ask(self, prompt: str, system_prompt: str = None) -> str:
|
|
44
|
+
payload = {
|
|
45
|
+
'model': self.model,
|
|
46
|
+
'messages': [
|
|
47
|
+
{"role": "system", "content": system_prompt or self.system_prompt},
|
|
48
|
+
{"role": "user", "content": prompt},
|
|
49
|
+
],
|
|
50
|
+
'temperature': 0.7,
|
|
51
|
+
'max_tokens': self.max_tokens,
|
|
52
|
+
'stop': []
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
response = self.client.post(self.chat_endpoint, json=payload, timeout=self.timeout)
|
|
56
|
+
if response.status_code != 200:
|
|
57
|
+
raise Exception(f"Failed to generate response - ({response.status_code}, {response.reason_phrase}) - {response.text}")
|
|
58
|
+
|
|
59
|
+
resp = response.json()
|
|
60
|
+
return resp["choices"][0]["message"]["content"]
|
|
61
|
+
|
|
62
|
+
class WebSearchAgent:
|
|
63
|
+
|
|
64
|
+
def __init__(self, model="Qwen/Qwen2-72B-Instruct"):
|
|
65
|
+
self.webs = WEBS()
|
|
66
|
+
self.deepinfra = DeepInfra(model=model)
|
|
67
|
+
|
|
68
|
+
def generate_search_query(self, information):
|
|
69
|
+
prompt = f"""
|
|
70
|
+
Instructions:
|
|
71
|
+
You are a smart online searcher for a large language model.
|
|
72
|
+
Given information, you must create a search query to search the internet for relevant information.
|
|
73
|
+
Your search query must be in the form of a json response.
|
|
74
|
+
Exact json response format must be as follows:
|
|
75
|
+
|
|
76
|
+
{{
|
|
77
|
+
"search_query": "your search query"
|
|
78
|
+
}}
|
|
79
|
+
- You must only provide ONE search query
|
|
80
|
+
- You must provide the BEST search query for the given information
|
|
81
|
+
- The search query must be normal text.
|
|
82
|
+
|
|
83
|
+
Information: {information}
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
response = self.deepinfra.ask(prompt)
|
|
87
|
+
return json.loads(response)["search_query"]
|
|
88
|
+
|
|
89
|
+
def search(self, information, region='wt-wt', safesearch='off', timelimit='y', max_results=5):
|
|
90
|
+
search_query = self.generate_search_query(information)
|
|
91
|
+
|
|
92
|
+
results = []
|
|
93
|
+
with self.webs as webs:
|
|
94
|
+
for result in webs.text(search_query, region=region, safesearch=safesearch, timelimit=timelimit, max_results=max_results):
|
|
95
|
+
results.append(result)
|
|
96
|
+
|
|
97
|
+
return results
|
|
98
|
+
|
|
99
|
+
def extract_urls(self, results):
|
|
100
|
+
urls = []
|
|
101
|
+
for result in results:
|
|
102
|
+
url = result.get('href')
|
|
103
|
+
if url:
|
|
104
|
+
urls.append(url)
|
|
105
|
+
return list(set(urls)) # Remove duplicates
|
|
106
|
+
|
|
107
|
+
def fetch_webpage(self, url: str) -> str:
|
|
108
|
+
try:
|
|
109
|
+
response = httpx.get(url, timeout=120)
|
|
110
|
+
if response.status_code == 200:
|
|
111
|
+
html = response.text
|
|
112
|
+
soup = BeautifulSoup(html, 'html.parser')
|
|
113
|
+
|
|
114
|
+
# Extract text from <p> tags
|
|
115
|
+
paragraphs = soup.find_all('p')
|
|
116
|
+
text = ' '.join([p.get_text() for p in paragraphs])
|
|
117
|
+
|
|
118
|
+
# Limit the text to around 4000 words
|
|
119
|
+
words = text.split()
|
|
120
|
+
if len(words) > 4000:
|
|
121
|
+
text = ' '.join(words[:4000]) + '...'
|
|
122
|
+
|
|
123
|
+
return text
|
|
124
|
+
else:
|
|
125
|
+
return f"Failed to fetch {url}: HTTP {response.status}"
|
|
126
|
+
except Exception as e:
|
|
127
|
+
return f"Error fetching {url}: {str(e)}"
|
|
128
|
+
|
|
129
|
+
def fetch_all_webpages(self, urls: List[str]) -> List[Dict[str, str]]:
|
|
130
|
+
contents = []
|
|
131
|
+
for url in urls:
|
|
132
|
+
content = self.fetch_webpage(url)
|
|
133
|
+
contents.append({"url": url, "content": content})
|
|
134
|
+
return contents
|
|
135
|
+
|
|
136
|
+
class OnlineSearcher:
|
|
137
|
+
def __init__(self, model="meta-llama/Meta-Llama-3.1-405B-Instruct"):
|
|
138
|
+
self.agent = WebSearchAgent(model)
|
|
139
|
+
self.deepinfra = DeepInfra(model="model")
|
|
140
|
+
|
|
141
|
+
def answer_question(self, question: str) -> str:
|
|
142
|
+
# Perform web search
|
|
143
|
+
search_results = self.agent.search(question)
|
|
144
|
+
|
|
145
|
+
# Extract URLs
|
|
146
|
+
urls = self.agent.extract_urls(search_results)
|
|
147
|
+
|
|
148
|
+
# Fetch webpage contents
|
|
149
|
+
webpage_contents = self.agent.fetch_all_webpages(urls)
|
|
150
|
+
|
|
151
|
+
# Prepare context for AI
|
|
152
|
+
context = "Based on the following search results and webpage contents:\n\n"
|
|
153
|
+
for i, result in enumerate(search_results, 1):
|
|
154
|
+
context += f"{i}. Title: {result['title']}\n URL: {result['href']}\n Snippet: {result['body']}\n\n"
|
|
155
|
+
|
|
156
|
+
context += "Extracted webpage contents:\n"
|
|
157
|
+
for i, webpage in enumerate(webpage_contents):
|
|
158
|
+
context += f"{i}. URL: {webpage['url']}\n Content: {webpage['content'][:4000]}...\n\n"
|
|
159
|
+
|
|
160
|
+
# Generate answer using AI
|
|
161
|
+
prompt = f"{context}\n\nQuestion: {question}\n\nPlease provide a comprehensive answer to the question based on the search results and webpage contents above. Include relevant webpage URLs in your answer when appropriate. If the search results and webpage contents don't contain relevant information, please state that and provide the best answer you can based on your general knowledge. [YOUR RESPONSE WITH SOURCE LINKS ([➊](URL))"
|
|
162
|
+
|
|
163
|
+
answer = self.deepinfra.ask(prompt)
|
|
164
|
+
return answer
|
|
165
|
+
|
|
166
|
+
# Usage example
|
|
167
|
+
if __name__ == "__main__":
|
|
168
|
+
assistant = OnlineSearcher()
|
|
169
|
+
while True:
|
|
170
|
+
question = input(">>> ")
|
|
171
|
+
if question.lower() == 'quit':
|
|
172
|
+
break
|
|
173
|
+
answer = assistant.answer_question(question)
|
|
174
|
+
print(answer)
|
|
175
|
+
print("\n" + "-"*50 + "\n")
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from webscout import DeepInfra, WEBS
|
|
4
|
+
|
|
5
|
+
class FunctionCallingAgent:
|
|
6
|
+
def __init__(self, model: str = "Qwen/Qwen2-72B-Instruct", system_prompt: str = 'You are a helpful assistant that will always answer what user wants', tools: list = None):
|
|
7
|
+
self.deepinfra = DeepInfra(model=model, system_prompt=system_prompt)
|
|
8
|
+
self.tools = tools if tools is not None else []
|
|
9
|
+
# logging.basicConfig(level=logging.INFO)
|
|
10
|
+
# self.webs = WEBS() # Initialize a WEBS object for web search
|
|
11
|
+
|
|
12
|
+
def function_call_handler(self, message_text: str):
|
|
13
|
+
"""Handles function calls based on the provided message text
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
message_text (str): The input message text from the user.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
dict: The extracted function call and arguments.
|
|
20
|
+
"""
|
|
21
|
+
system_message = f'[SYSTEM]You are a helpful assistant. You have access to the following functions: \n {str(self.tools)}\n\nTo use these functions respond with:\n<functioncall> {{ "name": "function_name", "arguments": {{ "arg_1": "value_1", "arg_2": "value_2", ... }} }} </functioncall> [USER] {message_text}'
|
|
22
|
+
|
|
23
|
+
response = self.deepinfra.chat(system_message)
|
|
24
|
+
# logging.info(f"Raw response: {response}")
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
# Extract the JSON-like part of the response
|
|
28
|
+
start_idx = response.find("{")
|
|
29
|
+
end_idx = response.rfind("}") + 1
|
|
30
|
+
if start_idx == -1 or end_idx == -1:
|
|
31
|
+
raise ValueError("JSON-like structure not found in the response")
|
|
32
|
+
|
|
33
|
+
response_json_str = response[start_idx:end_idx]
|
|
34
|
+
# Ensure the JSON string is properly formatted
|
|
35
|
+
response_json_str = response_json_str.replace("'", '"') # Replace single quotes with double quotes
|
|
36
|
+
response_json_str = response_json_str.strip()
|
|
37
|
+
response_data = json.loads(response_json_str)
|
|
38
|
+
except (ValueError, json.JSONDecodeError) as e:
|
|
39
|
+
# logging.error(f"An error occurred while parsing response: {e}")
|
|
40
|
+
return {"error": str(e)}
|
|
41
|
+
|
|
42
|
+
return response_data
|
|
43
|
+
|
|
44
|
+
def execute_function(self, function_call_data: dict) -> str:
|
|
45
|
+
"""Executes the specified function with the provided arguments.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
function_call_data (dict): A dictionary containing the function name and arguments.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
str: The result of the function execution.
|
|
52
|
+
"""
|
|
53
|
+
function_name = function_call_data.get("name")
|
|
54
|
+
arguments = function_call_data.get("arguments", "{}") # Default to empty dict if not present
|
|
55
|
+
|
|
56
|
+
# Parse the arguments string into a dictionary
|
|
57
|
+
try:
|
|
58
|
+
arguments_dict = json.loads(arguments)
|
|
59
|
+
except json.JSONDecodeError:
|
|
60
|
+
# logging.error("Failed to parse arguments as JSON.")
|
|
61
|
+
return "Invalid arguments format."
|
|
62
|
+
|
|
63
|
+
# logging.info(f"Executing function: {function_name} with arguments: {arguments_dict}")
|
|
64
|
+
|
|
65
|
+
# if function_name == "web_search":
|
|
66
|
+
# query = arguments_dict.get("query")
|
|
67
|
+
# if query:
|
|
68
|
+
# search_results = self.webs.text(query)
|
|
69
|
+
# # You can process the search results here, e.g., extract URLs, summarize, etc.
|
|
70
|
+
# return f"Here's what I found:\n\n{search_results}"
|
|
71
|
+
# else:
|
|
72
|
+
# return "Please provide a search query."
|
|
73
|
+
# else:
|
|
74
|
+
# return f"Function '{function_name}' is not yet implemented."
|
|
75
|
+
|
|
76
|
+
# Example usage
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
tools = [
|
|
79
|
+
{
|
|
80
|
+
"type": "function",
|
|
81
|
+
"function": {
|
|
82
|
+
"name": "UserDetail",
|
|
83
|
+
"parameters": {
|
|
84
|
+
"type": "object",
|
|
85
|
+
"title": "UserDetail",
|
|
86
|
+
"properties": {
|
|
87
|
+
"name": {
|
|
88
|
+
"title": "Name",
|
|
89
|
+
"type": "string"
|
|
90
|
+
},
|
|
91
|
+
"age": {
|
|
92
|
+
"title": "Age",
|
|
93
|
+
"type": "integer"
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
"required": ["name", "age"]
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"type": "function",
|
|
102
|
+
"function": {
|
|
103
|
+
"name": "web_search",
|
|
104
|
+
"description": "Search query on google",
|
|
105
|
+
"parameters": {
|
|
106
|
+
"type": "object",
|
|
107
|
+
"properties": {
|
|
108
|
+
"query": {
|
|
109
|
+
"type": "string",
|
|
110
|
+
"description": "web search query"
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
"required": ["query"]
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
agent = FunctionCallingAgent(tools=tools)
|
|
120
|
+
message = "tell me about HelpingAI flash"
|
|
121
|
+
function_call_data = agent.function_call_handler(message)
|
|
122
|
+
print(f"Function Call Data: {function_call_data}")
|
|
123
|
+
|
|
124
|
+
if "error" not in function_call_data:
|
|
125
|
+
result = agent.execute_function(function_call_data)
|
|
126
|
+
# print(f"Function Execution Result: {result}")
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import uuid
|
|
3
|
+
from selenium import webdriver
|
|
4
|
+
from selenium.webdriver.chrome.options import Options
|
|
5
|
+
from selenium.webdriver.common.by import By
|
|
6
|
+
from selenium.webdriver.support import expected_conditions as EC
|
|
7
|
+
from selenium.webdriver.support.ui import WebDriverWait
|
|
8
|
+
import click
|
|
9
|
+
import requests
|
|
10
|
+
from requests import get
|
|
11
|
+
from uuid import uuid4
|
|
12
|
+
from re import findall
|
|
13
|
+
from requests.exceptions import RequestException
|
|
14
|
+
from curl_cffi.requests import get, RequestsError
|
|
15
|
+
import g4f
|
|
16
|
+
from random import randint
|
|
17
|
+
from PIL import Image
|
|
18
|
+
import io
|
|
19
|
+
import re
|
|
20
|
+
import json
|
|
21
|
+
import yaml
|
|
22
|
+
from webscout.AIutel import Optimizers
|
|
23
|
+
from webscout.AIutel import Conversation
|
|
24
|
+
from webscout.AIutel import AwesomePrompts, sanitize_stream
|
|
25
|
+
from webscout.AIbase import Provider, AsyncProvider
|
|
26
|
+
from webscout import exceptions
|
|
27
|
+
from typing import Any, AsyncGenerator, Dict
|
|
28
|
+
import logging
|
|
29
|
+
import httpx
|
|
30
|
+
from webscout import WEBS
|
|
31
|
+
from rich import print
|
|
32
|
+
|
|
33
|
+
class AndiSearch(Provider):
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
is_conversation: bool = True,
|
|
37
|
+
max_tokens: int = 600,
|
|
38
|
+
timeout: int = 30,
|
|
39
|
+
intro: str = None,
|
|
40
|
+
filepath: str = None,
|
|
41
|
+
update_file: bool = True,
|
|
42
|
+
proxies: dict = {},
|
|
43
|
+
history_offset: int = 10250,
|
|
44
|
+
act: str = None,
|
|
45
|
+
):
|
|
46
|
+
"""Instantiates AndiSearch
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
|
|
50
|
+
max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
|
|
51
|
+
timeout (int, optional): Http request timeout. Defaults to 30.
|
|
52
|
+
intro (str, optional): Conversation introductory prompt. Defaults to None.
|
|
53
|
+
filepath (str, optional): Path to file containing conversation history. Defaults to None.
|
|
54
|
+
update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
|
|
55
|
+
proxies (dict, optional): Http request proxies. Defaults to {}.
|
|
56
|
+
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
57
|
+
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
58
|
+
"""
|
|
59
|
+
self.session = requests.Session()
|
|
60
|
+
self.is_conversation = is_conversation
|
|
61
|
+
self.max_tokens_to_sample = max_tokens
|
|
62
|
+
self.chat_endpoint = "https://write.andisearch.com/v1/write_streaming"
|
|
63
|
+
self.stream_chunk_size = 64
|
|
64
|
+
self.timeout = timeout
|
|
65
|
+
self.last_response = {}
|
|
66
|
+
self.headers = {
|
|
67
|
+
"accept": "text/event-stream",
|
|
68
|
+
"accept-encoding": "gzip, deflate, br, zstd",
|
|
69
|
+
"accept-language": "en-US,en;q=0.9,en-IN;q=0.8",
|
|
70
|
+
"andi-auth-key": "andi-summarizer",
|
|
71
|
+
"andi-origin": "x-andi-origin",
|
|
72
|
+
"authorization": str(uuid4()),
|
|
73
|
+
"content-type": "application/json",
|
|
74
|
+
"dnt": "1",
|
|
75
|
+
"origin": "https://andisearch.com",
|
|
76
|
+
"priority": "u=1, i",
|
|
77
|
+
"sec-ch-ua": '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
|
|
78
|
+
"sec-ch-ua-mobile": "?0",
|
|
79
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
80
|
+
"sec-fetch-dest": "empty",
|
|
81
|
+
"sec-fetch-mode": "cors",
|
|
82
|
+
"sec-fetch-site": "same-site",
|
|
83
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0",
|
|
84
|
+
"x-amz-date": "20240730T031106Z",
|
|
85
|
+
"x-amz-security-token": str(uuid4()),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
self.__available_optimizers = (
|
|
89
|
+
method
|
|
90
|
+
for method in dir(Optimizers)
|
|
91
|
+
if callable(getattr(Optimizers, method)) and not method.startswith("__")
|
|
92
|
+
)
|
|
93
|
+
self.session.headers.update(self.headers)
|
|
94
|
+
Conversation.intro = (
|
|
95
|
+
AwesomePrompts().get_act(
|
|
96
|
+
act, raise_not_found=True, default=None, case_insensitive=True
|
|
97
|
+
)
|
|
98
|
+
if act
|
|
99
|
+
else intro or Conversation.intro
|
|
100
|
+
)
|
|
101
|
+
self.conversation = Conversation(
|
|
102
|
+
is_conversation, self.max_tokens_to_sample, filepath, update_file
|
|
103
|
+
)
|
|
104
|
+
self.conversation.history_offset = history_offset
|
|
105
|
+
self.session.proxies = proxies
|
|
106
|
+
|
|
107
|
+
def ask(
|
|
108
|
+
self,
|
|
109
|
+
prompt: str,
|
|
110
|
+
stream: bool = False,
|
|
111
|
+
raw: bool = False,
|
|
112
|
+
optimizer: str = None,
|
|
113
|
+
conversationally: bool = False,
|
|
114
|
+
) -> dict:
|
|
115
|
+
"""Chat with AI
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
prompt (str): Prompt to be send.
|
|
119
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
120
|
+
raw (bool, optional): Stream back raw response as received. Defaults to False.
|
|
121
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
122
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
123
|
+
Returns:
|
|
124
|
+
dict : {}
|
|
125
|
+
```json
|
|
126
|
+
{
|
|
127
|
+
"id": "chatcmpl-TaREJpBZsRVQFRFic1wIA7Q7XfnaD",
|
|
128
|
+
"object": "chat.completion",
|
|
129
|
+
"created": 1704623244,
|
|
130
|
+
"model": "gpt-3.5-turbo",
|
|
131
|
+
"usage": {
|
|
132
|
+
"prompt_tokens": 0,
|
|
133
|
+
"completion_tokens": 0,
|
|
134
|
+
"total_tokens": 0
|
|
135
|
+
},
|
|
136
|
+
"choices": [
|
|
137
|
+
{
|
|
138
|
+
"message": {
|
|
139
|
+
"role": "assistant",
|
|
140
|
+
"content": "Hello! How can I assist you today?"
|
|
141
|
+
},
|
|
142
|
+
"finish_reason": "stop",
|
|
143
|
+
"index": 0
|
|
144
|
+
}
|
|
145
|
+
]
|
|
146
|
+
}
|
|
147
|
+
```
|
|
148
|
+
"""
|
|
149
|
+
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
|
|
150
|
+
if optimizer:
|
|
151
|
+
if optimizer in self.__available_optimizers:
|
|
152
|
+
conversation_prompt = getattr(Optimizers, optimizer)(
|
|
153
|
+
conversation_prompt if conversationally else prompt
|
|
154
|
+
)
|
|
155
|
+
else:
|
|
156
|
+
raise Exception(
|
|
157
|
+
f"Optimizer is not one of {self.__available_optimizers}"
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# Initialize the webscout instance
|
|
161
|
+
webs = WEBS()
|
|
162
|
+
|
|
163
|
+
# Fetch search results
|
|
164
|
+
search_query = prompt
|
|
165
|
+
search_results = webs.text(search_query, max_results=7)
|
|
166
|
+
|
|
167
|
+
# Format the search results into the required serp payload structure
|
|
168
|
+
serp_payload = {
|
|
169
|
+
"query": search_query,
|
|
170
|
+
"serp": {
|
|
171
|
+
"results_type": "Search",
|
|
172
|
+
"answer": "",
|
|
173
|
+
"type": "navigation",
|
|
174
|
+
"title": "",
|
|
175
|
+
"description": "",
|
|
176
|
+
"image": "",
|
|
177
|
+
"link": "",
|
|
178
|
+
"source": "liftndrift.com",
|
|
179
|
+
"engine": "andi-b",
|
|
180
|
+
"results": [
|
|
181
|
+
{
|
|
182
|
+
"title": result["title"],
|
|
183
|
+
"link": result["href"],
|
|
184
|
+
"desc": result["body"],
|
|
185
|
+
"image": "",
|
|
186
|
+
"type": "website",
|
|
187
|
+
"source": result["href"].split("//")[1].split("/")[0] # Extract the domain name
|
|
188
|
+
}
|
|
189
|
+
for result in search_results
|
|
190
|
+
]
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
self.session.headers.update(self.headers)
|
|
194
|
+
payload = serp_payload
|
|
195
|
+
|
|
196
|
+
def for_stream():
|
|
197
|
+
response = self.session.post(
|
|
198
|
+
self.chat_endpoint, json=payload, stream=True, timeout=self.timeout
|
|
199
|
+
)
|
|
200
|
+
if not response.ok:
|
|
201
|
+
raise exceptions.FailedToGenerateResponseError(
|
|
202
|
+
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
streaming_text = ""
|
|
206
|
+
for value in response.iter_lines(
|
|
207
|
+
decode_unicode=True,
|
|
208
|
+
chunk_size=self.stream_chunk_size,
|
|
209
|
+
delimiter="\n",
|
|
210
|
+
):
|
|
211
|
+
try:
|
|
212
|
+
if bool(value):
|
|
213
|
+
streaming_text += value + ("\n" if stream else "")
|
|
214
|
+
resp = dict(text=streaming_text)
|
|
215
|
+
self.last_response.update(resp)
|
|
216
|
+
yield value if raw else resp
|
|
217
|
+
except json.decoder.JSONDecodeError:
|
|
218
|
+
pass
|
|
219
|
+
self.conversation.update_chat_history(
|
|
220
|
+
prompt, self.get_message(self.last_response)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def for_non_stream():
|
|
224
|
+
for _ in for_stream():
|
|
225
|
+
pass
|
|
226
|
+
return self.last_response
|
|
227
|
+
|
|
228
|
+
return for_stream() if stream else for_non_stream()
|
|
229
|
+
|
|
230
|
+
def chat(
|
|
231
|
+
self,
|
|
232
|
+
prompt: str,
|
|
233
|
+
stream: bool = False,
|
|
234
|
+
optimizer: str = None,
|
|
235
|
+
conversationally: bool = False,
|
|
236
|
+
) -> str:
|
|
237
|
+
"""Generate response `str`
|
|
238
|
+
Args:
|
|
239
|
+
prompt (str): Prompt to be send.
|
|
240
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
241
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
242
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
243
|
+
Returns:
|
|
244
|
+
str: Response generated
|
|
245
|
+
"""
|
|
246
|
+
|
|
247
|
+
def for_stream():
|
|
248
|
+
for response in self.ask(
|
|
249
|
+
prompt, True, optimizer=optimizer, conversationally=conversationally
|
|
250
|
+
):
|
|
251
|
+
yield self.get_message(response)
|
|
252
|
+
|
|
253
|
+
def for_non_stream():
|
|
254
|
+
return self.get_message(
|
|
255
|
+
self.ask(
|
|
256
|
+
prompt,
|
|
257
|
+
False,
|
|
258
|
+
optimizer=optimizer,
|
|
259
|
+
conversationally=conversationally,
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
return for_stream() if stream else for_non_stream()
|
|
264
|
+
|
|
265
|
+
def get_message(self, response: dict) -> str:
|
|
266
|
+
"""Retrieves message only from response
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
response (dict): Response generated by `self.ask`
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
str: Message extracted
|
|
273
|
+
"""
|
|
274
|
+
assert isinstance(response, dict), "Response should be of dict data-type only"
|
|
275
|
+
return response["text"]
|
webscout/Provider/__init__.py
CHANGED
|
@@ -38,6 +38,7 @@ from .Geminipro import GEMINIPRO
|
|
|
38
38
|
from .Geminiflash import GEMINIFLASH
|
|
39
39
|
from .OLLAMA import OLLAMA
|
|
40
40
|
from .FreeGemini import FreeGemini
|
|
41
|
+
from .Andi import AndiSearch
|
|
41
42
|
__all__ = [
|
|
42
43
|
'ThinkAnyAI',
|
|
43
44
|
'Xjai',
|
|
@@ -78,7 +79,8 @@ __all__ = [
|
|
|
78
79
|
'GEMINIPRO',
|
|
79
80
|
'GEMINIFLASH',
|
|
80
81
|
'OLLAMA',
|
|
81
|
-
'FreeGemini'
|
|
82
|
+
'FreeGemini',
|
|
83
|
+
'AndiSearch'
|
|
82
84
|
|
|
83
85
|
|
|
84
86
|
]
|
webscout/__init__.py
CHANGED
|
@@ -7,14 +7,14 @@ from .voice import play_audio
|
|
|
7
7
|
from .websx_search import WEBSX
|
|
8
8
|
from .LLM import VLM, LLM
|
|
9
9
|
from .YTdownloader import *
|
|
10
|
-
|
|
10
|
+
|
|
11
11
|
import g4f
|
|
12
12
|
from .YTdownloader import *
|
|
13
13
|
from .Provider import *
|
|
14
14
|
from .Extra import gguf
|
|
15
15
|
from .Extra import autollama
|
|
16
16
|
from .Extra import weather_ascii, weather
|
|
17
|
-
|
|
17
|
+
from .Agents import *
|
|
18
18
|
__repo__ = "https://github.com/OE-LUCIFER/Webscout"
|
|
19
19
|
|
|
20
20
|
webai = [
|
|
@@ -44,7 +44,8 @@ webai = [
|
|
|
44
44
|
"vtlchat",
|
|
45
45
|
"geminiflash",
|
|
46
46
|
"geminipro",
|
|
47
|
-
"ollama"
|
|
47
|
+
"ollama",
|
|
48
|
+
"andi",
|
|
48
49
|
]
|
|
49
50
|
|
|
50
51
|
gpt4free_providers = [
|
webscout/voice.py
CHANGED
|
@@ -24,4 +24,11 @@ def play_audio(message: str, voice: str = "Brian") -> typing.Union[str, typing.N
|
|
|
24
24
|
result = requests.get(url=url, headers=headers)
|
|
25
25
|
return result.content
|
|
26
26
|
except:
|
|
27
|
-
return None
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
if __name__ == "__main__":
|
|
30
|
+
# Example usage of the play_audio function
|
|
31
|
+
message = "Hello, world!"
|
|
32
|
+
voice = "Brian"
|
|
33
|
+
audio_result = play_audio(message, voice)
|
|
34
|
+
print(audio_result)
|
webscout/webai.py
CHANGED
|
@@ -765,7 +765,20 @@ class Main(cmd.Cmd):
|
|
|
765
765
|
model=getOr(model, "Phind Model"),
|
|
766
766
|
quiet=quiet,
|
|
767
767
|
)
|
|
768
|
+
elif provider == "andi":
|
|
769
|
+
from webscout import AndiSearch
|
|
768
770
|
|
|
771
|
+
self.bot = AndiSearch(
|
|
772
|
+
is_conversation=disable_conversation,
|
|
773
|
+
max_tokens=max_tokens,
|
|
774
|
+
timeout=timeout,
|
|
775
|
+
intro=intro,
|
|
776
|
+
filepath=filepath,
|
|
777
|
+
update_file=update_file,
|
|
778
|
+
proxies=proxies,
|
|
779
|
+
history_offset=history_offset,
|
|
780
|
+
act=awesome_prompt,
|
|
781
|
+
)
|
|
769
782
|
elif provider == "blackboxai":
|
|
770
783
|
|
|
771
784
|
from webscout import BLACKBOXAI
|
webscout/webscout_search.py
CHANGED
webscout/websx_search.py
CHANGED
|
@@ -1,370 +1,19 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import os
|
|
4
|
-
from typing import Any, Dict, Optional
|
|
5
|
-
import json
|
|
6
|
-
from typing import Any, Dict, List, Optional
|
|
7
|
-
|
|
8
|
-
import aiohttp
|
|
9
1
|
import requests
|
|
10
|
-
from
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
Args:
|
|
30
|
-
env_var (str): The name of the environment variable.
|
|
31
|
-
|
|
32
|
-
Returns:
|
|
33
|
-
bool: True if the environment variable is set, False otherwise.
|
|
34
|
-
"""
|
|
35
|
-
return env_var in os.environ and os.environ[env_var] not in (
|
|
36
|
-
"",
|
|
37
|
-
"0",
|
|
38
|
-
"false",
|
|
39
|
-
"False",
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def get_from_dict_or_env(
|
|
44
|
-
data: Dict[str, Any], key: str, env_key: str, default: Optional[str] = None
|
|
45
|
-
) -> str:
|
|
46
|
-
"""Get a value from a dictionary or an environment variable."""
|
|
47
|
-
if key in data and data[key]:
|
|
48
|
-
return data[key]
|
|
49
|
-
else:
|
|
50
|
-
return get_from_env(key, env_key, default=default)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def get_from_env(key: str, env_key: str, default: Optional[str] = None) -> str:
|
|
54
|
-
"""Get a value from a dictionary or an environment variable."""
|
|
55
|
-
if env_key in os.environ and os.environ[env_key]:
|
|
56
|
-
return os.environ[env_key]
|
|
57
|
-
elif default is not None:
|
|
58
|
-
return default
|
|
59
|
-
else:
|
|
60
|
-
raise ValueError(
|
|
61
|
-
f"Did not find {key}, please add an environment variable"
|
|
62
|
-
f" `{env_key}` which contains it, or pass"
|
|
63
|
-
f" `{key}` as a named parameter."
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def _get_default_params() -> dict:
|
|
68
|
-
return {"language": "en", "format": "json"}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
class WEBSXResults(dict):
|
|
72
|
-
"""Dict like wrapper around search api results."""
|
|
73
|
-
|
|
74
|
-
_data: str = ""
|
|
75
|
-
|
|
76
|
-
def __init__(self, data: str):
|
|
77
|
-
"""Take a raw result from WEBSX and make it into a dict like object."""
|
|
78
|
-
json_data = json.loads(data)
|
|
79
|
-
super().__init__(json_data)
|
|
80
|
-
self.__dict__ = self
|
|
81
|
-
|
|
82
|
-
def __str__(self) -> str:
|
|
83
|
-
"""Text representation of WEBSX result."""
|
|
84
|
-
return self._data
|
|
85
|
-
|
|
86
|
-
@property
|
|
87
|
-
def results(self) -> Any:
|
|
88
|
-
"""Silence mypy for accessing this field.
|
|
89
|
-
|
|
90
|
-
:meta private:
|
|
91
|
-
"""
|
|
92
|
-
return self.get("results")
|
|
93
|
-
|
|
94
|
-
@property
|
|
95
|
-
def answers(self) -> Any:
|
|
96
|
-
"""Helper accessor on the json result."""
|
|
97
|
-
return self.get("answers")
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
class WEBSX(BaseModel):
|
|
101
|
-
|
|
102
|
-
_result: WEBSXResults = PrivateAttr()
|
|
103
|
-
WEBSX_host: str = "https://8080-01j06maryf9vpw2mm2afqkypps.cloudspaces.litng.ai/"
|
|
104
|
-
unsecure: bool = False
|
|
105
|
-
params: dict = Field(default_factory=_get_default_params)
|
|
106
|
-
headers: Optional[dict] = None
|
|
107
|
-
engines: Optional[List[str]] = []
|
|
108
|
-
categories: Optional[List[str]] = []
|
|
109
|
-
query_suffix: Optional[str] = ""
|
|
110
|
-
k: int = 10
|
|
111
|
-
aiosession: Optional[Any] = None
|
|
112
|
-
|
|
113
|
-
@validator("unsecure")
|
|
114
|
-
def disable_ssl_warnings(cls, v: bool) -> bool:
|
|
115
|
-
"""Disable SSL warnings."""
|
|
116
|
-
if v:
|
|
117
|
-
# requests.urllib3.disable_warnings()
|
|
118
|
-
try:
|
|
119
|
-
import urllib3
|
|
120
|
-
|
|
121
|
-
urllib3.disable_warnings()
|
|
122
|
-
except ImportError as e:
|
|
123
|
-
print(e) # noqa: T201
|
|
124
|
-
|
|
125
|
-
return v
|
|
126
|
-
|
|
127
|
-
@root_validator()
|
|
128
|
-
def validate_params(cls, values: Dict) -> Dict:
|
|
129
|
-
"""Validate that custom WEBSX params are merged with default ones."""
|
|
130
|
-
user_params = values["params"]
|
|
131
|
-
default = _get_default_params()
|
|
132
|
-
values["params"] = {**default, **user_params}
|
|
133
|
-
|
|
134
|
-
engines = values.get("engines")
|
|
135
|
-
if engines:
|
|
136
|
-
values["params"]["engines"] = ",".join(engines)
|
|
137
|
-
|
|
138
|
-
categories = values.get("categories")
|
|
139
|
-
if categories:
|
|
140
|
-
values["params"]["categories"] = ",".join(categories)
|
|
141
|
-
|
|
142
|
-
WEBSX_host = get_from_dict_or_env(values, "WEBSX_host", "WEBSX_HOST")
|
|
143
|
-
if not WEBSX_host.startswith("http"):
|
|
144
|
-
print( # noqa: T201
|
|
145
|
-
f"Warning: missing the url scheme on host \
|
|
146
|
-
! assuming secure https://{WEBSX_host} "
|
|
147
|
-
)
|
|
148
|
-
WEBSX_host = "https://" + WEBSX_host
|
|
149
|
-
elif WEBSX_host.startswith("http://"):
|
|
150
|
-
values["unsecure"] = True
|
|
151
|
-
cls.disable_ssl_warnings(True)
|
|
152
|
-
values["WEBSX_host"] = WEBSX_host
|
|
153
|
-
|
|
154
|
-
return values
|
|
155
|
-
|
|
156
|
-
class Config:
|
|
157
|
-
"""Configuration for this pydantic object."""
|
|
158
|
-
|
|
159
|
-
extra = Extra.forbid
|
|
160
|
-
|
|
161
|
-
def _WEBSX_api_query(self, params: dict) -> WEBSXResults:
|
|
162
|
-
"""Actual request to WEBSX API."""
|
|
163
|
-
raw_result = requests.get(
|
|
164
|
-
self.WEBSX_host,
|
|
165
|
-
headers=self.headers,
|
|
166
|
-
params=params,
|
|
167
|
-
verify=not self.unsecure,
|
|
168
|
-
)
|
|
169
|
-
# test if http result is ok
|
|
170
|
-
if not raw_result.ok:
|
|
171
|
-
raise ValueError("WEBSX API returned an error: ", raw_result.text)
|
|
172
|
-
res = WEBSXResults(raw_result.text)
|
|
173
|
-
self._result = res
|
|
174
|
-
return res
|
|
175
|
-
|
|
176
|
-
async def _aWEBSX_api_query(self, params: dict) -> WEBSXResults:
|
|
177
|
-
if not self.aiosession:
|
|
178
|
-
async with aiohttp.ClientSession() as session:
|
|
179
|
-
async with session.get(
|
|
180
|
-
self.WEBSX_host,
|
|
181
|
-
headers=self.headers,
|
|
182
|
-
params=params,
|
|
183
|
-
ssl=(lambda: False if self.unsecure else None)(),
|
|
184
|
-
) as response:
|
|
185
|
-
if not response.ok:
|
|
186
|
-
raise ValueError("WEBSX API returned an error: ", response.text)
|
|
187
|
-
result = WEBSXResults(await response.text())
|
|
188
|
-
self._result = result
|
|
189
|
-
else:
|
|
190
|
-
async with self.aiosession.get(
|
|
191
|
-
self.WEBSX_host,
|
|
192
|
-
headers=self.headers,
|
|
193
|
-
params=params,
|
|
194
|
-
verify=not self.unsecure,
|
|
195
|
-
) as response:
|
|
196
|
-
if not response.ok:
|
|
197
|
-
raise ValueError("WEBSX API returned an error: ", response.text)
|
|
198
|
-
result = WEBSXResults(await response.text())
|
|
199
|
-
self._result = result
|
|
200
|
-
|
|
201
|
-
return result
|
|
202
|
-
|
|
203
|
-
def run(
|
|
204
|
-
self,
|
|
205
|
-
query: str,
|
|
206
|
-
engines: Optional[List[str]] = None,
|
|
207
|
-
categories: Optional[List[str]] = None,
|
|
208
|
-
query_suffix: Optional[str] = "",
|
|
209
|
-
**kwargs: Any,
|
|
210
|
-
) -> str:
|
|
211
|
-
_params = {
|
|
212
|
-
"q": query,
|
|
213
|
-
}
|
|
214
|
-
params = {**self.params, **_params, **kwargs}
|
|
215
|
-
|
|
216
|
-
if self.query_suffix and len(self.query_suffix) > 0:
|
|
217
|
-
params["q"] += " " + self.query_suffix
|
|
218
|
-
|
|
219
|
-
if isinstance(query_suffix, str) and len(query_suffix) > 0:
|
|
220
|
-
params["q"] += " " + query_suffix
|
|
221
|
-
|
|
222
|
-
if isinstance(engines, list) and len(engines) > 0:
|
|
223
|
-
params["engines"] = ",".join(engines)
|
|
224
|
-
|
|
225
|
-
if isinstance(categories, list) and len(categories) > 0:
|
|
226
|
-
params["categories"] = ",".join(categories)
|
|
227
|
-
|
|
228
|
-
res = self._WEBSX_api_query(params)
|
|
229
|
-
|
|
230
|
-
if len(res.answers) > 0:
|
|
231
|
-
toret = res.answers[0]
|
|
232
|
-
|
|
233
|
-
# only return the content of the results list
|
|
234
|
-
elif len(res.results) > 0:
|
|
235
|
-
toret = "\n\n".join([r.get("content", "") for r in res.results[: self.k]])
|
|
236
|
-
else:
|
|
237
|
-
toret = "No good search result found"
|
|
238
|
-
|
|
239
|
-
return toret
|
|
240
|
-
|
|
241
|
-
async def arun(
|
|
242
|
-
self,
|
|
243
|
-
query: str,
|
|
244
|
-
engines: Optional[List[str]] = None,
|
|
245
|
-
query_suffix: Optional[str] = "",
|
|
246
|
-
**kwargs: Any,
|
|
247
|
-
) -> str:
|
|
248
|
-
"""Asynchronously version of `run`."""
|
|
249
|
-
_params = {
|
|
250
|
-
"q": query,
|
|
251
|
-
}
|
|
252
|
-
params = {**self.params, **_params, **kwargs}
|
|
253
|
-
|
|
254
|
-
if self.query_suffix and len(self.query_suffix) > 0:
|
|
255
|
-
params["q"] += " " + self.query_suffix
|
|
256
|
-
|
|
257
|
-
if isinstance(query_suffix, str) and len(query_suffix) > 0:
|
|
258
|
-
params["q"] += " " + query_suffix
|
|
259
|
-
|
|
260
|
-
if isinstance(engines, list) and len(engines) > 0:
|
|
261
|
-
params["engines"] = ",".join(engines)
|
|
262
|
-
|
|
263
|
-
res = await self._aWEBSX_api_query(params)
|
|
264
|
-
|
|
265
|
-
if len(res.answers) > 0:
|
|
266
|
-
toret = res.answers[0]
|
|
267
|
-
|
|
268
|
-
# only return the content of the results list
|
|
269
|
-
elif len(res.results) > 0:
|
|
270
|
-
toret = "\n\n".join([r.get("content", "") for r in res.results[: self.k]])
|
|
271
|
-
else:
|
|
272
|
-
toret = "No good search result found"
|
|
273
|
-
|
|
274
|
-
return toret
|
|
275
|
-
|
|
276
|
-
def results(
|
|
277
|
-
self,
|
|
278
|
-
query: str,
|
|
279
|
-
num_results: int,
|
|
280
|
-
engines: Optional[List[str]] = None,
|
|
281
|
-
categories: Optional[List[str]] = None,
|
|
282
|
-
query_suffix: Optional[str] = "",
|
|
283
|
-
**kwargs: Any,
|
|
284
|
-
) -> List[Dict]:
|
|
285
|
-
"""Run query through WEBSX API and returns the results with metadata.
|
|
286
|
-
|
|
287
|
-
Args:
|
|
288
|
-
query: The query to search for.
|
|
289
|
-
query_suffix: Extra suffix appended to the query.
|
|
290
|
-
num_results: Limit the number of results to return.
|
|
291
|
-
engines: List of engines to use for the query.
|
|
292
|
-
categories: List of categories to use for the query.
|
|
293
|
-
**kwargs: extra parameters to pass to the WEBSX API.
|
|
294
|
-
|
|
295
|
-
Returns:
|
|
296
|
-
Dict with the following keys:
|
|
297
|
-
{
|
|
298
|
-
snippet: The description of the result.
|
|
299
|
-
title: The title of the result.
|
|
300
|
-
link: The link to the result.
|
|
301
|
-
engines: The engines used for the result.
|
|
302
|
-
category: WEBSX category of the result.
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
"""
|
|
306
|
-
_params = {
|
|
307
|
-
"q": query,
|
|
308
|
-
}
|
|
309
|
-
params = {**self.params, **_params, **kwargs}
|
|
310
|
-
if self.query_suffix and len(self.query_suffix) > 0:
|
|
311
|
-
params["q"] += " " + self.query_suffix
|
|
312
|
-
if isinstance(query_suffix, str) and len(query_suffix) > 0:
|
|
313
|
-
params["q"] += " " + query_suffix
|
|
314
|
-
if isinstance(engines, list) and len(engines) > 0:
|
|
315
|
-
params["engines"] = ",".join(engines)
|
|
316
|
-
if isinstance(categories, list) and len(categories) > 0:
|
|
317
|
-
params["categories"] = ",".join(categories)
|
|
318
|
-
results = self._WEBSX_api_query(params).results[:num_results]
|
|
319
|
-
if len(results) == 0:
|
|
320
|
-
return [{"Result": "No good Search Result was found"}]
|
|
321
|
-
|
|
322
|
-
return [
|
|
323
|
-
{
|
|
324
|
-
"snippet": result.get("content", ""),
|
|
325
|
-
"title": result["title"],
|
|
326
|
-
"link": result["url"],
|
|
327
|
-
"engines": result["engines"],
|
|
328
|
-
"category": result["category"],
|
|
329
|
-
}
|
|
330
|
-
for result in results
|
|
331
|
-
]
|
|
332
|
-
|
|
333
|
-
async def aresults(
|
|
334
|
-
self,
|
|
335
|
-
query: str,
|
|
336
|
-
num_results: int,
|
|
337
|
-
engines: Optional[List[str]] = None,
|
|
338
|
-
query_suffix: Optional[str] = "",
|
|
339
|
-
**kwargs: Any,
|
|
340
|
-
) -> List[Dict]:
|
|
341
|
-
"""Asynchronously query with json results.
|
|
342
|
-
|
|
343
|
-
Uses aiohttp. See `results` for more info.
|
|
344
|
-
"""
|
|
345
|
-
_params = {
|
|
346
|
-
"q": query,
|
|
347
|
-
}
|
|
348
|
-
params = {**self.params, **_params, **kwargs}
|
|
349
|
-
|
|
350
|
-
if self.query_suffix and len(self.query_suffix) > 0:
|
|
351
|
-
params["q"] += " " + self.query_suffix
|
|
352
|
-
if isinstance(query_suffix, str) and len(query_suffix) > 0:
|
|
353
|
-
params["q"] += " " + query_suffix
|
|
354
|
-
if isinstance(engines, list) and len(engines) > 0:
|
|
355
|
-
params["engines"] = ",".join(engines)
|
|
356
|
-
results = (await self._aWEBSX_api_query(params)).results[:num_results]
|
|
357
|
-
if len(results) == 0:
|
|
358
|
-
return [{"Result": "No good Search Result was found"}]
|
|
359
|
-
|
|
360
|
-
return [
|
|
361
|
-
{
|
|
362
|
-
"snippet": result.get("content", ""),
|
|
363
|
-
"title": result["title"],
|
|
364
|
-
"link": result["url"],
|
|
365
|
-
"engines": result["engines"],
|
|
366
|
-
"category": result["category"],
|
|
367
|
-
}
|
|
368
|
-
for result in results
|
|
369
|
-
]
|
|
370
|
-
|
|
2
|
+
from rich import print
|
|
3
|
+
|
|
4
|
+
def WEBSX(query):
|
|
5
|
+
url = 'https://searx.bnngpt.com/api/v1/scrape/'
|
|
6
|
+
data = {'query': query}
|
|
7
|
+
response = requests.post(url, data=data)
|
|
8
|
+
responses = response.json().get('responses')
|
|
9
|
+
return responses
|
|
10
|
+
|
|
11
|
+
if __name__ == "__main__":
|
|
12
|
+
# Example search query
|
|
13
|
+
search_query = "Python development tools"
|
|
14
|
+
|
|
15
|
+
# Call the WEBSX function with the search query
|
|
16
|
+
result = WEBSX(search_query)
|
|
17
|
+
|
|
18
|
+
# Pretty-print the JSON response
|
|
19
|
+
print(result)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webscout
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.5
|
|
4
4
|
Summary: Search for anything using Google, DuckDuckGo, brave, qwant, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
|
|
5
5
|
Author: OEvortex
|
|
6
6
|
Author-email: helpingai5@gmail.com
|
|
@@ -742,30 +742,11 @@ with WEBS() as WEBS:
|
|
|
742
742
|
## usage of WEBSX -- Another Websearch thing
|
|
743
743
|
```python
|
|
744
744
|
from webscout import WEBSX
|
|
745
|
+
s = "Python development tools"
|
|
745
746
|
|
|
746
|
-
|
|
747
|
-
# Initialize the WEBSX client
|
|
748
|
-
search = WEBSX(
|
|
749
|
-
k=10,
|
|
750
|
-
)
|
|
747
|
+
result = WEBSX(s)
|
|
751
748
|
|
|
752
|
-
|
|
753
|
-
query = "What is the capital of France?"
|
|
754
|
-
answer = search.run(query)
|
|
755
|
-
print(f"Answer: {answer}\n")
|
|
756
|
-
|
|
757
|
-
# Example using `results` method - Get detailed results with metadata
|
|
758
|
-
query = "What is the capital of France?"
|
|
759
|
-
results = search.results(query, num_results=3)
|
|
760
|
-
print("Search Results:")
|
|
761
|
-
for result in results:
|
|
762
|
-
print(f"Title: {result['title']}")
|
|
763
|
-
print(f"Snippet: {result['snippet']}")
|
|
764
|
-
print(f"Link: {result['link']}\n")
|
|
765
|
-
print(f'Engines: {result["engines"]}')
|
|
766
|
-
|
|
767
|
-
if __name__ == "__main__":
|
|
768
|
-
main()
|
|
749
|
+
print(result)
|
|
769
750
|
```
|
|
770
751
|
## ALL acts
|
|
771
752
|
<details>
|
|
@@ -1028,7 +1009,7 @@ ___
|
|
|
1028
1009
|
### 0. `Duckchat` - chat with LLM
|
|
1029
1010
|
```python
|
|
1030
1011
|
from webscout import WEBS as w
|
|
1031
|
-
R = w().chat("
|
|
1012
|
+
R = w().chat("Who are you", model='gpt-4o-mini') # GPT-3.5 Turbo, mixtral-8x7b, llama-3-70b, claude-3-haiku, gpt-4o-mini
|
|
1032
1013
|
print(R)
|
|
1033
1014
|
```
|
|
1034
1015
|
### 1. `PhindSearch` - Search using Phind.com
|
|
@@ -1505,7 +1486,12 @@ llama = LLAMA()
|
|
|
1505
1486
|
r = llama.chat("What is the meaning of life?")
|
|
1506
1487
|
print(r)
|
|
1507
1488
|
```
|
|
1508
|
-
|
|
1489
|
+
### 25. AndiSearch
|
|
1490
|
+
```python
|
|
1491
|
+
from webscout import AndiSearch
|
|
1492
|
+
a = AndiSearch()
|
|
1493
|
+
print(a.chat("HelpingAI-9B"))
|
|
1494
|
+
```
|
|
1509
1495
|
### `LLM`
|
|
1510
1496
|
```python
|
|
1511
1497
|
from webscout.LLM import LLM
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
webscout/AIauto.py,sha256=gC01wLPpnqONf9DwKqkmbC_gIWo5Lh5V8YPu4OmYnhE,19923
|
|
2
2
|
webscout/AIbase.py,sha256=GoHbN8r0gq2saYRZv6LA-Fr9Jlcjv80STKFXUq2ZeGU,4710
|
|
3
|
-
webscout/AIutel.py,sha256=
|
|
3
|
+
webscout/AIutel.py,sha256=joj1W7SFEM0jpj-qA__vnIJXrLUPtfu0YdLxTF8AHbc,34089
|
|
4
4
|
webscout/DWEBS.py,sha256=QLuT1IKu0lnwdl7W6c-ctBAO7Jj0Zk3PYm6-13BC7rU,25740
|
|
5
5
|
webscout/GoogleS.py,sha256=dW_iArNTyFT5MWBEI1HQvqf-Noj3uJeJA_Eods8D4ms,11587
|
|
6
6
|
webscout/LLM.py,sha256=LbGCZdJf8A5dwfoGS4tyy39tAh5BDdhMZP0ScKaaQfU,4184
|
|
7
7
|
webscout/YTdownloader.py,sha256=uWpUWnw9pxeEGw9KJ_3XDyQ5gd38gH1dJpr-HJo4vzU,39144
|
|
8
|
-
webscout/__init__.py,sha256=
|
|
8
|
+
webscout/__init__.py,sha256=Enum5IvS-CEcQ-zs3HWGEHAs3PUk5JzV9I3r771UHVk,2269
|
|
9
9
|
webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
|
|
10
10
|
webscout/async_providers.py,sha256=MRj0klEhBYVQXnzZGG_15d0e-TPA0nOc2nn735H-wR4,622
|
|
11
11
|
webscout/cli.py,sha256=RlBKeS9CSIsiBMqlzxevWtKjbY9htkZvA7J0bM_hHE8,14999
|
|
@@ -16,11 +16,14 @@ webscout/tempid.py,sha256=5oc3UbXhPGKxrMRTfRABT-V-dNzH_hOKWtLYM6iCWd4,5896
|
|
|
16
16
|
webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
|
|
17
17
|
webscout/utils.py,sha256=2O8_lftBKsv5OEvVaXCN-h0sipup0m3jxzhFdWQrdY8,2873
|
|
18
18
|
webscout/version.py,sha256=Pp5thQN3CvwDpubKz9MHn-UvDhuocamnBfB2VckwBGI,44
|
|
19
|
-
webscout/voice.py,sha256=
|
|
20
|
-
webscout/webai.py,sha256=
|
|
21
|
-
webscout/webscout_search.py,sha256=
|
|
19
|
+
webscout/voice.py,sha256=AHyeb3D8rYuAa-zBJsuMDgHq_Zvi98ROMKAUnEsKldo,1169
|
|
20
|
+
webscout/webai.py,sha256=zUSjlckZSTSUpLSoLqZx0qXD7mrlwdaXjxhnFE7ZmXc,89575
|
|
21
|
+
webscout/webscout_search.py,sha256=evbJPy8vG2YgBuUwyHaOkinIdVlgM-esvjVOvy6N8jY,43729
|
|
22
22
|
webscout/webscout_search_async.py,sha256=dooKGwLm0cwTml55Vy6NHPPY-nymEqX2h8laX94Zg5A,14537
|
|
23
|
-
webscout/websx_search.py,sha256=
|
|
23
|
+
webscout/websx_search.py,sha256=5hfkkmGFhyQzojUpvMzIOJ3DBZIBNS90UReaacsfu6s,521
|
|
24
|
+
webscout/Agents/Onlinesearcher.py,sha256=GzF2JcMfj07d74mxQEoaxwtxahgLHl3b_ugTbXjOwq4,7113
|
|
25
|
+
webscout/Agents/__init__.py,sha256=VbGyW5pulh3LRqbVTv54n5TwWsrTqOANRioG18xtdJ0,58
|
|
26
|
+
webscout/Agents/functioncall.py,sha256=5Nfmh8gmFOs7ZV7jJgZElZlJhi7hHrhxbITgLT7UpeI,5242
|
|
24
27
|
webscout/Extra/__init__.py,sha256=GG1qUwS-HspT4TeeAIT4qFpM8PaO1ZdQhpelctaM7Rs,99
|
|
25
28
|
webscout/Extra/autollama.py,sha256=8lyodIWAgJABzlMMHytlolPCgvUKh8ynkZD6MMEltXs,5970
|
|
26
29
|
webscout/Extra/gguf.py,sha256=3QzQIClcVoHyAeb60xxv4msJudC2Maf41StdbzAq1bk,7009
|
|
@@ -34,6 +37,7 @@ webscout/Local/rawdog.py,sha256=ojY_O8Vb1KvR34OwWdfLgllgaAK_7HMf64ElMATvCXs,3668
|
|
|
34
37
|
webscout/Local/samplers.py,sha256=qXwU4eLXER-2aCYzcJcTgA6BeFmi5GMpTDUX1C9pTN4,4372
|
|
35
38
|
webscout/Local/thread.py,sha256=Lyf_N2CaGAn2usSWSiUXLPAgpWub8vUu_tgFgtnvZVA,27408
|
|
36
39
|
webscout/Local/utils.py,sha256=CSt9IqHhVGk_nJEnKvSFbLhC5nNf01e0MtwpgMmF9pA,6197
|
|
40
|
+
webscout/Provider/Andi.py,sha256=y7Y9sC83NeMvK3MheROFoMttrFs9nGwjYaLNLPZMGCQ,10485
|
|
37
41
|
webscout/Provider/BasedGPT.py,sha256=LhC9WdRXhmzPEUaCYTNQF9CRFqhH4BeV1KtVf-B_Hc8,8416
|
|
38
42
|
webscout/Provider/Berlin4h.py,sha256=zMpmWmdFCbcE3UWB-F9xbbTWZTfx4GnjnRf6sDoaiC0,8552
|
|
39
43
|
webscout/Provider/Blackboxai.py,sha256=HUk0moEGsgGvidD1LF9tbfaKdx7bPnGU_SrYPdcfHU8,17182
|
|
@@ -61,10 +65,10 @@ webscout/Provider/VTLchat.py,sha256=_sErGr-wOi16ZAfiGOo0bPsAEMkjzzwreEsIqjIZMIU,
|
|
|
61
65
|
webscout/Provider/Xjai.py,sha256=BIlk2ouz9Kh_0Gg9hPvTqhI7XtcmWdg5vHSX_4uGrIs,9039
|
|
62
66
|
webscout/Provider/Yepchat.py,sha256=2Eit-A7w1ph1GQKNQuur_yaDzI64r0yBGxCIjDefJxQ,19875
|
|
63
67
|
webscout/Provider/Youchat.py,sha256=fhMpt94pIPE_XDbC4z9xyfgA7NbkNE2wlRFJabsjv90,8069
|
|
64
|
-
webscout/Provider/__init__.py,sha256=
|
|
65
|
-
webscout-4.
|
|
66
|
-
webscout-4.
|
|
67
|
-
webscout-4.
|
|
68
|
-
webscout-4.
|
|
69
|
-
webscout-4.
|
|
70
|
-
webscout-4.
|
|
68
|
+
webscout/Provider/__init__.py,sha256=bi6ja0K5KWO6B4UT78CYR3LvjKTZpIpGlshC4-BqW90,2011
|
|
69
|
+
webscout-4.5.dist-info/LICENSE.md,sha256=9P0imsudI7MEvZe2pOcg8rKBn6E5FGHQ-riYozZI-Bk,2942
|
|
70
|
+
webscout-4.5.dist-info/METADATA,sha256=KB7C4B6I_Ml0H2DLfvp1PSed-P-UsCsC1TwLEYKn8MU,57233
|
|
71
|
+
webscout-4.5.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
|
|
72
|
+
webscout-4.5.dist-info/entry_points.txt,sha256=Hh4YIIjvkqB9SVxZ2ri4DZUkgEu_WF_5_r_nZDIvfG8,73
|
|
73
|
+
webscout-4.5.dist-info/top_level.txt,sha256=nYIw7OKBQDr_Z33IzZUKidRD3zQEo8jOJYkMVMeN334,9
|
|
74
|
+
webscout-4.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|