webscout 1.3.3__py3-none-any.whl → 1.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- DeepWEBS/networks/webpage_fetcher.py +8 -10
- DeepWEBS/utilsdw/enver.py +31 -13
- webscout/AI.py +249 -7
- webscout/AIutel.py +20 -3
- webscout/__init__.py +2 -2
- webscout/g4f.py +1 -1
- webscout/version.py +1 -1
- webscout/webai.py +32 -7
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/METADATA +51 -3
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/RECORD +14 -14
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/LICENSE.md +0 -0
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/WHEEL +0 -0
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/entry_points.txt +0 -0
- {webscout-1.3.3.dist-info → webscout-1.3.5.dist-info}/top_level.txt +0 -0
|
@@ -78,20 +78,18 @@ class BatchWebpageFetcher:
|
|
|
78
78
|
self.urls = urls
|
|
79
79
|
self.total_count = len(self.urls)
|
|
80
80
|
|
|
81
|
-
with concurrent.futures.
|
|
81
|
+
with concurrent.futures.ProcessPoolExecutor() as executor:
|
|
82
82
|
futures = [
|
|
83
|
-
executor.submit(
|
|
83
|
+
executor.submit(WebpageFetcher().fetch, url, overwrite, output_parent)
|
|
84
84
|
for url in urls
|
|
85
85
|
]
|
|
86
86
|
concurrent.futures.wait(futures)
|
|
87
87
|
|
|
88
|
+
self.url_and_html_path_list = [
|
|
89
|
+
{"url": future.result().url, "html_path": str(future.result().html_path)}
|
|
90
|
+
for future in futures
|
|
91
|
+
]
|
|
92
|
+
|
|
88
93
|
return self.url_and_html_path_list
|
|
89
94
|
|
|
90
|
-
|
|
91
|
-
urls = [
|
|
92
|
-
"https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename",
|
|
93
|
-
"https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528",
|
|
94
|
-
"https://docs.python.org/zh-cn/3/tutorial/interpreter.html",
|
|
95
|
-
]
|
|
96
|
-
batch_webpage_fetcher = BatchWebpageFetcher()
|
|
97
|
-
batch_webpage_fetcher.fetch(urls=urls, overwrite=True, output_parent="python tutorials")
|
|
95
|
+
|
DeepWEBS/utilsdw/enver.py
CHANGED
|
@@ -1,25 +1,41 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
|
-
|
|
4
3
|
from pathlib import Path
|
|
5
|
-
from
|
|
4
|
+
from typing import Dict, Optional
|
|
5
|
+
|
|
6
|
+
from DeepWEBS.utilsdw.logger import OSLogger
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
class OSEnver:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
"""Manages the OS environment variables."""
|
|
11
|
+
|
|
12
|
+
def __init__(self) -> None:
|
|
13
|
+
"""Initializes the OSEnver object."""
|
|
14
|
+
self.envs_stack: list[Dict[str, str]] = []
|
|
15
|
+
self.envs: Dict[str, str] = os.environ.copy()
|
|
12
16
|
|
|
13
|
-
def store_envs(self):
|
|
14
|
-
|
|
17
|
+
def store_envs(self) -> None:
|
|
18
|
+
"""Stores a copy of the current environment variables on a stack."""
|
|
19
|
+
self.envs_stack.append(self.envs.copy())
|
|
15
20
|
|
|
16
|
-
def restore_envs(self):
|
|
21
|
+
def restore_envs(self) -> None:
|
|
22
|
+
"""Restores environment variables from the top of the stack."""
|
|
17
23
|
self.envs = self.envs_stack.pop()
|
|
18
24
|
|
|
19
|
-
def set_envs(
|
|
20
|
-
|
|
21
|
-
|
|
25
|
+
def set_envs(
|
|
26
|
+
self,
|
|
27
|
+
secrets: bool = True,
|
|
28
|
+
proxies: Optional[str] = None,
|
|
29
|
+
store_envs: bool = True,
|
|
30
|
+
) -> None:
|
|
31
|
+
"""Sets environment variables based on the contents of secrets.json.
|
|
22
32
|
|
|
33
|
+
Args:
|
|
34
|
+
secrets (bool): Whether to load secrets from secrets.json.
|
|
35
|
+
proxies (Optional[str]): Proxy URL to set as environment variable.
|
|
36
|
+
store_envs (bool): Whether to store a copy of the environment variables
|
|
37
|
+
on the stack.
|
|
38
|
+
"""
|
|
23
39
|
if store_envs:
|
|
24
40
|
self.store_envs()
|
|
25
41
|
|
|
@@ -54,7 +70,9 @@ class OSEnver:
|
|
|
54
70
|
}
|
|
55
71
|
|
|
56
72
|
if self.proxy:
|
|
57
|
-
|
|
73
|
+
OSLogger().note(f"Using proxy: [{self.proxy}]")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
enver: OSEnver = OSEnver()
|
|
58
77
|
|
|
59
78
|
|
|
60
|
-
enver = OSEnver()
|
webscout/AI.py
CHANGED
|
@@ -25,6 +25,248 @@ from webscout.AIbase import Provider
|
|
|
25
25
|
from Helpingai_T2 import Perplexity
|
|
26
26
|
from typing import Any
|
|
27
27
|
import logging
|
|
28
|
+
class GROQ(Provider):
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
api_key: str,
|
|
32
|
+
is_conversation: bool = True,
|
|
33
|
+
max_tokens: int = 600,
|
|
34
|
+
temperature: float = 1,
|
|
35
|
+
presence_penalty: int = 0,
|
|
36
|
+
frequency_penalty: int = 0,
|
|
37
|
+
top_p: float = 1,
|
|
38
|
+
model: str = "mixtral-8x7b-32768",
|
|
39
|
+
timeout: int = 30,
|
|
40
|
+
intro: str = None,
|
|
41
|
+
filepath: str = None,
|
|
42
|
+
update_file: bool = True,
|
|
43
|
+
proxies: dict = {},
|
|
44
|
+
history_offset: int = 10250,
|
|
45
|
+
act: str = None,
|
|
46
|
+
):
|
|
47
|
+
"""Instantiates GROQ
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
api_key (key): GROQ's API key.
|
|
51
|
+
is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
|
|
52
|
+
max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
|
|
53
|
+
temperature (float, optional): Charge of the generated text's randomness. Defaults to 1.
|
|
54
|
+
presence_penalty (int, optional): Chances of topic being repeated. Defaults to 0.
|
|
55
|
+
frequency_penalty (int, optional): Chances of word being repeated. Defaults to 0.
|
|
56
|
+
top_p (float, optional): Sampling threshold during inference time. Defaults to 0.999.
|
|
57
|
+
model (str, optional): LLM model name. Defaults to "gpt-3.5-turbo".
|
|
58
|
+
timeout (int, optional): Http request timeout. Defaults to 30.
|
|
59
|
+
intro (str, optional): Conversation introductory prompt. Defaults to None.
|
|
60
|
+
filepath (str, optional): Path to file containing conversation history. Defaults to None.
|
|
61
|
+
update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
|
|
62
|
+
proxies (dict, optional): Http request proxies. Defaults to {}.
|
|
63
|
+
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
64
|
+
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
65
|
+
"""
|
|
66
|
+
self.session = requests.Session()
|
|
67
|
+
self.is_conversation = is_conversation
|
|
68
|
+
self.max_tokens_to_sample = max_tokens
|
|
69
|
+
self.api_key = api_key
|
|
70
|
+
self.model = model
|
|
71
|
+
self.temperature = temperature
|
|
72
|
+
self.presence_penalty = presence_penalty
|
|
73
|
+
self.frequency_penalty = frequency_penalty
|
|
74
|
+
self.top_p = top_p
|
|
75
|
+
self.chat_endpoint = "https://api.groq.com/openai/v1/chat/completions"
|
|
76
|
+
self.stream_chunk_size = 64
|
|
77
|
+
self.timeout = timeout
|
|
78
|
+
self.last_response = {}
|
|
79
|
+
self.headers = {
|
|
80
|
+
"Content-Type": "application/json",
|
|
81
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
self.__available_optimizers = (
|
|
85
|
+
method
|
|
86
|
+
for method in dir(Optimizers)
|
|
87
|
+
if callable(getattr(Optimizers, method)) and not method.startswith("__")
|
|
88
|
+
)
|
|
89
|
+
self.session.headers.update(self.headers)
|
|
90
|
+
Conversation.intro = (
|
|
91
|
+
AwesomePrompts().get_act(
|
|
92
|
+
act, raise_not_found=True, default=None, case_insensitive=True
|
|
93
|
+
)
|
|
94
|
+
if act
|
|
95
|
+
else intro or Conversation.intro
|
|
96
|
+
)
|
|
97
|
+
self.conversation = Conversation(
|
|
98
|
+
is_conversation, self.max_tokens_to_sample, filepath, update_file
|
|
99
|
+
)
|
|
100
|
+
self.conversation.history_offset = history_offset
|
|
101
|
+
self.session.proxies = proxies
|
|
102
|
+
|
|
103
|
+
def ask(
|
|
104
|
+
self,
|
|
105
|
+
prompt: str,
|
|
106
|
+
stream: bool = False,
|
|
107
|
+
raw: bool = False,
|
|
108
|
+
optimizer: str = None,
|
|
109
|
+
conversationally: bool = False,
|
|
110
|
+
) -> dict:
|
|
111
|
+
"""Chat with AI
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
prompt (str): Prompt to be send.
|
|
115
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
116
|
+
raw (bool, optional): Stream back raw response as received. Defaults to False.
|
|
117
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
118
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
119
|
+
Returns:
|
|
120
|
+
dict : {}
|
|
121
|
+
```json
|
|
122
|
+
{
|
|
123
|
+
"id": "c0c8d139-d2b9-9909-8aa1-14948bc28404",
|
|
124
|
+
"object": "chat.completion",
|
|
125
|
+
"created": 1710852779,
|
|
126
|
+
"model": "mixtral-8x7b-32768",
|
|
127
|
+
"choices": [
|
|
128
|
+
{
|
|
129
|
+
"index": 0,
|
|
130
|
+
"message": {
|
|
131
|
+
"role": "assistant",
|
|
132
|
+
"content": "Hello! How can I assist you today? I'm here to help answer your questions and engage in conversation on a wide variety of topics. Feel free to ask me anything!"
|
|
133
|
+
},
|
|
134
|
+
"logprobs": null,
|
|
135
|
+
"finish_reason": "stop"
|
|
136
|
+
}
|
|
137
|
+
],
|
|
138
|
+
"usage": {
|
|
139
|
+
"prompt_tokens": 47,
|
|
140
|
+
"prompt_time": 0.03,
|
|
141
|
+
"completion_tokens": 37,
|
|
142
|
+
"completion_time": 0.069,
|
|
143
|
+
"total_tokens": 84,
|
|
144
|
+
"total_time": 0.099
|
|
145
|
+
},
|
|
146
|
+
"system_fingerprint": null
|
|
147
|
+
}
|
|
148
|
+
```
|
|
149
|
+
"""
|
|
150
|
+
conversation_prompt = self.conversation.gen_complete_prompt(prompt)
|
|
151
|
+
if optimizer:
|
|
152
|
+
if optimizer in self.__available_optimizers:
|
|
153
|
+
conversation_prompt = getattr(Optimizers, optimizer)(
|
|
154
|
+
conversation_prompt if conversationally else prompt
|
|
155
|
+
)
|
|
156
|
+
else:
|
|
157
|
+
raise Exception(
|
|
158
|
+
f"Optimizer is not one of {self.__available_optimizers}"
|
|
159
|
+
)
|
|
160
|
+
self.session.headers.update(self.headers)
|
|
161
|
+
payload = {
|
|
162
|
+
"frequency_penalty": self.frequency_penalty,
|
|
163
|
+
"messages": [{"content": conversation_prompt, "role": "user"}],
|
|
164
|
+
"model": self.model,
|
|
165
|
+
"presence_penalty": self.presence_penalty,
|
|
166
|
+
"stream": stream,
|
|
167
|
+
"temperature": self.temperature,
|
|
168
|
+
"top_p": self.top_p,
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
def for_stream():
|
|
172
|
+
response = self.session.post(
|
|
173
|
+
self.chat_endpoint, json=payload, stream=True, timeout=self.timeout
|
|
174
|
+
)
|
|
175
|
+
if not response.ok:
|
|
176
|
+
raise Exception(
|
|
177
|
+
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
message_load = ""
|
|
181
|
+
for value in response.iter_lines(
|
|
182
|
+
decode_unicode=True,
|
|
183
|
+
delimiter="" if raw else "data:",
|
|
184
|
+
chunk_size=self.stream_chunk_size,
|
|
185
|
+
):
|
|
186
|
+
try:
|
|
187
|
+
resp = json.loads(value)
|
|
188
|
+
incomplete_message = self.get_message(resp)
|
|
189
|
+
if incomplete_message:
|
|
190
|
+
message_load += incomplete_message
|
|
191
|
+
resp["choices"][0]["delta"]["content"] = message_load
|
|
192
|
+
self.last_response.update(resp)
|
|
193
|
+
yield value if raw else resp
|
|
194
|
+
elif raw:
|
|
195
|
+
yield value
|
|
196
|
+
except json.decoder.JSONDecodeError:
|
|
197
|
+
pass
|
|
198
|
+
self.conversation.update_chat_history(
|
|
199
|
+
prompt, self.get_message(self.last_response)
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def for_non_stream():
|
|
203
|
+
response = self.session.post(
|
|
204
|
+
self.chat_endpoint, json=payload, stream=False, timeout=self.timeout
|
|
205
|
+
)
|
|
206
|
+
if not response.ok:
|
|
207
|
+
raise Exception(
|
|
208
|
+
f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
|
|
209
|
+
)
|
|
210
|
+
resp = response.json()
|
|
211
|
+
self.last_response.update(resp)
|
|
212
|
+
self.conversation.update_chat_history(
|
|
213
|
+
prompt, self.get_message(self.last_response)
|
|
214
|
+
)
|
|
215
|
+
return resp
|
|
216
|
+
|
|
217
|
+
return for_stream() if stream else for_non_stream()
|
|
218
|
+
|
|
219
|
+
def chat(
|
|
220
|
+
self,
|
|
221
|
+
prompt: str,
|
|
222
|
+
stream: bool = False,
|
|
223
|
+
optimizer: str = None,
|
|
224
|
+
conversationally: bool = False,
|
|
225
|
+
) -> str:
|
|
226
|
+
"""Generate response `str`
|
|
227
|
+
Args:
|
|
228
|
+
prompt (str): Prompt to be send.
|
|
229
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
230
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
231
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
232
|
+
Returns:
|
|
233
|
+
str: Response generated
|
|
234
|
+
"""
|
|
235
|
+
|
|
236
|
+
def for_stream():
|
|
237
|
+
for response in self.ask(
|
|
238
|
+
prompt, True, optimizer=optimizer, conversationally=conversationally
|
|
239
|
+
):
|
|
240
|
+
yield self.get_message(response)
|
|
241
|
+
|
|
242
|
+
def for_non_stream():
|
|
243
|
+
return self.get_message(
|
|
244
|
+
self.ask(
|
|
245
|
+
prompt,
|
|
246
|
+
False,
|
|
247
|
+
optimizer=optimizer,
|
|
248
|
+
conversationally=conversationally,
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
return for_stream() if stream else for_non_stream()
|
|
253
|
+
|
|
254
|
+
def get_message(self, response: dict) -> str:
|
|
255
|
+
"""Retrieves message only from response
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
response (dict): Response generated by `self.ask`
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
str: Message extracted
|
|
262
|
+
"""
|
|
263
|
+
assert isinstance(response, dict), "Response should be of dict data-type only"
|
|
264
|
+
try:
|
|
265
|
+
if response["choices"][0].get("delta"):
|
|
266
|
+
return response["choices"][0]["delta"]["content"]
|
|
267
|
+
return response["choices"][0]["message"]["content"]
|
|
268
|
+
except KeyError:
|
|
269
|
+
return ""
|
|
28
270
|
#----------------------------------------------------------Sean-----------------------------------------------------------
|
|
29
271
|
class Sean:
|
|
30
272
|
def __init__(
|
|
@@ -294,6 +536,7 @@ class OPENAI(Provider):
|
|
|
294
536
|
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
295
537
|
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
296
538
|
"""
|
|
539
|
+
self.session = requests.Session()
|
|
297
540
|
self.is_conversation = is_conversation
|
|
298
541
|
self.max_tokens_to_sample = max_tokens
|
|
299
542
|
self.api_key = api_key
|
|
@@ -498,9 +741,9 @@ class OPENAI(Provider):
|
|
|
498
741
|
#--------------------------------------LEO-----------------------------------------
|
|
499
742
|
class LEO(Provider):
|
|
500
743
|
|
|
501
|
-
model = "llama-2-13b-chat"
|
|
744
|
+
# model = "llama-2-13b-chat"
|
|
502
745
|
|
|
503
|
-
key = "qztbjzBqJueQZLFkwTTJrieu8Vw3789u"
|
|
746
|
+
# key = "qztbjzBqJueQZLFkwTTJrieu8Vw3789u"
|
|
504
747
|
def __init__(
|
|
505
748
|
self,
|
|
506
749
|
is_conversation: bool = True,
|
|
@@ -508,8 +751,8 @@ class LEO(Provider):
|
|
|
508
751
|
temperature: float = 0.2,
|
|
509
752
|
top_k: int = -1,
|
|
510
753
|
top_p: float = 0.999,
|
|
511
|
-
model: str =
|
|
512
|
-
brave_key: str =
|
|
754
|
+
model: str = "llama-2-13b-chat",
|
|
755
|
+
brave_key: str = "qztbjzBqJueQZLFkwTTJrieu8Vw3789u",
|
|
513
756
|
timeout: int = 30,
|
|
514
757
|
intro: str = None,
|
|
515
758
|
filepath: str = None,
|
|
@@ -536,6 +779,7 @@ class LEO(Provider):
|
|
|
536
779
|
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
537
780
|
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
538
781
|
"""
|
|
782
|
+
self.session = requests.Session()
|
|
539
783
|
self.is_conversation = is_conversation
|
|
540
784
|
self.max_tokens_to_sample = max_tokens
|
|
541
785
|
self.model = model
|
|
@@ -559,7 +803,7 @@ class LEO(Provider):
|
|
|
559
803
|
for method in dir(Optimizers)
|
|
560
804
|
if callable(getattr(Optimizers, method)) and not method.startswith("__")
|
|
561
805
|
)
|
|
562
|
-
self.
|
|
806
|
+
self.session.headers.update(self.headers)
|
|
563
807
|
Conversation.intro = (
|
|
564
808
|
AwesomePrompts().get_act(
|
|
565
809
|
act, raise_not_found=True, default=None, case_insensitive=True
|
|
@@ -1932,9 +2176,7 @@ from os import path
|
|
|
1932
2176
|
from json import load
|
|
1933
2177
|
from json import dumps
|
|
1934
2178
|
import warnings
|
|
1935
|
-
|
|
1936
2179
|
logging.getLogger("httpx").setLevel(logging.ERROR)
|
|
1937
|
-
|
|
1938
2180
|
warnings.simplefilter("ignore", category=UserWarning)
|
|
1939
2181
|
class GEMINI(Provider):
|
|
1940
2182
|
def __init__(
|
webscout/AIutel.py
CHANGED
|
@@ -10,14 +10,32 @@ import sys
|
|
|
10
10
|
import click
|
|
11
11
|
from rich.markdown import Markdown
|
|
12
12
|
from rich.console import Console
|
|
13
|
-
|
|
13
|
+
import g4f
|
|
14
14
|
appdir = appdirs.AppDirs("AIWEBS", "vortex")
|
|
15
15
|
|
|
16
16
|
default_path = appdir.user_cache_dir
|
|
17
17
|
|
|
18
18
|
if not os.path.exists(default_path):
|
|
19
19
|
os.makedirs(default_path)
|
|
20
|
-
|
|
20
|
+
webai = [
|
|
21
|
+
"leo",
|
|
22
|
+
"openai",
|
|
23
|
+
"opengpt",
|
|
24
|
+
"koboldai",
|
|
25
|
+
"gemini",
|
|
26
|
+
"phind",
|
|
27
|
+
"blackboxai",
|
|
28
|
+
"g4fauto",
|
|
29
|
+
"perplexity",
|
|
30
|
+
"sean",
|
|
31
|
+
"groq",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
gpt4free_providers = [
|
|
35
|
+
provider.__name__ for provider in g4f.Provider.__providers__ # if provider.working
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
available_providers = webai + gpt4free_providers
|
|
21
39
|
|
|
22
40
|
def run_system_command(
|
|
23
41
|
command: str,
|
|
@@ -468,7 +486,6 @@ print("The essay is about...")
|
|
|
468
486
|
```
|
|
469
487
|
"""
|
|
470
488
|
|
|
471
|
-
# Idea borrowed from https://github.com/AbanteAI/rawdog
|
|
472
489
|
|
|
473
490
|
def __init__(
|
|
474
491
|
self,
|
webscout/__init__.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Webscout.
|
|
2
2
|
|
|
3
|
-
Search for
|
|
4
|
-
using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
|
|
3
|
+
Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
|
|
5
4
|
"""
|
|
6
5
|
import g4f
|
|
7
6
|
import logging
|
|
@@ -26,6 +25,7 @@ webai = [
|
|
|
26
25
|
"g4fauto",
|
|
27
26
|
"perplexity",
|
|
28
27
|
"sean",
|
|
28
|
+
"groq",
|
|
29
29
|
]
|
|
30
30
|
|
|
31
31
|
gpt4free_providers = [
|
webscout/g4f.py
CHANGED
|
@@ -2,7 +2,7 @@ import g4f
|
|
|
2
2
|
from webscout.AIutel import Optimizers
|
|
3
3
|
from webscout.AIutel import Conversation
|
|
4
4
|
from webscout.AIutel import AwesomePrompts
|
|
5
|
-
from webscout.
|
|
5
|
+
from webscout.AIbase import Provider
|
|
6
6
|
from webscout.AIutel import available_providers
|
|
7
7
|
|
|
8
8
|
|
webscout/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "1.3.
|
|
1
|
+
__version__ = "1.3.5"
|
|
2
2
|
|
webscout/webai.py
CHANGED
|
@@ -29,6 +29,7 @@ from webscout.AIutel import Optimizers
|
|
|
29
29
|
from webscout.AIutel import default_path
|
|
30
30
|
from webscout.AIutel import AwesomePrompts
|
|
31
31
|
from webscout.AIutel import RawDog
|
|
32
|
+
from webscout import available_providers
|
|
32
33
|
from colorama import Fore
|
|
33
34
|
from colorama import init as init_colorama
|
|
34
35
|
from dotenv import load_dotenv
|
|
@@ -41,7 +42,7 @@ init_colorama(autoreset=True)
|
|
|
41
42
|
load_dotenv() # loads .env variables
|
|
42
43
|
|
|
43
44
|
logging.basicConfig(
|
|
44
|
-
format="%(asctime)s - %(levelname)s : %(message)s ",
|
|
45
|
+
format="%(asctime)s - %(levelname)s : %(message)s ",
|
|
45
46
|
datefmt="%H:%M:%S",
|
|
46
47
|
level=logging.INFO,
|
|
47
48
|
)
|
|
@@ -61,7 +62,7 @@ class this:
|
|
|
61
62
|
|
|
62
63
|
rich_code_themes = ["monokai", "paraiso-dark", "igor", "vs", "fruity", "xcode"]
|
|
63
64
|
|
|
64
|
-
default_provider = "
|
|
65
|
+
default_provider = "phind"
|
|
65
66
|
|
|
66
67
|
getExc = lambda e: e.args[1] if len(e.args) > 1 else str(e)
|
|
67
68
|
|
|
@@ -415,14 +416,14 @@ class Main(cmd.Cmd):
|
|
|
415
416
|
elif provider == "leo":
|
|
416
417
|
from webscout.AI import LEO
|
|
417
418
|
|
|
418
|
-
self.bot = LEO
|
|
419
|
+
self.bot = LEO(
|
|
419
420
|
is_conversation=disable_conversation,
|
|
420
421
|
max_tokens=max_tokens,
|
|
421
422
|
temperature=temperature,
|
|
422
423
|
top_k=top_k,
|
|
423
424
|
top_p=top_p,
|
|
424
|
-
model=getOr(model,
|
|
425
|
-
brave_key=getOr(auth,
|
|
425
|
+
model=getOr(model, "llama-2-13b-chat"),
|
|
426
|
+
brave_key=getOr(auth, "qztbjzBqJueQZLFkwTTJrieu8Vw3789u"),
|
|
426
427
|
timeout=timeout,
|
|
427
428
|
intro=intro,
|
|
428
429
|
filepath=filepath,
|
|
@@ -470,6 +471,30 @@ class Main(cmd.Cmd):
|
|
|
470
471
|
history_offset=history_offset,
|
|
471
472
|
act=awesome_prompt,
|
|
472
473
|
)
|
|
474
|
+
elif provider == "groq":
|
|
475
|
+
assert auth, (
|
|
476
|
+
"GROQ's API-key is required. " "Use the flag `--key` or `-k`"
|
|
477
|
+
)
|
|
478
|
+
from webscout.AI import GROQ
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
self.bot = GROQ(
|
|
482
|
+
api_key=auth,
|
|
483
|
+
is_conversation=disable_conversation,
|
|
484
|
+
max_tokens=max_tokens,
|
|
485
|
+
temperature=temperature,
|
|
486
|
+
presence_penalty=top_p,
|
|
487
|
+
frequency_penalty=top_k,
|
|
488
|
+
top_p=top_p,
|
|
489
|
+
model=getOr(model, "mixtral-8x7b-32768"),
|
|
490
|
+
timeout=timeout,
|
|
491
|
+
intro=intro,
|
|
492
|
+
filepath=filepath,
|
|
493
|
+
update_file=update_file,
|
|
494
|
+
proxies=proxies,
|
|
495
|
+
history_offset=history_offset,
|
|
496
|
+
act=awesome_prompt,
|
|
497
|
+
)
|
|
473
498
|
elif provider == "sean":
|
|
474
499
|
from webscout.AI import Sean
|
|
475
500
|
|
|
@@ -1077,7 +1102,7 @@ class EntryGroup:
|
|
|
1077
1102
|
pass
|
|
1078
1103
|
|
|
1079
1104
|
|
|
1080
|
-
|
|
1105
|
+
import webscout
|
|
1081
1106
|
class Chatwebai:
|
|
1082
1107
|
"""webai command"""
|
|
1083
1108
|
|
|
@@ -1194,7 +1219,7 @@ class Chatwebai:
|
|
|
1194
1219
|
@click.option(
|
|
1195
1220
|
"-p",
|
|
1196
1221
|
"--provider",
|
|
1197
|
-
type=click.Choice(
|
|
1222
|
+
type=click.Choice(available_providers),
|
|
1198
1223
|
default=this.default_provider,
|
|
1199
1224
|
help="Name of LLM provider.",
|
|
1200
1225
|
metavar=(
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webscout
|
|
3
|
-
Version: 1.3.
|
|
4
|
-
Summary: Search for
|
|
3
|
+
Version: 1.3.5
|
|
4
|
+
Summary: Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
|
|
5
5
|
Author: OEvortex
|
|
6
6
|
Author-email: helpingai5@gmail.com
|
|
7
7
|
License: HelpingAI Simplified Universal License
|
|
@@ -56,7 +56,7 @@ Requires-Dist: pytest >=7.4.2 ; extra == 'dev'
|
|
|
56
56
|
<a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
|
|
57
57
|
<a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
|
|
58
58
|
|
|
59
|
-
Search for
|
|
59
|
+
Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
## Table of Contents
|
|
@@ -100,6 +100,8 @@ Search for words, documents, images, videos, news, maps and text translation usi
|
|
|
100
100
|
- [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
|
|
101
101
|
- [`LLM`](#llm)
|
|
102
102
|
- [`LLM` with internet](#llm-with-internet)
|
|
103
|
+
- [`Webai` - terminal gpt and a open interpeter](#webai---terminal-gpt-and-a-open-interpeter)
|
|
104
|
+
- [for using as terminal gpt](#for-using-as-terminal-gpt)
|
|
103
105
|
|
|
104
106
|
## Install
|
|
105
107
|
```python
|
|
@@ -801,3 +803,49 @@ if __name__ == "__main__":
|
|
|
801
803
|
else:
|
|
802
804
|
print("No response")
|
|
803
805
|
```
|
|
806
|
+
## `Webai` - terminal gpt and a open interpeter
|
|
807
|
+
|
|
808
|
+
```python
|
|
809
|
+
from webscout.webai import Main
|
|
810
|
+
|
|
811
|
+
def use_rawdog_with_webai(prompt):
|
|
812
|
+
"""
|
|
813
|
+
Wrap the webscout default method in a try-except block to catch any unhandled
|
|
814
|
+
exceptions and print a helpful message.
|
|
815
|
+
"""
|
|
816
|
+
try:
|
|
817
|
+
webai_bot = Main(
|
|
818
|
+
max_tokens=500,
|
|
819
|
+
provider="phind",
|
|
820
|
+
temperature=0.7,
|
|
821
|
+
top_k=40,
|
|
822
|
+
top_p=0.95,
|
|
823
|
+
model="Phind Model", # Replace with your desired model
|
|
824
|
+
auth=None, # Replace with your auth key/value (if needed)
|
|
825
|
+
timeout=30,
|
|
826
|
+
disable_conversation=True,
|
|
827
|
+
filepath=None,
|
|
828
|
+
update_file=True,
|
|
829
|
+
intro=None,
|
|
830
|
+
rawdog=True,
|
|
831
|
+
history_offset=10250,
|
|
832
|
+
awesome_prompt=None,
|
|
833
|
+
proxy_path=None,
|
|
834
|
+
quiet=True
|
|
835
|
+
)
|
|
836
|
+
webai_response = webai_bot.default(prompt)
|
|
837
|
+
except Exception as e:
|
|
838
|
+
print("Unexpected error:", e)
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
if __name__ == "__main__":
|
|
842
|
+
user_prompt = input("Enter your prompt: ")
|
|
843
|
+
use_rawdog_with_webai(user_prompt)
|
|
844
|
+
```
|
|
845
|
+
```shell
|
|
846
|
+
python -m webscout.webai webai --provider "phind" --rawdog
|
|
847
|
+
```
|
|
848
|
+
### for using as terminal gpt
|
|
849
|
+
```python
|
|
850
|
+
python -m webscout.webai webai --provider "sean"
|
|
851
|
+
```
|
|
@@ -6,32 +6,32 @@ DeepWEBS/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
|
6
6
|
DeepWEBS/networks/filepath_converter.py,sha256=JKMBew1TYe4TVoGTqgTWerq2Pam49_9u9TVUFCTDQyk,3183
|
|
7
7
|
DeepWEBS/networks/google_searcher.py,sha256=-AdIpVkRgemsARnOt8WPkF2Id1baVlqDHyqX2qz8Aew,1966
|
|
8
8
|
DeepWEBS/networks/network_configs.py,sha256=-Hb78_7SBx32h219FnU14qcHTvBdDUf_QAU6-RTL_e0,726
|
|
9
|
-
DeepWEBS/networks/webpage_fetcher.py,sha256=
|
|
9
|
+
DeepWEBS/networks/webpage_fetcher.py,sha256=vRB9T3o-nMgrMkG2NPHTDctNeXaPSKCmBXqu189h2ZI,3590
|
|
10
10
|
DeepWEBS/utilsdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
DeepWEBS/utilsdw/enver.py,sha256=
|
|
11
|
+
DeepWEBS/utilsdw/enver.py,sha256=vpI7s4_o_VL9govSryOv-z1zYK3pTEW3-H9QNN8JYtc,2472
|
|
12
12
|
DeepWEBS/utilsdw/logger.py,sha256=Z0nFUcEGyU8r28yKiIyvEtO26xxpmJgbvNToTfwZecc,8174
|
|
13
|
-
webscout/AI.py,sha256=
|
|
13
|
+
webscout/AI.py,sha256=iuNRGcRY-StwH0PwwUYNC_JhUKG_yAk8RYOh4-t67Gs,103168
|
|
14
14
|
webscout/AIbase.py,sha256=vQi2ougu5bG-QdmoYmxCQsOg7KTEgG7EF6nZh5qqUGw,2343
|
|
15
|
-
webscout/AIutel.py,sha256=
|
|
15
|
+
webscout/AIutel.py,sha256=1C2HA-xgpW3OalFrI8r6md-uJwzCPvnUG0dBWYOV2KI,24426
|
|
16
16
|
webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
|
|
17
17
|
webscout/HelpingAI.py,sha256=YeZw0zYVHMcBFFPNdd3_Ghpm9ebt_EScQjHO_IIs4lg,8103
|
|
18
18
|
webscout/LLM.py,sha256=CiDz0okZNEoXuxMwadZnwRGSLpqk2zg0vzvXSxQZjcE,1910
|
|
19
|
-
webscout/__init__.py,sha256=
|
|
19
|
+
webscout/__init__.py,sha256=6l3n5S_nDg1uhoG6eBW7C7Bhb7B23uBqQwNN35E7P3E,1015
|
|
20
20
|
webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
|
|
21
21
|
webscout/cli.py,sha256=F888fdrFUQgczMBN4yMOSf6Nh-IbvkqpPhDsbnA2FtQ,17059
|
|
22
22
|
webscout/exceptions.py,sha256=4AOO5wexeL96nvUS-badcckcwrPS7UpZyAgB9vknHZE,276
|
|
23
|
-
webscout/g4f.py,sha256=
|
|
23
|
+
webscout/g4f.py,sha256=NEZbXOoVfmHiKcSjVpBMNKZzHgbTJLsd8xOXjtn4js4,16358
|
|
24
24
|
webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
|
|
25
25
|
webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
|
|
26
26
|
webscout/utils.py,sha256=c_98M4oqpb54pUun3fpGGlCerFD6ZHUbghyp5b7Mwgo,2605
|
|
27
|
-
webscout/version.py,sha256=
|
|
27
|
+
webscout/version.py,sha256=84L_BIC4IlYF8PSO3raVy2A4eNETbCETloSoLV9OffA,25
|
|
28
28
|
webscout/voice.py,sha256=1Ids_2ToPBMX0cH_UyPMkY_6eSE9H4Gazrl0ujPmFag,941
|
|
29
|
-
webscout/webai.py,sha256=
|
|
29
|
+
webscout/webai.py,sha256=ogbIOgLUwGtXg2zdYgJq7H46l8mzfyncWLR-g0vXLcU,74812
|
|
30
30
|
webscout/webscout_search.py,sha256=3_lli-hDb8_kCGwscK29xuUcOS833ROgpNhDzrxh0dk,3085
|
|
31
31
|
webscout/webscout_search_async.py,sha256=Y5frH0k3hLqBCR-8dn7a_b7EvxdYxn6wHiKl3jWosE0,40670
|
|
32
|
-
webscout-1.3.
|
|
33
|
-
webscout-1.3.
|
|
34
|
-
webscout-1.3.
|
|
35
|
-
webscout-1.3.
|
|
36
|
-
webscout-1.3.
|
|
37
|
-
webscout-1.3.
|
|
32
|
+
webscout-1.3.5.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
|
|
33
|
+
webscout-1.3.5.dist-info/METADATA,sha256=piJwnW1M9ZmLGx7als7tWJyY9X5vZXnBahqapGK3Ju0,31961
|
|
34
|
+
webscout-1.3.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
35
|
+
webscout-1.3.5.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
|
|
36
|
+
webscout-1.3.5.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
|
|
37
|
+
webscout-1.3.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|