webscout 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AI.py +1 -0
- webscout/AIauto.py +452 -0
- webscout/AIutel.py +3 -1
- webscout/__init__.py +3 -2
- webscout/async_providers.py +1 -11
- webscout/exceptions.py +5 -0
- webscout/version.py +1 -1
- webscout/webai.py +24 -4
- webscout/webscout_search.py +14 -17
- webscout/webscout_search_async.py +132 -76
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/METADATA +9 -9
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/RECORD +16 -15
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/LICENSE.md +0 -0
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/WHEEL +0 -0
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/entry_points.txt +0 -0
- {webscout-1.4.4.dist-info → webscout-1.4.6.dist-info}/top_level.txt +0 -0
webscout/AI.py
CHANGED
webscout/AIauto.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
from webscout.AI import Provider, AsyncProvider
|
|
2
|
+
from webscout.AI import OPENGPT, AsyncOPENGPT
|
|
3
|
+
from webscout.AI import KOBOLDAI, AsyncKOBOLDAI
|
|
4
|
+
from webscout.AI import PhindSearch, AsyncPhindSearch
|
|
5
|
+
from webscout.AI import LLAMA2, AsyncLLAMA2
|
|
6
|
+
from webscout.AI import BLACKBOXAI, AsyncBLACKBOXAI
|
|
7
|
+
from webscout.AI import PERPLEXITY
|
|
8
|
+
from webscout.AI import ThinkAnyAI
|
|
9
|
+
from webscout.AI import YouChat
|
|
10
|
+
from webscout.AI import YEPCHAT
|
|
11
|
+
from webscout.g4f import GPT4FREE, AsyncGPT4FREE
|
|
12
|
+
from webscout.g4f import TestProviders
|
|
13
|
+
from webscout.exceptions import AllProvidersFailure
|
|
14
|
+
from webscout.async_providers import mapper as async_provider_map
|
|
15
|
+
from typing import AsyncGenerator
|
|
16
|
+
|
|
17
|
+
from typing import Union
|
|
18
|
+
from typing import Any
|
|
19
|
+
import logging
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
provider_map: dict[
|
|
23
|
+
str, Union[OPENGPT, KOBOLDAI, PhindSearch, LLAMA2, BLACKBOXAI, PERPLEXITY, GPT4FREE, ThinkAnyAI, YEPCHAT, YouChat]
|
|
24
|
+
] = {
|
|
25
|
+
"PhindSearch": PhindSearch,
|
|
26
|
+
"perplexity": PERPLEXITY,
|
|
27
|
+
"opengpt": OPENGPT,
|
|
28
|
+
"koboldai": KOBOLDAI,
|
|
29
|
+
"llama2": LLAMA2,
|
|
30
|
+
"blackboxai": BLACKBOXAI,
|
|
31
|
+
"gpt4free": GPT4FREE,
|
|
32
|
+
"thinkany": ThinkAnyAI,
|
|
33
|
+
"yepchat": YEPCHAT,
|
|
34
|
+
"you": YouChat,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class AUTO(Provider):
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
is_conversation: bool = True,
|
|
42
|
+
max_tokens: int = 600,
|
|
43
|
+
timeout: int = 30,
|
|
44
|
+
intro: str = None,
|
|
45
|
+
filepath: str = None,
|
|
46
|
+
update_file: bool = True,
|
|
47
|
+
proxies: dict = {},
|
|
48
|
+
history_offset: int = 10250,
|
|
49
|
+
act: str = None,
|
|
50
|
+
exclude: list[str] = [],
|
|
51
|
+
):
|
|
52
|
+
"""Instantiates AUTO
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True
|
|
56
|
+
max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
|
|
57
|
+
timeout (int, optional): Http request timeout. Defaults to 30.
|
|
58
|
+
intro (str, optional): Conversation introductory prompt. Defaults to None.
|
|
59
|
+
filepath (str, optional): Path to file containing conversation history. Defaults to None.
|
|
60
|
+
update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
|
|
61
|
+
proxies (dict, optional): Http request proxies. Defaults to {}.
|
|
62
|
+
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
63
|
+
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
64
|
+
exclude(list[str], optional): List of providers to be excluded. Defaults to [].
|
|
65
|
+
"""
|
|
66
|
+
self.provider: Union[OPENGPT, KOBOLDAI, PhindSearch, LLAMA2, BLACKBOXAI, PERPLEXITY, GPT4FREE, ThinkAnyAI, YEPCHAT, YouChat] = None
|
|
67
|
+
self.provider_name: str = None
|
|
68
|
+
self.is_conversation = is_conversation
|
|
69
|
+
self.max_tokens = max_tokens
|
|
70
|
+
self.timeout = timeout
|
|
71
|
+
self.intro = intro
|
|
72
|
+
self.filepath = filepath
|
|
73
|
+
self.update_file = update_file
|
|
74
|
+
self.proxies = proxies
|
|
75
|
+
self.history_offset = history_offset
|
|
76
|
+
self.act = act
|
|
77
|
+
self.exclude = exclude
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def last_response(self) -> dict[str, Any]:
|
|
81
|
+
return self.provider.last_response
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def conversation(self) -> object:
|
|
85
|
+
return self.provider.conversation
|
|
86
|
+
|
|
87
|
+
def ask(
|
|
88
|
+
self,
|
|
89
|
+
prompt: str,
|
|
90
|
+
stream: bool = False,
|
|
91
|
+
raw: bool = False,
|
|
92
|
+
optimizer: str = None,
|
|
93
|
+
conversationally: bool = False,
|
|
94
|
+
run_new_test: bool = False,
|
|
95
|
+
) -> dict:
|
|
96
|
+
"""Chat with AI
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
prompt (str): Prompt to be send.
|
|
100
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
101
|
+
raw (bool, optional): Stream back raw response as received. Defaults to False.
|
|
102
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
103
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
104
|
+
run_new_test (bool, optional): Perform new test on g4f-based providers. Defaults to False.
|
|
105
|
+
Returns:
|
|
106
|
+
dict : {}
|
|
107
|
+
"""
|
|
108
|
+
ask_kwargs: dict[str, Union[str, bool]] = {
|
|
109
|
+
"prompt": prompt,
|
|
110
|
+
"stream": stream,
|
|
111
|
+
"raw": raw,
|
|
112
|
+
"optimizer": optimizer,
|
|
113
|
+
"conversationally": conversationally,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
# tgpt-based providers
|
|
117
|
+
for provider_name, provider_obj in provider_map.items():
|
|
118
|
+
# continue
|
|
119
|
+
if provider_name in self.exclude:
|
|
120
|
+
continue
|
|
121
|
+
try:
|
|
122
|
+
self.provider_name = f"tgpt-{provider_name}"
|
|
123
|
+
self.provider = provider_obj(
|
|
124
|
+
is_conversation=self.is_conversation,
|
|
125
|
+
max_tokens=self.max_tokens,
|
|
126
|
+
timeout=self.timeout,
|
|
127
|
+
intro=self.intro,
|
|
128
|
+
filepath=self.filepath,
|
|
129
|
+
update_file=self.update_file,
|
|
130
|
+
proxies=self.proxies,
|
|
131
|
+
history_offset=self.history_offset,
|
|
132
|
+
act=self.act,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def for_stream():
|
|
136
|
+
for chunk in self.provider.ask(**ask_kwargs):
|
|
137
|
+
yield chunk
|
|
138
|
+
|
|
139
|
+
def for_non_stream():
|
|
140
|
+
return self.provider.ask(**ask_kwargs)
|
|
141
|
+
|
|
142
|
+
return for_stream() if stream else for_non_stream()
|
|
143
|
+
|
|
144
|
+
except Exception as e:
|
|
145
|
+
logging.debug(
|
|
146
|
+
f"Failed to generate response using provider {provider_name} - {e}"
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# g4f-based providers
|
|
150
|
+
|
|
151
|
+
for provider_info in TestProviders(timeout=self.timeout).get_results(
|
|
152
|
+
run=run_new_test
|
|
153
|
+
):
|
|
154
|
+
if provider_info["name"] in self.exclude:
|
|
155
|
+
continue
|
|
156
|
+
try:
|
|
157
|
+
self.provider_name = f"g4f-{provider_info['name']}"
|
|
158
|
+
self.provider = GPT4FREE(
|
|
159
|
+
provider=provider_info["name"],
|
|
160
|
+
is_conversation=self.is_conversation,
|
|
161
|
+
max_tokens=self.max_tokens,
|
|
162
|
+
intro=self.intro,
|
|
163
|
+
filepath=self.filepath,
|
|
164
|
+
update_file=self.update_file,
|
|
165
|
+
proxies=self.proxies,
|
|
166
|
+
history_offset=self.history_offset,
|
|
167
|
+
act=self.act,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def for_stream():
|
|
171
|
+
for chunk in self.provider.ask(**ask_kwargs):
|
|
172
|
+
yield chunk
|
|
173
|
+
|
|
174
|
+
def for_non_stream():
|
|
175
|
+
return self.provider.ask(**ask_kwargs)
|
|
176
|
+
|
|
177
|
+
return for_stream() if stream else for_non_stream()
|
|
178
|
+
|
|
179
|
+
except Exception as e:
|
|
180
|
+
logging.debug(
|
|
181
|
+
f"Failed to generate response using GPT4FREE-base provider {provider_name} - {e}"
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
raise AllProvidersFailure(
|
|
185
|
+
"None of the providers generated response successfully."
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
def chat(
|
|
189
|
+
self,
|
|
190
|
+
prompt: str,
|
|
191
|
+
stream: bool = False,
|
|
192
|
+
optimizer: str = None,
|
|
193
|
+
conversationally: bool = False,
|
|
194
|
+
run_new_test: bool = False,
|
|
195
|
+
) -> str:
|
|
196
|
+
"""Generate response `str`
|
|
197
|
+
Args:
|
|
198
|
+
prompt (str): Prompt to be send.
|
|
199
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
200
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
201
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
202
|
+
run_new_test (bool, optional): Perform new test on g4f-based providers. Defaults to False.
|
|
203
|
+
Returns:
|
|
204
|
+
str: Response generated
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
def for_stream():
|
|
208
|
+
for response in self.ask(
|
|
209
|
+
prompt,
|
|
210
|
+
True,
|
|
211
|
+
optimizer=optimizer,
|
|
212
|
+
conversationally=conversationally,
|
|
213
|
+
run_new_test=run_new_test,
|
|
214
|
+
):
|
|
215
|
+
yield self.get_message(response)
|
|
216
|
+
|
|
217
|
+
def for_non_stream():
|
|
218
|
+
ask_response = self.ask(
|
|
219
|
+
prompt,
|
|
220
|
+
False,
|
|
221
|
+
optimizer=optimizer,
|
|
222
|
+
conversationally=conversationally,
|
|
223
|
+
run_new_test=run_new_test,
|
|
224
|
+
)
|
|
225
|
+
return self.get_message(ask_response)
|
|
226
|
+
|
|
227
|
+
return for_stream() if stream else for_non_stream()
|
|
228
|
+
|
|
229
|
+
def get_message(self, response: dict) -> str:
|
|
230
|
+
"""Retrieves message only from response
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
response (dict): Response generated by `self.ask`
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
str: Message extracted
|
|
237
|
+
"""
|
|
238
|
+
assert self.provider is not None, "Chat with AI first"
|
|
239
|
+
return self.provider.get_message(response)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class AsyncAUTO(AsyncProvider):
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
is_conversation: bool = True,
|
|
246
|
+
max_tokens: int = 600,
|
|
247
|
+
timeout: int = 30,
|
|
248
|
+
intro: str = None,
|
|
249
|
+
filepath: str = None,
|
|
250
|
+
update_file: bool = True,
|
|
251
|
+
proxies: dict = {},
|
|
252
|
+
history_offset: int = 10250,
|
|
253
|
+
act: str = None,
|
|
254
|
+
exclude: list[str] = [],
|
|
255
|
+
):
|
|
256
|
+
"""Instantiates AsyncAUTO
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True
|
|
260
|
+
max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
|
|
261
|
+
timeout (int, optional): Http request timeout. Defaults to 30.
|
|
262
|
+
intro (str, optional): Conversation introductory prompt. Defaults to None.
|
|
263
|
+
filepath (str, optional): Path to file containing conversation history. Defaults to None.
|
|
264
|
+
update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
|
|
265
|
+
proxies (dict, optional): Http request proxies. Defaults to {}.
|
|
266
|
+
history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
|
|
267
|
+
act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
|
|
268
|
+
exclude(list[str], optional): List of providers to be excluded. Defaults to [].
|
|
269
|
+
"""
|
|
270
|
+
self.provider: Union[
|
|
271
|
+
AsyncOPENGPT,
|
|
272
|
+
AsyncKOBOLDAI,
|
|
273
|
+
AsyncPhindSearch,
|
|
274
|
+
AsyncLLAMA2,
|
|
275
|
+
AsyncBLACKBOXAI,
|
|
276
|
+
AsyncGPT4FREE,
|
|
277
|
+
] = None
|
|
278
|
+
self.provider_name: str = None
|
|
279
|
+
self.is_conversation = is_conversation
|
|
280
|
+
self.max_tokens = max_tokens
|
|
281
|
+
self.timeout = timeout
|
|
282
|
+
self.intro = intro
|
|
283
|
+
self.filepath = filepath
|
|
284
|
+
self.update_file = update_file
|
|
285
|
+
self.proxies = proxies
|
|
286
|
+
self.history_offset = history_offset
|
|
287
|
+
self.act = act
|
|
288
|
+
self.exclude = exclude
|
|
289
|
+
|
|
290
|
+
@property
|
|
291
|
+
def last_response(self) -> dict[str, Any]:
|
|
292
|
+
return self.provider.last_response
|
|
293
|
+
|
|
294
|
+
@property
|
|
295
|
+
def conversation(self) -> object:
|
|
296
|
+
return self.provider.conversation
|
|
297
|
+
|
|
298
|
+
async def ask(
|
|
299
|
+
self,
|
|
300
|
+
prompt: str,
|
|
301
|
+
stream: bool = False,
|
|
302
|
+
raw: bool = False,
|
|
303
|
+
optimizer: str = None,
|
|
304
|
+
conversationally: bool = False,
|
|
305
|
+
run_new_test: bool = False,
|
|
306
|
+
) -> dict | AsyncGenerator:
|
|
307
|
+
"""Chat with AI asynchronously.
|
|
308
|
+
|
|
309
|
+
Args:
|
|
310
|
+
prompt (str): Prompt to be send.
|
|
311
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
312
|
+
raw (bool, optional): Stream back raw response as received. Defaults to False.
|
|
313
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
314
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
315
|
+
run_new_test (bool, optional): Perform new test on g4f-based providers. Defaults to False.
|
|
316
|
+
Returns:
|
|
317
|
+
dict|AsyncGenerator : ai response.
|
|
318
|
+
"""
|
|
319
|
+
ask_kwargs: dict[str, Union[str, bool]] = {
|
|
320
|
+
"prompt": prompt,
|
|
321
|
+
"stream": stream,
|
|
322
|
+
"raw": raw,
|
|
323
|
+
"optimizer": optimizer,
|
|
324
|
+
"conversationally": conversationally,
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
# tgpt-based providers
|
|
328
|
+
for provider_name, provider_obj in async_provider_map.items():
|
|
329
|
+
if provider_name in self.exclude:
|
|
330
|
+
continue
|
|
331
|
+
try:
|
|
332
|
+
self.provider_name = f"tgpt-{provider_name}"
|
|
333
|
+
self.provider = provider_obj(
|
|
334
|
+
is_conversation=self.is_conversation,
|
|
335
|
+
max_tokens=self.max_tokens,
|
|
336
|
+
timeout=self.timeout,
|
|
337
|
+
intro=self.intro,
|
|
338
|
+
filepath=self.filepath,
|
|
339
|
+
update_file=self.update_file,
|
|
340
|
+
proxies=self.proxies,
|
|
341
|
+
history_offset=self.history_offset,
|
|
342
|
+
act=self.act,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
async def for_stream():
|
|
346
|
+
async_ask = await self.provider.ask(**ask_kwargs)
|
|
347
|
+
async for chunk in async_ask:
|
|
348
|
+
yield chunk
|
|
349
|
+
|
|
350
|
+
async def for_non_stream():
|
|
351
|
+
return await self.provider.ask(**ask_kwargs)
|
|
352
|
+
|
|
353
|
+
return for_stream() if stream else await for_non_stream()
|
|
354
|
+
|
|
355
|
+
except Exception as e:
|
|
356
|
+
logging.debug(
|
|
357
|
+
f"Failed to generate response using provider {provider_name} - {e}"
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
# g4f-based providers
|
|
361
|
+
|
|
362
|
+
for provider_info in TestProviders(timeout=self.timeout).get_results(
|
|
363
|
+
run=run_new_test
|
|
364
|
+
):
|
|
365
|
+
if provider_info["name"] in self.exclude:
|
|
366
|
+
continue
|
|
367
|
+
try:
|
|
368
|
+
self.provider_name = f"g4f-{provider_info['name']}"
|
|
369
|
+
self.provider = AsyncGPT4FREE(
|
|
370
|
+
provider=provider_info["name"],
|
|
371
|
+
is_conversation=self.is_conversation,
|
|
372
|
+
max_tokens=self.max_tokens,
|
|
373
|
+
intro=self.intro,
|
|
374
|
+
filepath=self.filepath,
|
|
375
|
+
update_file=self.update_file,
|
|
376
|
+
proxies=self.proxies,
|
|
377
|
+
history_offset=self.history_offset,
|
|
378
|
+
act=self.act,
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
async def for_stream():
|
|
382
|
+
async_ask = await self.provider.ask(**ask_kwargs)
|
|
383
|
+
async for chunk in async_ask:
|
|
384
|
+
yield chunk
|
|
385
|
+
|
|
386
|
+
async def for_non_stream():
|
|
387
|
+
return await self.provider.ask(**ask_kwargs)
|
|
388
|
+
|
|
389
|
+
return for_stream() if stream else await for_non_stream()
|
|
390
|
+
|
|
391
|
+
except Exception as e:
|
|
392
|
+
logging.debug(
|
|
393
|
+
f"Failed to generate response using GPT4FREE-base provider {provider_name} - {e}"
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
raise AllProvidersFailure(
|
|
397
|
+
"None of the providers generated response successfully."
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
async def chat(
|
|
401
|
+
self,
|
|
402
|
+
prompt: str,
|
|
403
|
+
stream: bool = False,
|
|
404
|
+
optimizer: str = None,
|
|
405
|
+
conversationally: bool = False,
|
|
406
|
+
run_new_test: bool = False,
|
|
407
|
+
) -> str | AsyncGenerator:
|
|
408
|
+
"""Generate response `str` asynchronously.
|
|
409
|
+
Args:
|
|
410
|
+
prompt (str): Prompt to be send.
|
|
411
|
+
stream (bool, optional): Flag for streaming response. Defaults to False.
|
|
412
|
+
optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
|
|
413
|
+
conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
|
|
414
|
+
run_new_test (bool, optional): Perform new test on g4f-based providers. Defaults to False.
|
|
415
|
+
Returns:
|
|
416
|
+
str|AsyncGenerator: Response generated
|
|
417
|
+
"""
|
|
418
|
+
|
|
419
|
+
async def for_stream():
|
|
420
|
+
async_ask = await self.ask(
|
|
421
|
+
prompt,
|
|
422
|
+
True,
|
|
423
|
+
optimizer=optimizer,
|
|
424
|
+
conversationally=conversationally,
|
|
425
|
+
run_new_test=run_new_test,
|
|
426
|
+
)
|
|
427
|
+
async for response in async_ask:
|
|
428
|
+
yield await self.get_message(response)
|
|
429
|
+
|
|
430
|
+
async def for_non_stream():
|
|
431
|
+
ask_response = await self.ask(
|
|
432
|
+
prompt,
|
|
433
|
+
False,
|
|
434
|
+
optimizer=optimizer,
|
|
435
|
+
conversationally=conversationally,
|
|
436
|
+
run_new_test=run_new_test,
|
|
437
|
+
)
|
|
438
|
+
return await self.get_message(ask_response)
|
|
439
|
+
|
|
440
|
+
return for_stream() if stream else await for_non_stream()
|
|
441
|
+
|
|
442
|
+
async def get_message(self, response: dict) -> str:
|
|
443
|
+
"""Retrieves message only from response
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
response (dict): Response generated by `self.ask`
|
|
447
|
+
|
|
448
|
+
Returns:
|
|
449
|
+
str: Message extracted
|
|
450
|
+
"""
|
|
451
|
+
assert self.provider is not None, "Chat with AI first"
|
|
452
|
+
return await self.provider.get_message(response)
|
webscout/AIutel.py
CHANGED
webscout/__init__.py
CHANGED
|
@@ -11,7 +11,6 @@ from .DWEBS import DeepWEBS
|
|
|
11
11
|
from .transcriber import transcriber
|
|
12
12
|
from .voice import play_audio
|
|
13
13
|
|
|
14
|
-
|
|
15
14
|
__repo__ = "https://github.com/OE-LUCIFER/Webscout"
|
|
16
15
|
|
|
17
16
|
webai = [
|
|
@@ -30,7 +29,9 @@ webai = [
|
|
|
30
29
|
"yepchat",
|
|
31
30
|
"you",
|
|
32
31
|
"xjai",
|
|
33
|
-
"thinkany"
|
|
32
|
+
"thinkany",
|
|
33
|
+
"auto",
|
|
34
|
+
|
|
34
35
|
]
|
|
35
36
|
|
|
36
37
|
gpt4free_providers = [
|
webscout/async_providers.py
CHANGED
|
@@ -7,7 +7,7 @@ from webscout.AI import AsyncLEO
|
|
|
7
7
|
from webscout.AI import AsyncKOBOLDAI
|
|
8
8
|
from webscout.AI import AsyncGROQ
|
|
9
9
|
from webscout.AI import AsyncBLACKBOXAI
|
|
10
|
-
from webscout.
|
|
10
|
+
from webscout.g4f import AsyncGPT4FREE
|
|
11
11
|
|
|
12
12
|
mapper: dict[str, object] = {
|
|
13
13
|
"phind": AsyncPhindSearch,
|
|
@@ -21,13 +21,3 @@ mapper: dict[str, object] = {
|
|
|
21
21
|
"groq": AsyncGROQ,
|
|
22
22
|
"openai": AsyncOPENAI,
|
|
23
23
|
}
|
|
24
|
-
|
|
25
|
-
tgpt_mapper: dict[str, object] = {
|
|
26
|
-
"phind": AsyncPhindSearch,
|
|
27
|
-
"opengpt": AsyncOPENGPT,
|
|
28
|
-
"koboldai": AsyncKOBOLDAI,
|
|
29
|
-
# "gpt4free": AsyncGPT4FREE,
|
|
30
|
-
"blackboxai": AsyncBLACKBOXAI,
|
|
31
|
-
"llama2": AsyncLLAMA2,
|
|
32
|
-
"yepchat": AsyncYEPCHAT,
|
|
33
|
-
}
|
webscout/exceptions.py
CHANGED
|
@@ -10,4 +10,9 @@ class TimeoutE(Exception):
|
|
|
10
10
|
"""Raised for timeout errors during API requests."""
|
|
11
11
|
|
|
12
12
|
class FailedToGenerateResponseError(Exception):
|
|
13
|
+
|
|
13
14
|
"""Provider failed to fetch response"""
|
|
15
|
+
class AllProvidersFailure(Exception):
|
|
16
|
+
"""None of the providers generated response successfully"""
|
|
17
|
+
|
|
18
|
+
pass
|
webscout/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "1.4.
|
|
1
|
+
__version__ = "1.4.5"
|
|
2
2
|
|
webscout/webai.py
CHANGED
|
@@ -456,7 +456,20 @@ class Main(cmd.Cmd):
|
|
|
456
456
|
history_offset=history_offset,
|
|
457
457
|
act=awesome_prompt,
|
|
458
458
|
)
|
|
459
|
+
if provider == "auto":
|
|
460
|
+
from webscout.AIauto import AUTO
|
|
459
461
|
|
|
462
|
+
self.bot = AUTO(
|
|
463
|
+
is_conversation=disable_conversation,
|
|
464
|
+
max_tokens=max_tokens,
|
|
465
|
+
timeout=timeout,
|
|
466
|
+
intro=intro,
|
|
467
|
+
filepath=filepath,
|
|
468
|
+
update_file=update_file,
|
|
469
|
+
proxies=proxies,
|
|
470
|
+
history_offset=history_offset,
|
|
471
|
+
act=awesome_prompt,
|
|
472
|
+
)
|
|
460
473
|
elif provider == "opengpt":
|
|
461
474
|
from webscout.AI import OPENGPT
|
|
462
475
|
|
|
@@ -470,6 +483,7 @@ class Main(cmd.Cmd):
|
|
|
470
483
|
proxies=proxies,
|
|
471
484
|
history_offset=history_offset,
|
|
472
485
|
act=awesome_prompt,
|
|
486
|
+
assistant_id="bca37014-6f97-4f2b-8928-81ea8d478d88"
|
|
473
487
|
)
|
|
474
488
|
elif provider == "thinkany":
|
|
475
489
|
from webscout.AI import ThinkAnyAI
|
|
@@ -725,7 +739,13 @@ class Main(cmd.Cmd):
|
|
|
725
739
|
self.__init_time = time.time()
|
|
726
740
|
self.__start_time = time.time()
|
|
727
741
|
self.__end_time = time.time()
|
|
728
|
-
|
|
742
|
+
|
|
743
|
+
@property
|
|
744
|
+
def get_provider(self):
|
|
745
|
+
if self.provider == "auto" and self.bot.provider_name is not None:
|
|
746
|
+
return self.bot.provider_name
|
|
747
|
+
else:
|
|
748
|
+
return self.provider
|
|
729
749
|
@property
|
|
730
750
|
def prompt(self):
|
|
731
751
|
current_time = datetime.datetime.now().strftime("%H:%M:%S")
|
|
@@ -740,7 +760,7 @@ class Main(cmd.Cmd):
|
|
|
740
760
|
if not self.disable_coloring:
|
|
741
761
|
cmd_prompt = (
|
|
742
762
|
f"╭─[`{Fore.GREEN}{getpass.getuser().capitalize()}@webai]`"
|
|
743
|
-
f"(`{Fore.YELLOW}{self.
|
|
763
|
+
f"(`{Fore.YELLOW}{self.get_provider})`"
|
|
744
764
|
f"~[`{Fore.LIGHTWHITE_EX}⏰{Fore.MAGENTA}{current_time}-`"
|
|
745
765
|
f"{Fore.LIGHTWHITE_EX}💻{Fore.BLUE}{find_range(self.__init_time, time.time(), True)}-`"
|
|
746
766
|
f"{Fore.LIGHTWHITE_EX}⚡️{Fore.RED}{find_range(self.__start_time, self.__end_time)}s]`"
|
|
@@ -753,7 +773,7 @@ class Main(cmd.Cmd):
|
|
|
753
773
|
|
|
754
774
|
else:
|
|
755
775
|
return (
|
|
756
|
-
f"╭─[{getpass.getuser().capitalize()}@webscout]({self.
|
|
776
|
+
f"╭─[{getpass.getuser().capitalize()}@webscout]({self.get_provider})"
|
|
757
777
|
f"~[⏰{current_time}"
|
|
758
778
|
f"-💻{find_range(self.__init_time, time.time(), True)}"
|
|
759
779
|
f"-⚡️{find_range(self.__start_time, self.__end_time)}s]"
|
|
@@ -1125,7 +1145,7 @@ class Main(cmd.Cmd):
|
|
|
1125
1145
|
busy_bar.stop_spinning()
|
|
1126
1146
|
this.stream_output(
|
|
1127
1147
|
generated_response,
|
|
1128
|
-
title="
|
|
1148
|
+
title="Webscout",
|
|
1129
1149
|
is_markdown=self.prettify,
|
|
1130
1150
|
style=Style(
|
|
1131
1151
|
color=self.color,
|
webscout/webscout_search.py
CHANGED
|
@@ -4,12 +4,6 @@ from threading import Thread
|
|
|
4
4
|
import sys
|
|
5
5
|
from types import TracebackType
|
|
6
6
|
from typing import Any, Awaitable, Dict, Optional, Type, Union
|
|
7
|
-
if sys.platform == 'win32':
|
|
8
|
-
try:
|
|
9
|
-
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
|
10
|
-
except AttributeError:
|
|
11
|
-
# If WindowsSelectorEventLoopPolicy is not available, do nothing
|
|
12
|
-
pass
|
|
13
7
|
from .webscout_search_async import AsyncWEBS
|
|
14
8
|
|
|
15
9
|
|
|
@@ -20,11 +14,19 @@ class WEBS(AsyncWEBS):
|
|
|
20
14
|
def __init__(
|
|
21
15
|
self,
|
|
22
16
|
headers: Optional[Dict[str, str]] = None,
|
|
23
|
-
|
|
17
|
+
proxy: Optional[str] = None,
|
|
18
|
+
proxies: Union[Dict[str, str], str, None] = None, # deprecated
|
|
24
19
|
timeout: Optional[int] = 10,
|
|
25
20
|
) -> None:
|
|
26
|
-
|
|
27
|
-
|
|
21
|
+
"""Initialize the WEBS object.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
headers (dict, optional): Dictionary of headers for the HTTP client. Defaults to None.
|
|
25
|
+
proxy (str, optional): proxy for the HTTP client, supports http/https/socks5 protocols.
|
|
26
|
+
example: "http://user:pass@example.com:3128". Defaults to None.
|
|
27
|
+
timeout (int, optional): Timeout value for the HTTP client. Defaults to 10.
|
|
28
|
+
"""
|
|
29
|
+
super().__init__(headers=headers, proxy=proxy, proxies=proxies, timeout=timeout)
|
|
28
30
|
|
|
29
31
|
def __enter__(self) -> "WEBS":
|
|
30
32
|
return self
|
|
@@ -42,13 +44,8 @@ class WEBS(AsyncWEBS):
|
|
|
42
44
|
|
|
43
45
|
def _close_session(self) -> None:
|
|
44
46
|
"""Close the curl-cffi async session."""
|
|
45
|
-
if self.
|
|
46
|
-
|
|
47
|
-
coro = self._asession.close()
|
|
48
|
-
# Check if coro is a coroutine object
|
|
49
|
-
if asyncio.iscoroutine(coro):
|
|
50
|
-
self._run_async_in_thread(coro)
|
|
51
|
-
self._exit_done = True
|
|
47
|
+
if hasattr(self, "_asession") and self._asession._closed is False:
|
|
48
|
+
self._run_async_in_thread(self._asession.close()) # type: ignore
|
|
52
49
|
|
|
53
50
|
def _run_async_in_thread(self, coro: Awaitable[Any]) -> Any:
|
|
54
51
|
"""Runs an async coroutine in a separate thread."""
|
|
@@ -78,4 +75,4 @@ class WEBS(AsyncWEBS):
|
|
|
78
75
|
return self._run_async_in_thread(super().maps(*args, **kwargs))
|
|
79
76
|
|
|
80
77
|
def translate(self, *args: Any, **kwargs: Any) -> Any:
|
|
81
|
-
return self._run_async_in_thread(super().translate(*args, **kwargs))
|
|
78
|
+
return self._run_async_in_thread(super().translate(*args, **kwargs))
|
|
@@ -5,10 +5,10 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
5
5
|
from contextlib import suppress
|
|
6
6
|
from datetime import datetime, timezone
|
|
7
7
|
from decimal import Decimal
|
|
8
|
-
from functools import partial
|
|
8
|
+
from functools import cached_property, partial
|
|
9
9
|
from itertools import cycle, islice
|
|
10
10
|
from types import TracebackType
|
|
11
|
-
from typing import Dict, List, Optional, Tuple, Union
|
|
11
|
+
from typing import Dict, List, Optional, Tuple, Type, Union, cast
|
|
12
12
|
|
|
13
13
|
from curl_cffi import requests
|
|
14
14
|
|
|
@@ -34,68 +34,71 @@ logger = logging.getLogger("webscout_search.AsyncWEBS")
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
class AsyncWEBS:
|
|
37
|
-
"""
|
|
37
|
+
"""webscout_search async class to get search results from duckduckgo.com."""
|
|
38
38
|
|
|
39
39
|
_executor: Optional[ThreadPoolExecutor] = None
|
|
40
40
|
|
|
41
41
|
def __init__(
|
|
42
42
|
self,
|
|
43
43
|
headers: Optional[Dict[str, str]] = None,
|
|
44
|
-
|
|
44
|
+
proxy: Optional[str] = None,
|
|
45
|
+
proxies: Union[Dict[str, str], str, None] = None, # deprecated
|
|
45
46
|
timeout: Optional[int] = 10,
|
|
46
47
|
) -> None:
|
|
47
48
|
"""Initialize the AsyncWEBS object.
|
|
48
49
|
|
|
49
50
|
Args:
|
|
50
51
|
headers (dict, optional): Dictionary of headers for the HTTP client. Defaults to None.
|
|
51
|
-
|
|
52
|
+
proxy (str, optional): proxy for the HTTP client, supports http/https/socks5 protocols.
|
|
53
|
+
example: "http://user:pass@example.com:3128". Defaults to None.
|
|
52
54
|
timeout (int, optional): Timeout value for the HTTP client. Defaults to 10.
|
|
53
55
|
"""
|
|
54
|
-
self.
|
|
56
|
+
self.proxy: Optional[str] = proxy
|
|
57
|
+
assert self.proxy is None or isinstance(self.proxy, str), "proxy must be a str"
|
|
58
|
+
if not proxy and proxies:
|
|
59
|
+
warnings.warn("'proxies' is deprecated, use 'proxy' instead.", stacklevel=1)
|
|
60
|
+
self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies
|
|
55
61
|
self._asession = requests.AsyncSession(
|
|
56
62
|
headers=headers,
|
|
57
|
-
|
|
63
|
+
proxy=self.proxy,
|
|
58
64
|
timeout=timeout,
|
|
59
65
|
impersonate="chrome",
|
|
60
66
|
allow_redirects=False,
|
|
61
67
|
)
|
|
62
68
|
self._asession.headers["Referer"] = "https://duckduckgo.com/"
|
|
63
|
-
self._parser: Optional[LHTMLParser] = None
|
|
64
69
|
self._exception_event = asyncio.Event()
|
|
65
|
-
self._exit_done = False
|
|
66
70
|
|
|
67
71
|
async def __aenter__(self) -> "AsyncWEBS":
|
|
68
72
|
return self
|
|
69
73
|
|
|
70
74
|
async def __aexit__(
|
|
71
75
|
self,
|
|
72
|
-
exc_type: Optional[BaseException] = None,
|
|
76
|
+
exc_type: Optional[Type[BaseException]] = None,
|
|
73
77
|
exc_val: Optional[BaseException] = None,
|
|
74
78
|
exc_tb: Optional[TracebackType] = None,
|
|
75
79
|
) -> None:
|
|
76
|
-
await self.
|
|
80
|
+
await self._asession.__aexit__(exc_type, exc_val, exc_tb) # type: ignore
|
|
77
81
|
|
|
78
82
|
def __del__(self) -> None:
|
|
79
|
-
if self.
|
|
80
|
-
|
|
83
|
+
if hasattr(self, "_asession") and self._asession._closed is False:
|
|
84
|
+
with suppress(RuntimeError, RuntimeWarning):
|
|
85
|
+
asyncio.create_task(self._asession.close()) # type: ignore
|
|
81
86
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
if self._exit_done is False:
|
|
85
|
-
await self._asession.close()
|
|
86
|
-
self._exit_done = True
|
|
87
|
-
|
|
88
|
-
def _get_parser(self) -> "LHTMLParser":
|
|
87
|
+
@cached_property
|
|
88
|
+
def parser(self) -> Optional["LHTMLParser"]:
|
|
89
89
|
"""Get HTML parser."""
|
|
90
|
-
|
|
91
|
-
self._parser = LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False)
|
|
92
|
-
return self._parser
|
|
90
|
+
return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False)
|
|
93
91
|
|
|
94
|
-
|
|
92
|
+
@classmethod
|
|
93
|
+
def _get_executor(cls, max_workers: int = 1) -> ThreadPoolExecutor:
|
|
95
94
|
"""Get ThreadPoolExecutor. Default max_workers=1, because >=2 leads to a big overhead"""
|
|
96
|
-
if
|
|
97
|
-
|
|
98
|
-
return
|
|
95
|
+
if cls._executor is None:
|
|
96
|
+
cls._executor = ThreadPoolExecutor(max_workers=max_workers)
|
|
97
|
+
return cls._executor
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def executor(cls) -> Optional[ThreadPoolExecutor]:
|
|
101
|
+
return cls._get_executor()
|
|
99
102
|
|
|
100
103
|
async def _aget_url(
|
|
101
104
|
self,
|
|
@@ -107,19 +110,18 @@ class AsyncWEBS:
|
|
|
107
110
|
if self._exception_event.is_set():
|
|
108
111
|
raise WebscoutE("Exception occurred in previous call.")
|
|
109
112
|
try:
|
|
110
|
-
resp = await self._asession.request(method, url, data=data, params=params
|
|
111
|
-
resp_content: bytes = await resp.acontent()
|
|
113
|
+
resp = await self._asession.request(method, url, data=data, params=params)
|
|
112
114
|
except Exception as ex:
|
|
113
115
|
self._exception_event.set()
|
|
114
116
|
if "time" in str(ex).lower():
|
|
115
117
|
raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
|
|
116
118
|
raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
|
|
117
|
-
logger.debug(f"_aget_url() {resp.url} {resp.status_code} {resp.elapsed:.2f} {len(
|
|
119
|
+
logger.debug(f"_aget_url() {resp.url} {resp.status_code} {resp.elapsed:.2f} {len(resp.content)}")
|
|
118
120
|
if resp.status_code == 200:
|
|
119
|
-
return
|
|
121
|
+
return cast(bytes, resp.content)
|
|
120
122
|
self._exception_event.set()
|
|
121
123
|
if resp.status_code in (202, 301, 403):
|
|
122
|
-
raise RatelimitE(f"{resp.url} {resp.status_code}")
|
|
124
|
+
raise RatelimitE(f"{resp.url} {resp.status_code} Ratelimit")
|
|
123
125
|
raise WebscoutE(f"{resp.url} return None. {params=} {data=}")
|
|
124
126
|
|
|
125
127
|
async def _aget_vqd(self, keywords: str) -> str:
|
|
@@ -136,7 +138,7 @@ class AsyncWEBS:
|
|
|
136
138
|
backend: str = "api",
|
|
137
139
|
max_results: Optional[int] = None,
|
|
138
140
|
) -> List[Dict[str, str]]:
|
|
139
|
-
"""
|
|
141
|
+
"""webscout text search generator. Query params: https://duckduckgo.com/params.
|
|
140
142
|
|
|
141
143
|
Args:
|
|
142
144
|
keywords: keywords for query.
|
|
@@ -153,7 +155,7 @@ class AsyncWEBS:
|
|
|
153
155
|
List of dictionaries with search results, or None if there was an error.
|
|
154
156
|
|
|
155
157
|
Raises:
|
|
156
|
-
WebscoutE: Base exception for
|
|
158
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
157
159
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
158
160
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
159
161
|
"""
|
|
@@ -177,7 +179,7 @@ class AsyncWEBS:
|
|
|
177
179
|
timelimit: Optional[str] = None,
|
|
178
180
|
max_results: Optional[int] = None,
|
|
179
181
|
) -> List[Dict[str, str]]:
|
|
180
|
-
"""
|
|
182
|
+
"""webscout text search generator. Query params: https://duckduckgo.com/params.
|
|
181
183
|
|
|
182
184
|
Args:
|
|
183
185
|
keywords: keywords for query.
|
|
@@ -190,7 +192,7 @@ class AsyncWEBS:
|
|
|
190
192
|
List of dictionaries with search results.
|
|
191
193
|
|
|
192
194
|
Raises:
|
|
193
|
-
WebscoutE: Base exception for
|
|
195
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
194
196
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
195
197
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
196
198
|
"""
|
|
@@ -241,11 +243,19 @@ class AsyncWEBS:
|
|
|
241
243
|
}
|
|
242
244
|
results[priority] = result
|
|
243
245
|
|
|
244
|
-
tasks = [_text_api_page(0, 0)]
|
|
246
|
+
tasks = [asyncio.create_task(_text_api_page(0, 0))]
|
|
245
247
|
if max_results:
|
|
246
248
|
max_results = min(max_results, 500)
|
|
247
|
-
tasks.extend(
|
|
248
|
-
|
|
249
|
+
tasks.extend(
|
|
250
|
+
asyncio.create_task(_text_api_page(s, i)) for i, s in enumerate(range(23, max_results, 50), start=1)
|
|
251
|
+
)
|
|
252
|
+
try:
|
|
253
|
+
await asyncio.gather(*tasks)
|
|
254
|
+
except Exception as e:
|
|
255
|
+
for task in tasks:
|
|
256
|
+
task.cancel()
|
|
257
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
258
|
+
raise e
|
|
249
259
|
|
|
250
260
|
return list(islice(filter(None, results), max_results))
|
|
251
261
|
|
|
@@ -257,7 +267,7 @@ class AsyncWEBS:
|
|
|
257
267
|
timelimit: Optional[str] = None,
|
|
258
268
|
max_results: Optional[int] = None,
|
|
259
269
|
) -> List[Dict[str, str]]:
|
|
260
|
-
"""
|
|
270
|
+
"""webscout text search generator. Query params: https://duckduckgo.com/params.
|
|
261
271
|
|
|
262
272
|
Args:
|
|
263
273
|
keywords: keywords for query.
|
|
@@ -270,7 +280,7 @@ class AsyncWEBS:
|
|
|
270
280
|
List of dictionaries with search results.
|
|
271
281
|
|
|
272
282
|
Raises:
|
|
273
|
-
WebscoutE: Base exception for
|
|
283
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
274
284
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
275
285
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
276
286
|
"""
|
|
@@ -302,7 +312,7 @@ class AsyncWEBS:
|
|
|
302
312
|
return
|
|
303
313
|
|
|
304
314
|
tree = await self._asession.loop.run_in_executor(
|
|
305
|
-
self.
|
|
315
|
+
self.executor, partial(document_fromstring, resp_content, self.parser)
|
|
306
316
|
)
|
|
307
317
|
|
|
308
318
|
for e in tree.xpath("//div[h2]"):
|
|
@@ -327,11 +337,19 @@ class AsyncWEBS:
|
|
|
327
337
|
}
|
|
328
338
|
results[priority] = result
|
|
329
339
|
|
|
330
|
-
tasks = [_text_html_page(0, 0)]
|
|
340
|
+
tasks = [asyncio.create_task(_text_html_page(0, 0))]
|
|
331
341
|
if max_results:
|
|
332
342
|
max_results = min(max_results, 500)
|
|
333
|
-
tasks.extend(
|
|
334
|
-
|
|
343
|
+
tasks.extend(
|
|
344
|
+
asyncio.create_task(_text_html_page(s, i)) for i, s in enumerate(range(23, max_results, 50), start=1)
|
|
345
|
+
)
|
|
346
|
+
try:
|
|
347
|
+
await asyncio.gather(*tasks)
|
|
348
|
+
except Exception as e:
|
|
349
|
+
for task in tasks:
|
|
350
|
+
task.cancel()
|
|
351
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
352
|
+
raise e
|
|
335
353
|
|
|
336
354
|
return list(islice(filter(None, results), max_results))
|
|
337
355
|
|
|
@@ -342,7 +360,7 @@ class AsyncWEBS:
|
|
|
342
360
|
timelimit: Optional[str] = None,
|
|
343
361
|
max_results: Optional[int] = None,
|
|
344
362
|
) -> List[Dict[str, str]]:
|
|
345
|
-
"""
|
|
363
|
+
"""webscout text search generator. Query params: https://duckduckgo.com/params.
|
|
346
364
|
|
|
347
365
|
Args:
|
|
348
366
|
keywords: keywords for query.
|
|
@@ -354,7 +372,7 @@ class AsyncWEBS:
|
|
|
354
372
|
List of dictionaries with search results.
|
|
355
373
|
|
|
356
374
|
Raises:
|
|
357
|
-
WebscoutE: Base exception for
|
|
375
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
358
376
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
359
377
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
360
378
|
"""
|
|
@@ -381,7 +399,7 @@ class AsyncWEBS:
|
|
|
381
399
|
return
|
|
382
400
|
|
|
383
401
|
tree = await self._asession.loop.run_in_executor(
|
|
384
|
-
self.
|
|
402
|
+
self.executor, partial(document_fromstring, resp_content, self.parser)
|
|
385
403
|
)
|
|
386
404
|
|
|
387
405
|
data = zip(cycle(range(1, 5)), tree.xpath("//table[last()]//tr"))
|
|
@@ -410,11 +428,19 @@ class AsyncWEBS:
|
|
|
410
428
|
}
|
|
411
429
|
results[priority] = result
|
|
412
430
|
|
|
413
|
-
tasks = [_text_lite_page(0, 0)]
|
|
431
|
+
tasks = [asyncio.create_task(_text_lite_page(0, 0))]
|
|
414
432
|
if max_results:
|
|
415
433
|
max_results = min(max_results, 500)
|
|
416
|
-
tasks.extend(
|
|
417
|
-
|
|
434
|
+
tasks.extend(
|
|
435
|
+
asyncio.create_task(_text_lite_page(s, i)) for i, s in enumerate(range(23, max_results, 50), start=1)
|
|
436
|
+
)
|
|
437
|
+
try:
|
|
438
|
+
await asyncio.gather(*tasks)
|
|
439
|
+
except Exception as e:
|
|
440
|
+
for task in tasks:
|
|
441
|
+
task.cancel()
|
|
442
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
443
|
+
raise e
|
|
418
444
|
|
|
419
445
|
return list(islice(filter(None, results), max_results))
|
|
420
446
|
|
|
@@ -431,7 +457,7 @@ class AsyncWEBS:
|
|
|
431
457
|
license_image: Optional[str] = None,
|
|
432
458
|
max_results: Optional[int] = None,
|
|
433
459
|
) -> List[Dict[str, str]]:
|
|
434
|
-
"""
|
|
460
|
+
"""webscout images search. Query params: https://duckduckgo.com/params.
|
|
435
461
|
|
|
436
462
|
Args:
|
|
437
463
|
keywords: keywords for query.
|
|
@@ -454,7 +480,7 @@ class AsyncWEBS:
|
|
|
454
480
|
List of dictionaries with images search results.
|
|
455
481
|
|
|
456
482
|
Raises:
|
|
457
|
-
WebscoutE: Base exception for
|
|
483
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
458
484
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
459
485
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
460
486
|
"""
|
|
@@ -505,11 +531,19 @@ class AsyncWEBS:
|
|
|
505
531
|
}
|
|
506
532
|
results[priority] = result
|
|
507
533
|
|
|
508
|
-
tasks = [_images_page(0, page=0)]
|
|
534
|
+
tasks = [asyncio.create_task(_images_page(0, page=0))]
|
|
509
535
|
if max_results:
|
|
510
536
|
max_results = min(max_results, 500)
|
|
511
|
-
tasks.extend(
|
|
512
|
-
|
|
537
|
+
tasks.extend(
|
|
538
|
+
asyncio.create_task(_images_page(s, i)) for i, s in enumerate(range(100, max_results, 100), start=1)
|
|
539
|
+
)
|
|
540
|
+
try:
|
|
541
|
+
await asyncio.gather(*tasks)
|
|
542
|
+
except Exception as e:
|
|
543
|
+
for task in tasks:
|
|
544
|
+
task.cancel()
|
|
545
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
546
|
+
raise e
|
|
513
547
|
|
|
514
548
|
return list(islice(filter(None, results), max_results))
|
|
515
549
|
|
|
@@ -524,7 +558,7 @@ class AsyncWEBS:
|
|
|
524
558
|
license_videos: Optional[str] = None,
|
|
525
559
|
max_results: Optional[int] = None,
|
|
526
560
|
) -> List[Dict[str, str]]:
|
|
527
|
-
"""
|
|
561
|
+
"""webscout videos search. Query params: https://duckduckgo.com/params.
|
|
528
562
|
|
|
529
563
|
Args:
|
|
530
564
|
keywords: keywords for query.
|
|
@@ -540,7 +574,7 @@ class AsyncWEBS:
|
|
|
540
574
|
List of dictionaries with videos search results.
|
|
541
575
|
|
|
542
576
|
Raises:
|
|
543
|
-
WebscoutE: Base exception for
|
|
577
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
544
578
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
545
579
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
546
580
|
"""
|
|
@@ -579,11 +613,19 @@ class AsyncWEBS:
|
|
|
579
613
|
priority += 1
|
|
580
614
|
results[priority] = row
|
|
581
615
|
|
|
582
|
-
tasks = [_videos_page(0, 0)]
|
|
616
|
+
tasks = [asyncio.create_task(_videos_page(0, 0))]
|
|
583
617
|
if max_results:
|
|
584
618
|
max_results = min(max_results, 400)
|
|
585
|
-
tasks.extend(
|
|
586
|
-
|
|
619
|
+
tasks.extend(
|
|
620
|
+
asyncio.create_task(_videos_page(s, i)) for i, s in enumerate(range(59, max_results, 59), start=1)
|
|
621
|
+
)
|
|
622
|
+
try:
|
|
623
|
+
await asyncio.gather(*tasks)
|
|
624
|
+
except Exception as e:
|
|
625
|
+
for task in tasks:
|
|
626
|
+
task.cancel()
|
|
627
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
628
|
+
raise e
|
|
587
629
|
|
|
588
630
|
return list(islice(filter(None, results), max_results))
|
|
589
631
|
|
|
@@ -595,7 +637,7 @@ class AsyncWEBS:
|
|
|
595
637
|
timelimit: Optional[str] = None,
|
|
596
638
|
max_results: Optional[int] = None,
|
|
597
639
|
) -> List[Dict[str, str]]:
|
|
598
|
-
"""
|
|
640
|
+
"""webscout news search. Query params: https://duckduckgo.com/params.
|
|
599
641
|
|
|
600
642
|
Args:
|
|
601
643
|
keywords: keywords for query.
|
|
@@ -608,7 +650,7 @@ class AsyncWEBS:
|
|
|
608
650
|
List of dictionaries with news search results.
|
|
609
651
|
|
|
610
652
|
Raises:
|
|
611
|
-
WebscoutE: Base exception for
|
|
653
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
612
654
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
613
655
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
614
656
|
"""
|
|
@@ -653,16 +695,24 @@ class AsyncWEBS:
|
|
|
653
695
|
}
|
|
654
696
|
results[priority] = result
|
|
655
697
|
|
|
656
|
-
tasks = [_news_page(0, 0)]
|
|
698
|
+
tasks = [asyncio.create_task(_news_page(0, 0))]
|
|
657
699
|
if max_results:
|
|
658
700
|
max_results = min(max_results, 200)
|
|
659
|
-
tasks.extend(
|
|
660
|
-
|
|
701
|
+
tasks.extend(
|
|
702
|
+
asyncio.create_task(_news_page(s, i)) for i, s in enumerate(range(29, max_results, 29), start=1)
|
|
703
|
+
)
|
|
704
|
+
try:
|
|
705
|
+
await asyncio.gather(*tasks)
|
|
706
|
+
except Exception as e:
|
|
707
|
+
for task in tasks:
|
|
708
|
+
task.cancel()
|
|
709
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
710
|
+
raise e
|
|
661
711
|
|
|
662
712
|
return list(islice(filter(None, results), max_results))
|
|
663
713
|
|
|
664
714
|
async def answers(self, keywords: str) -> List[Dict[str, str]]:
|
|
665
|
-
"""
|
|
715
|
+
"""webscout instant answers. Query params: https://duckduckgo.com/params.
|
|
666
716
|
|
|
667
717
|
Args:
|
|
668
718
|
keywords: keywords for query,
|
|
@@ -671,7 +721,7 @@ class AsyncWEBS:
|
|
|
671
721
|
List of dictionaries with instant answers results.
|
|
672
722
|
|
|
673
723
|
Raises:
|
|
674
|
-
WebscoutE: Base exception for
|
|
724
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
675
725
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
676
726
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
677
727
|
"""
|
|
@@ -733,7 +783,7 @@ class AsyncWEBS:
|
|
|
733
783
|
return results
|
|
734
784
|
|
|
735
785
|
async def suggestions(self, keywords: str, region: str = "wt-wt") -> List[Dict[str, str]]:
|
|
736
|
-
"""
|
|
786
|
+
"""webscout suggestions. Query params: https://duckduckgo.com/params.
|
|
737
787
|
|
|
738
788
|
Args:
|
|
739
789
|
keywords: keywords for query.
|
|
@@ -743,7 +793,7 @@ class AsyncWEBS:
|
|
|
743
793
|
List of dictionaries with suggestions results.
|
|
744
794
|
|
|
745
795
|
Raises:
|
|
746
|
-
WebscoutE: Base exception for
|
|
796
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
747
797
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
748
798
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
749
799
|
"""
|
|
@@ -772,7 +822,7 @@ class AsyncWEBS:
|
|
|
772
822
|
radius: int = 0,
|
|
773
823
|
max_results: Optional[int] = None,
|
|
774
824
|
) -> List[Dict[str, str]]:
|
|
775
|
-
"""
|
|
825
|
+
"""webscout maps search. Query params: https://duckduckgo.com/params.
|
|
776
826
|
|
|
777
827
|
Args:
|
|
778
828
|
keywords: keywords for query
|
|
@@ -793,7 +843,7 @@ class AsyncWEBS:
|
|
|
793
843
|
List of dictionaries with maps search results, or None if there was an error.
|
|
794
844
|
|
|
795
845
|
Raises:
|
|
796
|
-
WebscoutE: Base exception for
|
|
846
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
797
847
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
798
848
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
799
849
|
"""
|
|
@@ -945,7 +995,7 @@ class AsyncWEBS:
|
|
|
945
995
|
async def translate(
|
|
946
996
|
self, keywords: Union[List[str], str], from_: Optional[str] = None, to: str = "en"
|
|
947
997
|
) -> List[Dict[str, str]]:
|
|
948
|
-
"""
|
|
998
|
+
"""webscout translate.
|
|
949
999
|
|
|
950
1000
|
Args:
|
|
951
1001
|
keywords: string or list of strings to translate.
|
|
@@ -956,7 +1006,7 @@ class AsyncWEBS:
|
|
|
956
1006
|
List od dictionaries with translated keywords.
|
|
957
1007
|
|
|
958
1008
|
Raises:
|
|
959
|
-
WebscoutE: Base exception for
|
|
1009
|
+
WebscoutE: Base exception for webscout_search errors.
|
|
960
1010
|
RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
|
|
961
1011
|
TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
|
|
962
1012
|
"""
|
|
@@ -987,7 +1037,13 @@ class AsyncWEBS:
|
|
|
987
1037
|
|
|
988
1038
|
if isinstance(keywords, str):
|
|
989
1039
|
keywords = [keywords]
|
|
990
|
-
tasks = [_translate_keyword(keyword) for keyword in keywords]
|
|
991
|
-
|
|
1040
|
+
tasks = [asyncio.create_task(_translate_keyword(keyword)) for keyword in keywords]
|
|
1041
|
+
try:
|
|
1042
|
+
await asyncio.gather(*tasks)
|
|
1043
|
+
except Exception as e:
|
|
1044
|
+
for task in tasks:
|
|
1045
|
+
task.cancel()
|
|
1046
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1047
|
+
raise e
|
|
992
1048
|
|
|
993
1049
|
return results
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webscout
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.6
|
|
4
4
|
Summary: Search for anything using the Google, DuckDuckGo, phind.com. Also containes AI models, can transcribe yt videos, temporary email and phone number generation, have TTS support and webai(terminal gpt and open interpeter)
|
|
5
5
|
Author: OEvortex
|
|
6
6
|
Author-email: helpingai5@gmail.com
|
|
@@ -24,14 +24,14 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
|
24
24
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE.md
|
|
27
|
-
Requires-Dist: docstring-inheritance
|
|
28
|
-
Requires-Dist: click
|
|
29
|
-
Requires-Dist: curl-cffi
|
|
30
|
-
Requires-Dist: lxml
|
|
31
|
-
Requires-Dist: nest-asyncio
|
|
32
|
-
Requires-Dist: selenium
|
|
33
|
-
Requires-Dist: tqdm
|
|
34
|
-
Requires-Dist: webdriver-manager
|
|
27
|
+
Requires-Dist: docstring-inheritance
|
|
28
|
+
Requires-Dist: click
|
|
29
|
+
Requires-Dist: curl-cffi
|
|
30
|
+
Requires-Dist: lxml
|
|
31
|
+
Requires-Dist: nest-asyncio
|
|
32
|
+
Requires-Dist: selenium
|
|
33
|
+
Requires-Dist: tqdm
|
|
34
|
+
Requires-Dist: webdriver-manager
|
|
35
35
|
Requires-Dist: halo >=0.0.31
|
|
36
36
|
Requires-Dist: g4f >=0.2.2.3
|
|
37
37
|
Requires-Dist: rich
|
|
@@ -10,29 +10,30 @@ DeepWEBS/networks/webpage_fetcher.py,sha256=vRB9T3o-nMgrMkG2NPHTDctNeXaPSKCmBXqu
|
|
|
10
10
|
DeepWEBS/utilsdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
11
|
DeepWEBS/utilsdw/enver.py,sha256=vpI7s4_o_VL9govSryOv-z1zYK3pTEW3-H9QNN8JYtc,2472
|
|
12
12
|
DeepWEBS/utilsdw/logger.py,sha256=Z0nFUcEGyU8r28yKiIyvEtO26xxpmJgbvNToTfwZecc,8174
|
|
13
|
-
webscout/AI.py,sha256=
|
|
13
|
+
webscout/AI.py,sha256=Iw19aegKqEUD87r0LDMK0qXQ0uCv9qKVSH6ZYJyCJYI,226181
|
|
14
|
+
webscout/AIauto.py,sha256=NlIx-Nfuq-xJ3uZUOUJUXtZ2tzwcbx1ViIlnVK2aCrw,17297
|
|
14
15
|
webscout/AIbase.py,sha256=GoHbN8r0gq2saYRZv6LA-Fr9Jlcjv80STKFXUq2ZeGU,4710
|
|
15
|
-
webscout/AIutel.py,sha256=
|
|
16
|
+
webscout/AIutel.py,sha256=Kan_bxd3KqNKZbbOQq_4MVg7fG_Caz-WgYOliUQClU0,33233
|
|
16
17
|
webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
|
|
17
18
|
webscout/LLM.py,sha256=CiDz0okZNEoXuxMwadZnwRGSLpqk2zg0vzvXSxQZjcE,1910
|
|
18
|
-
webscout/__init__.py,sha256=
|
|
19
|
+
webscout/__init__.py,sha256=k67obDLdOiBMKlrqId7y9DuwI_vxLS__3J6DdZaOJ5k,1104
|
|
19
20
|
webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
|
|
20
|
-
webscout/async_providers.py,sha256=
|
|
21
|
+
webscout/async_providers.py,sha256=KGWKAhdEh4nMntLtyCaO0p827Tcg__gBLT_MLRO4l5o,711
|
|
21
22
|
webscout/cli.py,sha256=F888fdrFUQgczMBN4yMOSf6Nh-IbvkqpPhDsbnA2FtQ,17059
|
|
22
|
-
webscout/exceptions.py,sha256=
|
|
23
|
+
webscout/exceptions.py,sha256=Wx8bEN3bz1nNZ9PAZHX8jwvFPddF9Y2pHAEwCMu_VJc,498
|
|
23
24
|
webscout/g4f.py,sha256=F7POjR03ek7eZvcTX-p7gMe1b0nLNoIqF-L_vZwos0c,24489
|
|
24
25
|
webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
|
|
25
26
|
webscout/tempid.py,sha256=5oc3UbXhPGKxrMRTfRABT-V-dNzH_hOKWtLYM6iCWd4,5896
|
|
26
27
|
webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
|
|
27
28
|
webscout/utils.py,sha256=CxeXvp0rWIulUrEaPZMaNfg_tSuQLRSV8uuHA2chyKE,2603
|
|
28
|
-
webscout/version.py,sha256=
|
|
29
|
+
webscout/version.py,sha256=2IusSRAul_UY0-wnbdAHj0XD7AIfWOrO2BBUSV-Sep0,25
|
|
29
30
|
webscout/voice.py,sha256=0QjXTHAQmCK07IDZXRc7JXem47cnPJH7u3X0sVP1-UQ,967
|
|
30
|
-
webscout/webai.py,sha256=
|
|
31
|
-
webscout/webscout_search.py,sha256=
|
|
32
|
-
webscout/webscout_search_async.py,sha256=
|
|
33
|
-
webscout-1.4.
|
|
34
|
-
webscout-1.4.
|
|
35
|
-
webscout-1.4.
|
|
36
|
-
webscout-1.4.
|
|
37
|
-
webscout-1.4.
|
|
38
|
-
webscout-1.4.
|
|
31
|
+
webscout/webai.py,sha256=_bwgdEG14-zkiLmToaz7rD6iUJ3mALRM_-BFpBs263Y,83660
|
|
32
|
+
webscout/webscout_search.py,sha256=TvbrRYVMXbFGgEh0CoFHNYVY3iQ8SmejxEmv8Csu4IA,3159
|
|
33
|
+
webscout/webscout_search_async.py,sha256=4_L_t_I9WlvpPEI3FI0K3v6Aayr0pNvD3chYOp7JR8o,42902
|
|
34
|
+
webscout-1.4.6.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
|
|
35
|
+
webscout-1.4.6.dist-info/METADATA,sha256=7-I6QU3jBTJ6ZrIB_vpBbYh39nSWHhI-PrVDAMHTLwM,43363
|
|
36
|
+
webscout-1.4.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
37
|
+
webscout-1.4.6.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
|
|
38
|
+
webscout-1.4.6.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
|
|
39
|
+
webscout-1.4.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|