webscout 5.2__tar.gz → 5.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of webscout might be problematic. Click here for more details.

Files changed (102) hide show
  1. {webscout-5.2/webscout.egg-info → webscout-5.3}/PKG-INFO +12 -73
  2. {webscout-5.2 → webscout-5.3}/README.md +11 -72
  3. {webscout-5.2 → webscout-5.3}/setup.py +1 -1
  4. {webscout-5.2 → webscout-5.3}/webscout/AIutel.py +10 -10
  5. webscout-5.3/webscout/DWEBS.py +179 -0
  6. webscout-5.3/webscout/Provider/Chatify.py +174 -0
  7. {webscout-5.2 → webscout-5.3}/webscout/Provider/NetFly.py +21 -61
  8. {webscout-5.2 → webscout-5.3}/webscout/Provider/RUBIKSAI.py +11 -5
  9. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTS/streamElements.py +4 -8
  10. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTS/voicepod.py +11 -7
  11. {webscout-5.2 → webscout-5.3}/webscout/Provider/__init__.py +6 -4
  12. {webscout-5.2 → webscout-5.3}/webscout/Provider/ai4chat.py +14 -8
  13. webscout-5.3/webscout/Provider/cerebras.py +199 -0
  14. {webscout-5.2 → webscout-5.3}/webscout/Provider/felo_search.py +28 -68
  15. webscout-5.3/webscout/Provider/x0gpt.py +181 -0
  16. {webscout-5.2 → webscout-5.3}/webscout/__init__.py +2 -2
  17. {webscout-5.2 → webscout-5.3}/webscout/exceptions.py +2 -1
  18. {webscout-5.2 → webscout-5.3}/webscout/transcriber.py +195 -140
  19. {webscout-5.2 → webscout-5.3/webscout.egg-info}/PKG-INFO +12 -73
  20. {webscout-5.2 → webscout-5.3}/webscout.egg-info/SOURCES.txt +3 -3
  21. webscout-5.2/webscout/DWEBS.py +0 -157
  22. webscout-5.2/webscout/Provider/Berlin4h.py +0 -217
  23. webscout-5.2/webscout/Provider/liaobots.py +0 -268
  24. webscout-5.2/webscout/voice.py +0 -34
  25. {webscout-5.2 → webscout-5.3}/LICENSE.md +0 -0
  26. {webscout-5.2 → webscout-5.3}/setup.cfg +0 -0
  27. {webscout-5.2 → webscout-5.3}/webscout/AIauto.py +0 -0
  28. {webscout-5.2 → webscout-5.3}/webscout/AIbase.py +0 -0
  29. {webscout-5.2 → webscout-5.3}/webscout/Agents/Onlinesearcher.py +0 -0
  30. {webscout-5.2 → webscout-5.3}/webscout/Agents/__init__.py +0 -0
  31. {webscout-5.2 → webscout-5.3}/webscout/Agents/ai.py +0 -0
  32. {webscout-5.2 → webscout-5.3}/webscout/Agents/functioncall.py +0 -0
  33. {webscout-5.2 → webscout-5.3}/webscout/Bard.py +0 -0
  34. {webscout-5.2 → webscout-5.3}/webscout/Bing_search.py +0 -0
  35. {webscout-5.2 → webscout-5.3}/webscout/Extra/__init__.py +0 -0
  36. {webscout-5.2 → webscout-5.3}/webscout/Extra/autollama.py +0 -0
  37. {webscout-5.2 → webscout-5.3}/webscout/Extra/gguf.py +0 -0
  38. {webscout-5.2 → webscout-5.3}/webscout/Extra/weather.py +0 -0
  39. {webscout-5.2 → webscout-5.3}/webscout/Extra/weather_ascii.py +0 -0
  40. {webscout-5.2 → webscout-5.3}/webscout/LLM.py +0 -0
  41. {webscout-5.2 → webscout-5.3}/webscout/Local/__init__.py +0 -0
  42. {webscout-5.2 → webscout-5.3}/webscout/Local/_version.py +0 -0
  43. {webscout-5.2 → webscout-5.3}/webscout/Local/formats.py +0 -0
  44. {webscout-5.2 → webscout-5.3}/webscout/Local/model.py +0 -0
  45. {webscout-5.2 → webscout-5.3}/webscout/Local/rawdog.py +0 -0
  46. {webscout-5.2 → webscout-5.3}/webscout/Local/samplers.py +0 -0
  47. {webscout-5.2 → webscout-5.3}/webscout/Local/thread.py +0 -0
  48. {webscout-5.2 → webscout-5.3}/webscout/Local/utils.py +0 -0
  49. {webscout-5.2 → webscout-5.3}/webscout/Provider/AI21.py +0 -0
  50. {webscout-5.2 → webscout-5.3}/webscout/Provider/Andi.py +0 -0
  51. {webscout-5.2 → webscout-5.3}/webscout/Provider/BasedGPT.py +0 -0
  52. {webscout-5.2 → webscout-5.3}/webscout/Provider/Blackboxai.py +0 -0
  53. {webscout-5.2 → webscout-5.3}/webscout/Provider/Cloudflare.py +0 -0
  54. {webscout-5.2 → webscout-5.3}/webscout/Provider/Cohere.py +0 -0
  55. {webscout-5.2 → webscout-5.3}/webscout/Provider/DARKAI.py +0 -0
  56. {webscout-5.2 → webscout-5.3}/webscout/Provider/Deepinfra.py +0 -0
  57. {webscout-5.2 → webscout-5.3}/webscout/Provider/Deepseek.py +0 -0
  58. {webscout-5.2 → webscout-5.3}/webscout/Provider/DiscordRocks.py +0 -0
  59. {webscout-5.2 → webscout-5.3}/webscout/Provider/EDITEE.py +0 -0
  60. {webscout-5.2 → webscout-5.3}/webscout/Provider/Farfalle.py +0 -0
  61. {webscout-5.2 → webscout-5.3}/webscout/Provider/Gemini.py +0 -0
  62. {webscout-5.2 → webscout-5.3}/webscout/Provider/Groq.py +0 -0
  63. {webscout-5.2 → webscout-5.3}/webscout/Provider/Koboldai.py +0 -0
  64. {webscout-5.2 → webscout-5.3}/webscout/Provider/Llama.py +0 -0
  65. {webscout-5.2 → webscout-5.3}/webscout/Provider/Llama3.py +0 -0
  66. {webscout-5.2 → webscout-5.3}/webscout/Provider/OLLAMA.py +0 -0
  67. {webscout-5.2 → webscout-5.3}/webscout/Provider/Openai.py +0 -0
  68. {webscout-5.2 → webscout-5.3}/webscout/Provider/PI.py +0 -0
  69. {webscout-5.2 → webscout-5.3}/webscout/Provider/Perplexity.py +0 -0
  70. {webscout-5.2 → webscout-5.3}/webscout/Provider/Phind.py +0 -0
  71. {webscout-5.2 → webscout-5.3}/webscout/Provider/PizzaGPT.py +0 -0
  72. {webscout-5.2 → webscout-5.3}/webscout/Provider/Poe.py +0 -0
  73. {webscout-5.2 → webscout-5.3}/webscout/Provider/Reka.py +0 -0
  74. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTI/PollinationsAI.py +0 -0
  75. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTI/__init__.py +0 -0
  76. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTI/deepinfra.py +0 -0
  77. {webscout-5.2 → webscout-5.3}/webscout/Provider/TTS/__init__.py +0 -0
  78. {webscout-5.2 → webscout-5.3}/webscout/Provider/TeachAnything.py +0 -0
  79. {webscout-5.2 → webscout-5.3}/webscout/Provider/ThinkAnyAI.py +0 -0
  80. {webscout-5.2 → webscout-5.3}/webscout/Provider/Youchat.py +0 -0
  81. {webscout-5.2 → webscout-5.3}/webscout/Provider/julius.py +0 -0
  82. {webscout-5.2 → webscout-5.3}/webscout/Provider/koala.py +0 -0
  83. {webscout-5.2 → webscout-5.3}/webscout/Provider/meta.py +0 -0
  84. {webscout-5.2 → webscout-5.3}/webscout/Provider/turboseek.py +0 -0
  85. {webscout-5.2 → webscout-5.3}/webscout/Provider/xdash.py +0 -0
  86. {webscout-5.2 → webscout-5.3}/webscout/Provider/yep.py +0 -0
  87. {webscout-5.2 → webscout-5.3}/webscout/YTdownloader.py +0 -0
  88. {webscout-5.2 → webscout-5.3}/webscout/__main__.py +0 -0
  89. {webscout-5.2 → webscout-5.3}/webscout/cli.py +0 -0
  90. {webscout-5.2 → webscout-5.3}/webscout/g4f.py +0 -0
  91. {webscout-5.2 → webscout-5.3}/webscout/models.py +0 -0
  92. {webscout-5.2 → webscout-5.3}/webscout/tempid.py +0 -0
  93. {webscout-5.2 → webscout-5.3}/webscout/utils.py +0 -0
  94. {webscout-5.2 → webscout-5.3}/webscout/version.py +0 -0
  95. {webscout-5.2 → webscout-5.3}/webscout/webai.py +0 -0
  96. {webscout-5.2 → webscout-5.3}/webscout/webscout_search.py +0 -0
  97. {webscout-5.2 → webscout-5.3}/webscout/webscout_search_async.py +0 -0
  98. {webscout-5.2 → webscout-5.3}/webscout/websx_search.py +0 -0
  99. {webscout-5.2 → webscout-5.3}/webscout.egg-info/dependency_links.txt +0 -0
  100. {webscout-5.2 → webscout-5.3}/webscout.egg-info/entry_points.txt +0 -0
  101. {webscout-5.2 → webscout-5.3}/webscout.egg-info/requires.txt +0 -0
  102. {webscout-5.2 → webscout-5.3}/webscout.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: webscout
3
- Version: 5.2
3
+ Version: 5.3
4
4
  Summary: Search for anything using Google, DuckDuckGo, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
5
5
  Author: OEvortex
6
6
  Author-email: helpingai5@gmail.com
@@ -339,57 +339,12 @@ if __name__ == '__main__':
339
339
  ## Transcriber
340
340
  The transcriber function in webscout is a handy tool that transcribes YouTube videos. Here's an example code demonstrating its usage:
341
341
  ```python
342
- import sys
343
- from webscout import transcriber
344
-
345
- def extract_transcript(video_id):
346
- """Extracts the transcript from a YouTube video."""
347
- try:
348
- transcript_list = transcriber.list_transcripts(video_id)
349
- for transcript in transcript_list:
350
- transcript_data_list = transcript.fetch()
351
- lang = transcript.language
352
- transcript_text = ""
353
- if transcript.language_code == 'en':
354
- for line in transcript_data_list:
355
- start_time = line['start']
356
- end_time = start_time + line['duration']
357
- formatted_line = f"{start_time:.2f} - {end_time:.2f}: {line['text']}\n"
358
- transcript_text += formatted_line
359
- return transcript_text
360
- elif transcript.is_translatable:
361
- english_transcript_list = transcript.translate('en').fetch()
362
- for line in english_transcript_list:
363
- start_time = line['start']
364
- end_time = start_time + line['duration']
365
- formatted_line = f"{start_time:.2f} - {end_time:.2f}: {line['text']}\n"
366
- transcript_text += formatted_line
367
- return transcript_text
368
- print("Transcript extraction failed. Please check the video URL.")
369
- except Exception as e:
370
- print(f"Error: {e}")
371
-
372
- def main():
373
- video_url = input("Enter the video link: ")
374
-
375
- if video_url:
376
- video_id = video_url.split("=")[1]
377
- print("Video URL:", video_url)
378
- submit = input("Press 'Enter' to get the transcript or type 'exit' to quit: ")
379
- if submit == '':
380
- print("Extracting Transcript...")
381
- transcript = extract_transcript(video_id)
382
- print('Transcript:')
383
- print(transcript)
384
- print("__________________________________________________________________________________")
385
- elif submit.lower() == 'exit':
386
- print("Exiting...")
387
- sys.exit()
388
- else:
389
- print("Invalid input. Please try again.")
390
-
391
- if __name__ == "__main__":
392
- main()
342
+ from webscout import YTTranscriber
343
+ yt = YTTranscriber()
344
+ from rich import print
345
+ video_url = input("Enter the YouTube video URL: ")
346
+ transcript = yt.get_transcript(video_url, languages=None)
347
+ print(transcript)
393
348
  ```
394
349
 
395
350
  ## GoogleS -- formerly DWEBS
@@ -397,7 +352,7 @@ if __name__ == "__main__":
397
352
  from webscout import GoogleS
398
353
  from rich import print
399
354
  searcher = GoogleS()
400
- results = searcher.search("HelpingAI-9B", max_results=20, extract_webpage_text=False, max_extract_characters=100)
355
+ results = searcher.search("HelpingAI-9B", max_results=20, extract_text=False, max_text_length=200)
401
356
  for result in results:
402
357
  print(result)
403
358
  ```
@@ -411,36 +366,20 @@ for result in results:
411
366
  print(result)
412
367
  ```
413
368
 
414
- ## Text-to-Speech:
415
- ```python
416
- from webscout import play_audio
417
-
418
- message = "This is an example of text-to-speech."
419
- audio_content = play_audio(message, voice="Brian")
420
-
421
- # Save the audio to a file
422
- with open("output.mp3", "wb") as f:
423
- f.write(audio_content)
424
- ```
425
- ### Available TTS Voices:
426
- You can choose from a wide range of voices, including:
427
- - Filiz, Astrid, Tatyana, Maxim, Carmen, Ines, Cristiano, Vitoria, Ricardo, Maja, Jan, Jacek, Ewa, Ruben, Lotte, Liv, Seoyeon, Takumi, Mizuki, Giorgio, Carla, Bianca, Karl, Dora, Mathieu, Celine, Chantal, Penelope, Miguel, Mia, Enrique, Conchita, Geraint, Salli, Matthew, Kimberly, Kendra, Justin, Joey, Joanna, Ivy, Raveena, Aditi, Emma, Brian, Amy, Russell, Nicole, Vicki, Marlene, Hans, Naja, Mads, Gwyneth, Zhiyu
428
- - Standard and WaveNet voices for various languages (e.g., en-US, es-ES, ja-JP, etc.)
429
-
430
369
 
431
370
  The WEBS and AsyncWEBS classes are used to retrieve search results from DuckDuckGo.com
432
371
  To use the AsyncWEBS class, you can perform asynchronous operations using Python's asyncio library.
433
372
  To initialize an instance of the WEBS or AsyncWEBS classes, you can provide the following optional arguments:
434
373
 
435
374
  Here is an example of initializing the WEBS class:
436
- ```python3
375
+ ```python
437
376
  from webscout import WEBS
438
377
 
439
378
  R = WEBS().text("python programming", max_results=5)
440
379
  print(R)
441
380
  ```
442
381
  Here is an example of initializing the AsyncWEBS class:
443
- ```python3
382
+ ```python
444
383
  import asyncio
445
384
  import logging
446
385
  import sys
@@ -913,7 +852,7 @@ resp = bot.generate("AI-generated image - webscout", 1)
913
852
  print(bot.save(resp))
914
853
  ```
915
854
 
916
- ### Text to Speach - Voicepods, StreamElements
855
+ ### Text to speech - Voicepods, StreamElements
917
856
  ```python
918
857
  from webscout import Voicepods
919
858
  voicepods = Voicepods()
@@ -1492,7 +1431,7 @@ if "error" not in function_call_data:
1492
1431
  else:
1493
1432
  print(f"Error: {function_call_data['error']}")
1494
1433
  ```
1495
- ### LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, NetFly, Editee
1434
+ ### LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, NetFly, Editee, AI21, Chatify, Cerebras, X0GPT
1496
1435
  code similar to other provider
1497
1436
  ### `LLM`
1498
1437
  ```python
@@ -260,57 +260,12 @@ if __name__ == '__main__':
260
260
  ## Transcriber
261
261
  The transcriber function in webscout is a handy tool that transcribes YouTube videos. Here's an example code demonstrating its usage:
262
262
  ```python
263
- import sys
264
- from webscout import transcriber
265
-
266
- def extract_transcript(video_id):
267
- """Extracts the transcript from a YouTube video."""
268
- try:
269
- transcript_list = transcriber.list_transcripts(video_id)
270
- for transcript in transcript_list:
271
- transcript_data_list = transcript.fetch()
272
- lang = transcript.language
273
- transcript_text = ""
274
- if transcript.language_code == 'en':
275
- for line in transcript_data_list:
276
- start_time = line['start']
277
- end_time = start_time + line['duration']
278
- formatted_line = f"{start_time:.2f} - {end_time:.2f}: {line['text']}\n"
279
- transcript_text += formatted_line
280
- return transcript_text
281
- elif transcript.is_translatable:
282
- english_transcript_list = transcript.translate('en').fetch()
283
- for line in english_transcript_list:
284
- start_time = line['start']
285
- end_time = start_time + line['duration']
286
- formatted_line = f"{start_time:.2f} - {end_time:.2f}: {line['text']}\n"
287
- transcript_text += formatted_line
288
- return transcript_text
289
- print("Transcript extraction failed. Please check the video URL.")
290
- except Exception as e:
291
- print(f"Error: {e}")
292
-
293
- def main():
294
- video_url = input("Enter the video link: ")
295
-
296
- if video_url:
297
- video_id = video_url.split("=")[1]
298
- print("Video URL:", video_url)
299
- submit = input("Press 'Enter' to get the transcript or type 'exit' to quit: ")
300
- if submit == '':
301
- print("Extracting Transcript...")
302
- transcript = extract_transcript(video_id)
303
- print('Transcript:')
304
- print(transcript)
305
- print("__________________________________________________________________________________")
306
- elif submit.lower() == 'exit':
307
- print("Exiting...")
308
- sys.exit()
309
- else:
310
- print("Invalid input. Please try again.")
311
-
312
- if __name__ == "__main__":
313
- main()
263
+ from webscout import YTTranscriber
264
+ yt = YTTranscriber()
265
+ from rich import print
266
+ video_url = input("Enter the YouTube video URL: ")
267
+ transcript = yt.get_transcript(video_url, languages=None)
268
+ print(transcript)
314
269
  ```
315
270
 
316
271
  ## GoogleS -- formerly DWEBS
@@ -318,7 +273,7 @@ if __name__ == "__main__":
318
273
  from webscout import GoogleS
319
274
  from rich import print
320
275
  searcher = GoogleS()
321
- results = searcher.search("HelpingAI-9B", max_results=20, extract_webpage_text=False, max_extract_characters=100)
276
+ results = searcher.search("HelpingAI-9B", max_results=20, extract_text=False, max_text_length=200)
322
277
  for result in results:
323
278
  print(result)
324
279
  ```
@@ -332,36 +287,20 @@ for result in results:
332
287
  print(result)
333
288
  ```
334
289
 
335
- ## Text-to-Speech:
336
- ```python
337
- from webscout import play_audio
338
-
339
- message = "This is an example of text-to-speech."
340
- audio_content = play_audio(message, voice="Brian")
341
-
342
- # Save the audio to a file
343
- with open("output.mp3", "wb") as f:
344
- f.write(audio_content)
345
- ```
346
- ### Available TTS Voices:
347
- You can choose from a wide range of voices, including:
348
- - Filiz, Astrid, Tatyana, Maxim, Carmen, Ines, Cristiano, Vitoria, Ricardo, Maja, Jan, Jacek, Ewa, Ruben, Lotte, Liv, Seoyeon, Takumi, Mizuki, Giorgio, Carla, Bianca, Karl, Dora, Mathieu, Celine, Chantal, Penelope, Miguel, Mia, Enrique, Conchita, Geraint, Salli, Matthew, Kimberly, Kendra, Justin, Joey, Joanna, Ivy, Raveena, Aditi, Emma, Brian, Amy, Russell, Nicole, Vicki, Marlene, Hans, Naja, Mads, Gwyneth, Zhiyu
349
- - Standard and WaveNet voices for various languages (e.g., en-US, es-ES, ja-JP, etc.)
350
-
351
290
 
352
291
  The WEBS and AsyncWEBS classes are used to retrieve search results from DuckDuckGo.com
353
292
  To use the AsyncWEBS class, you can perform asynchronous operations using Python's asyncio library.
354
293
  To initialize an instance of the WEBS or AsyncWEBS classes, you can provide the following optional arguments:
355
294
 
356
295
  Here is an example of initializing the WEBS class:
357
- ```python3
296
+ ```python
358
297
  from webscout import WEBS
359
298
 
360
299
  R = WEBS().text("python programming", max_results=5)
361
300
  print(R)
362
301
  ```
363
302
  Here is an example of initializing the AsyncWEBS class:
364
- ```python3
303
+ ```python
365
304
  import asyncio
366
305
  import logging
367
306
  import sys
@@ -834,7 +773,7 @@ resp = bot.generate("AI-generated image - webscout", 1)
834
773
  print(bot.save(resp))
835
774
  ```
836
775
 
837
- ### Text to Speach - Voicepods, StreamElements
776
+ ### Text to speech - Voicepods, StreamElements
838
777
  ```python
839
778
  from webscout import Voicepods
840
779
  voicepods = Voicepods()
@@ -1413,7 +1352,7 @@ if "error" not in function_call_data:
1413
1352
  else:
1414
1353
  print(f"Error: {function_call_data['error']}")
1415
1354
  ```
1416
- ### LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, NetFly, Editee
1355
+ ### LLAMA3, pizzagpt, RUBIKSAI, Koala, Darkai, AI4Chat, Farfalle, PIAI, Felo, XDASH, Julius, YouChat, YEPCHAT, Cloudflare, TurboSeek, NetFly, Editee, AI21, Chatify, Cerebras, X0GPT
1417
1356
  code similar to other provider
1418
1357
  ### `LLM`
1419
1358
  ```python
@@ -5,7 +5,7 @@ with open("README.md", encoding="utf-8") as f:
5
5
 
6
6
  setup(
7
7
  name="webscout",
8
- version="5.2",
8
+ version="5.3",
9
9
  description="Search for anything using Google, DuckDuckGo, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more",
10
10
  long_description=README,
11
11
  long_description_content_type="text/markdown",
@@ -594,16 +594,16 @@ w.print_weather(weather)
594
594
  interpreter (str, optional): Python's interpreter name. Defaults to Python.
595
595
  prettify (bool, optional): Prettify the code on stdout. Defaults to True.
596
596
  """
597
- if not quiet:
598
- print(
599
- "Rawdog is an experimental tool that generates and auto-executes Python scripts in the cli.\n"
600
- "To get the most out of Rawdog. Ensure the following are installed:\n"
601
- " 1. Python 3.x\n"
602
- " 2. Dependency:\n"
603
- " - Matplotlib\n"
604
- "Be alerted on the risk posed! (Experimental)\n"
605
- "Use '--quiet' to suppress this message and code/logs stdout.\n"
606
- )
597
+ # if not quiet:
598
+ # print(
599
+ # "Rawdog is an experimental tool that generates and auto-executes Python scripts in the cli.\n"
600
+ # "To get the most out of Rawdog. Ensure the following are installed:\n"
601
+ # " 1. Python 3.x\n"
602
+ # " 2. Dependency:\n"
603
+ # " - Matplotlib\n"
604
+ # "Be alerted on the risk posed! (Experimental)\n"
605
+ # "Use '--quiet' to suppress this message and code/logs stdout.\n"
606
+ # )
607
607
  self.internal_exec = internal_exec
608
608
  self.confirm_script = confirm_script
609
609
  self.quiet = quiet
@@ -0,0 +1,179 @@
1
+ from bs4 import BeautifulSoup
2
+ import requests
3
+ from typing import Dict, List, Optional, Union
4
+ from concurrent.futures import ThreadPoolExecutor, as_completed
5
+ from urllib.parse import quote
6
+ from termcolor import colored
7
+ import time
8
+ import random
9
+
10
+ class GoogleS:
11
+ """
12
+ Class to perform Google searches and retrieve results.
13
+ """
14
+
15
+ def __init__(
16
+ self,
17
+ headers: Optional[Dict[str, str]] = None,
18
+ proxy: Optional[str] = None,
19
+ timeout: Optional[int] = 10,
20
+ max_workers: int = 20 # Increased max workers for thread pool
21
+ ):
22
+ """Initializes the GoogleS object."""
23
+ self.proxy = proxy
24
+ self.headers = headers if headers else {
25
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62"
26
+ }
27
+ self.headers["Referer"] = "https://www.google.com/"
28
+ self.client = requests.Session()
29
+ self.client.headers.update(self.headers)
30
+ self.client.proxies.update({"http": self.proxy, "https": self.proxy})
31
+ self.timeout = timeout
32
+ self._executor = ThreadPoolExecutor(max_workers=max_workers)
33
+
34
+ def __enter__(self):
35
+ return self
36
+
37
+ def __exit__(self, exc_type, exc_val, exc_tb):
38
+ self.client.close()
39
+
40
+ def _get_url(self, method: str, url: str, params: Optional[Dict[str, str]] = None,
41
+ data: Optional[Union[Dict[str, str], bytes]] = None) -> bytes:
42
+ """
43
+ Makes an HTTP request and returns the response content.
44
+ """
45
+ try:
46
+ resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout)
47
+ except Exception as ex:
48
+ raise Exception(f"{url} {type(ex).__name__}: {ex}") from ex
49
+ if resp.status_code == 200:
50
+ return resp.content
51
+ raise Exception(f"{resp.url} returned status code {resp.status_code}. {params=} {data=}")
52
+
53
+ def _extract_text_from_webpage(self, html_content: bytes, max_characters: Optional[int] = None) -> str:
54
+ """
55
+ Extracts visible text from HTML content using lxml parser.
56
+ """
57
+ soup = BeautifulSoup(html_content, 'lxml') # Use lxml parser
58
+ for tag in soup(["script", "style", "header", "footer", "nav"]):
59
+ tag.extract()
60
+ visible_text = soup.get_text(strip=True)
61
+ if max_characters:
62
+ visible_text = visible_text[:max_characters]
63
+ return visible_text
64
+
65
+ def search(
66
+ self,
67
+ query: str,
68
+ region: str = "us-en",
69
+ language: str = "en",
70
+ safe: str = "off",
71
+ time_period: Optional[str] = None,
72
+ max_results: int = 10,
73
+ extract_text: bool = False,
74
+ max_text_length: Optional[int] = 100,
75
+ ) -> List[Dict[str, Union[str, int]]]:
76
+ """
77
+ Performs a Google search and returns the results.
78
+
79
+ Args:
80
+ query (str): The search query.
81
+ region (str, optional): The region to search in (e.g., "us-en"). Defaults to "us-en".
82
+ language (str, optional): The language of the search results (e.g., "en"). Defaults to "en".
83
+ safe (str, optional): Safe search setting ("off", "active"). Defaults to "off".
84
+ time_period (Optional[str], optional): Time period filter (e.g., "h" for past hour, "d" for past day).
85
+ Defaults to None.
86
+ max_results (int, optional): The maximum number of results to retrieve. Defaults to 10.
87
+ extract_text (bool, optional): Whether to extract text from the linked web pages. Defaults to False.
88
+ max_text_length (Optional[int], optional): The maximum length of the extracted text (in characters).
89
+ Defaults to 100.
90
+
91
+ Returns:
92
+ List[Dict[str, Union[str, int]]]: A list of dictionaries, each representing a search result, containing:
93
+ - 'title': The title of the result.
94
+ - 'href': The URL of the result.
95
+ - 'abstract': The description snippet of the result.
96
+ - 'index': The index of the result in the list.
97
+ - 'type': The type of result (currently always "web").
98
+ - 'visible_text': The extracted text from the web page (if `extract_text` is True).
99
+ """
100
+ assert query, "Query cannot be empty."
101
+
102
+ results = []
103
+ futures = []
104
+ start = 0
105
+
106
+ while len(results) < max_results:
107
+ params = {
108
+ "q": query,
109
+ "num": 10,
110
+ "hl": language,
111
+ "start": start,
112
+ "safe": safe,
113
+ "gl": region,
114
+ }
115
+ if time_period:
116
+ params["tbs"] = f"qdr:{time_period}"
117
+
118
+ futures.append(self._executor.submit(self._get_url, "GET", "https://www.google.com/search", params=params))
119
+ start += 10
120
+
121
+ for future in as_completed(futures):
122
+ try:
123
+ resp_content = future.result()
124
+ soup = BeautifulSoup(resp_content, 'lxml') # Use lxml parser
125
+ result_blocks = soup.find_all("div", class_="g")
126
+
127
+ if not result_blocks:
128
+ break
129
+
130
+ # Extract links and titles first
131
+ for result_block in result_blocks:
132
+ link = result_block.find("a", href=True)
133
+ title = result_block.find("h3")
134
+ description_box = result_block.find(
135
+ "div", {"style": "-webkit-line-clamp:2"}
136
+ )
137
+
138
+ if link and title and description_box:
139
+ url = link["href"]
140
+ results.append({
141
+ "title": title.text,
142
+ "href": url,
143
+ "abstract": description_box.text,
144
+ "index": len(results),
145
+ "type": "web",
146
+ "visible_text": "" # Initialize visible_text as empty string
147
+ })
148
+
149
+ if len(results) >= max_results:
150
+ break # Stop if we have enough results
151
+
152
+ # Parallelize text extraction if needed
153
+ if extract_text:
154
+ with ThreadPoolExecutor(max_workers=self._executor._max_workers) as text_extractor:
155
+ extraction_futures = [
156
+ text_extractor.submit(self._extract_text_from_webpage,
157
+ self._get_url("GET", result['href']),
158
+ max_characters=max_text_length)
159
+ for result in results
160
+ if 'href' in result
161
+ ]
162
+ for i, future in enumerate(as_completed(extraction_futures)):
163
+ try:
164
+ results[i]['visible_text'] = future.result()
165
+ except Exception as e:
166
+ print(f"Error extracting text: {e}")
167
+
168
+ except Exception as e:
169
+ print(f"Error: {e}")
170
+
171
+ return results
172
+
173
+
174
+ if __name__ == "__main__":
175
+ from rich import print
176
+ searcher = GoogleS()
177
+ results = searcher.search("HelpingAI-9B", max_results=20, extract_text=False, max_text_length=200)
178
+ for result in results:
179
+ print(result)
@@ -0,0 +1,174 @@
1
+ from webscout.AIutel import Optimizers
2
+ from webscout.AIutel import Conversation
3
+ from webscout.AIutel import AwesomePrompts
4
+ from webscout.AIbase import Provider
5
+ from webscout import exceptions
6
+ import requests
7
+
8
+ class Chatify(Provider):
9
+ """
10
+ A class to interact with the Chatify AI API.
11
+ """
12
+
13
+ def __init__(
14
+ self,
15
+ is_conversation: bool = True,
16
+ max_tokens: int = 600,
17
+ timeout: int = 30,
18
+ intro: str = None,
19
+ filepath: str = None,
20
+ update_file: bool = True,
21
+ proxies: dict = {},
22
+ history_offset: int = 10250,
23
+ act: str = None,
24
+ system_prompt: str = "You are a helpful and informative AI assistant.",
25
+ ):
26
+ """
27
+ Initializes the Chatify AI API with given parameters.
28
+ """
29
+ self.session = requests.Session()
30
+ self.is_conversation = is_conversation
31
+ self.max_tokens_to_sample = max_tokens
32
+ self.api_endpoint = "https://chatify-ai.vercel.app/api/chat"
33
+ self.timeout = timeout
34
+ self.last_response = {}
35
+ self.headers = {
36
+ 'Accept': '*/*',
37
+ 'Accept-Encoding': 'gzip, deflate, br, zstd',
38
+ 'Accept-Language': 'en-US,en;q=0.9,en-IN;q=0.8',
39
+ 'Content-Type': 'application/json',
40
+ 'DNT': '1',
41
+ 'Origin': 'https://chatify-ai.vercel.app',
42
+ 'Referer': 'https://chatify-ai.vercel.app/',
43
+ 'Sec-CH-UA': '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
44
+ 'Sec-CH-UA-Mobile': '?0',
45
+ 'Sec-CH-UA-Platform': '"Windows"',
46
+ 'Sec-Fetch-Dest': 'empty',
47
+ 'Sec-Fetch-Mode': 'cors',
48
+ 'Sec-Fetch-Site': 'same-origin',
49
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0',
50
+ }
51
+
52
+ self.__available_optimizers = (
53
+ method
54
+ for method in dir(Optimizers)
55
+ if callable(getattr(Optimizers, method)) and not method.startswith("__")
56
+ )
57
+ self.session.headers.update(self.headers)
58
+ Conversation.intro = (
59
+ AwesomePrompts().get_act(
60
+ act, raise_not_found=True, default=None, case_insensitive=True
61
+ )
62
+ if act
63
+ else intro or Conversation.intro
64
+ )
65
+ self.conversation = Conversation(
66
+ is_conversation, self.max_tokens_to_sample, filepath, update_file
67
+ )
68
+ self.conversation.history_offset = history_offset
69
+ self.session.proxies = proxies
70
+ self.system_prompt = system_prompt
71
+
72
+ def ask(
73
+ self,
74
+ prompt: str,
75
+ stream: bool = False,
76
+ raw: bool = False,
77
+ optimizer: str = None,
78
+ conversationally: bool = False,
79
+ ) -> dict:
80
+ """
81
+ Sends a prompt to the Chatify API and returns the response.
82
+ """
83
+ conversation_prompt = self.conversation.gen_complete_prompt(prompt)
84
+ if optimizer:
85
+ if optimizer in self.__available_optimizers:
86
+ conversation_prompt = getattr(Optimizers, optimizer)(
87
+ conversation_prompt if conversationally else prompt
88
+ )
89
+ else:
90
+ raise Exception(
91
+ f"Optimizer is not one of {self.__available_optimizers}"
92
+ )
93
+
94
+ messages = []
95
+ if self.system_prompt:
96
+ messages.append({"role": "system", "content": self.system_prompt})
97
+ messages.append({"role": "user", "content": conversation_prompt})
98
+
99
+ payload = {
100
+ "messages": messages
101
+ }
102
+
103
+ def for_stream():
104
+ response = self.session.post(self.api_endpoint, headers=self.headers, json=payload, stream=True, timeout=self.timeout)
105
+ if not response.ok:
106
+ raise exceptions.FailedToGenerateResponseError(
107
+ f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
108
+ )
109
+
110
+ streaming_text = ""
111
+ for line in response.iter_lines():
112
+ if line:
113
+ decoded_line = line.decode('utf-8') # Decode the line
114
+ parts = decoded_line.split(':', 1)
115
+ if len(parts) > 1:
116
+ content = parts[1].strip().strip('"')
117
+ streaming_text += content
118
+ yield content if raw else dict(text=streaming_text)
119
+ self.last_response.update(dict(text=streaming_text))
120
+ self.conversation.update_chat_history(
121
+ prompt, self.get_message(self.last_response)
122
+ )
123
+
124
+ def for_non_stream():
125
+ for _ in for_stream():
126
+ pass
127
+ return self.last_response
128
+
129
+ return for_stream() if stream else for_non_stream()
130
+
131
+ def chat(
132
+ self,
133
+ prompt: str,
134
+ stream: bool = False,
135
+ optimizer: str = None,
136
+ conversationally: bool = False,
137
+ ) -> str:
138
+ """
139
+ Generates a response from the Chatify API.
140
+ """
141
+
142
+ def for_stream():
143
+ for response in self.ask(
144
+ prompt, True, optimizer=optimizer, conversationally=conversationally
145
+ ):
146
+ yield self.get_message(response)
147
+
148
+ def for_non_stream():
149
+ return self.get_message(
150
+ self.ask(
151
+ prompt,
152
+ False,
153
+ optimizer=optimizer,
154
+ conversationally=conversationally,
155
+ )
156
+ )
157
+
158
+ return for_stream() if stream else for_non_stream()
159
+
160
+ def get_message(self, response: dict) -> str:
161
+ """
162
+ Extracts the message from the API response.
163
+ """
164
+ assert isinstance(response, dict), "Response should be of dict data-type only"
165
+ return response["text"]
166
+
167
+ # Example usage
168
+ if __name__ == "__main__":
169
+ from rich import print
170
+
171
+ ai = Chatify()
172
+ response = ai.chat(input(">>> "))
173
+ for chunk in response:
174
+ print(chunk, end="", flush=True)