webscout 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

DeepWEBS/networks/webpage_fetcher.py

@@ -78,20 +78,18 @@ class BatchWebpageFetcher:
         self.urls = urls
         self.total_count = len(self.urls)
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+        with concurrent.futures.ProcessPoolExecutor() as executor:
             futures = [
-                executor.submit(self.fetch_single_webpage, url, overwrite, output_parent)
+                executor.submit(WebpageFetcher().fetch, url, overwrite, output_parent)
                 for url in urls
             ]
             concurrent.futures.wait(futures)
 
+        self.url_and_html_path_list = [
+            {"url": future.result().url, "html_path": str(future.result().html_path)}
+            for future in futures
+        ]
+
         return self.url_and_html_path_list
 
-if __name__ == "__main__":
-    urls = [
-        "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename",
-        "https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528",
-        "https://docs.python.org/zh-cn/3/tutorial/interpreter.html",
-    ]
-    batch_webpage_fetcher = BatchWebpageFetcher()
-    batch_webpage_fetcher.fetch(urls=urls, overwrite=True, output_parent="python tutorials")
+

DeepWEBS/utilsdw/enver.py

@@ -1,25 +1,41 @@
 import json
 import os
-
 from pathlib import Path
-from DeepWEBS.utilsdw.logger import logger
+from typing import Dict, Optional
+
+from DeepWEBS.utilsdw.logger import OSLogger
 
 
 class OSEnver:
-    def __init__(self):
-        self.envs_stack = []
-        self.envs = os.environ.copy()
+    """Manages the OS environment variables."""
+
+    def __init__(self) -> None:
+        """Initializes the OSEnver object."""
+        self.envs_stack: list[Dict[str, str]] = []
+        self.envs: Dict[str, str] = os.environ.copy()
 
-    def store_envs(self):
-        self.envs_stack.append(self.envs)
+    def store_envs(self) -> None:
+        """Stores a copy of the current environment variables on a stack."""
+        self.envs_stack.append(self.envs.copy())
 
-    def restore_envs(self):
+    def restore_envs(self) -> None:
+        """Restores environment variables from the top of the stack."""
         self.envs = self.envs_stack.pop()
 
-    def set_envs(self, secrets=True, proxies=None, store_envs=True):
-        # caller_info = inspect.stack()[1]
-        # logger.back(f"OS Envs is set by: {caller_info.filename}")
+    def set_envs(
+        self,
+        secrets: bool = True,
+        proxies: Optional[str] = None,
+        store_envs: bool = True,
+    ) -> None:
+        """Sets environment variables based on the contents of secrets.json.
 
+        Args:
+            secrets (bool): Whether to load secrets from secrets.json.
+            proxies (Optional[str]): Proxy URL to set as environment variable.
+            store_envs (bool): Whether to store a copy of the environment variables
+                on the stack.
+        """
         if store_envs:
             self.store_envs()
 
@@ -54,7 +70,9 @@ class OSEnver:
         }
 
         if self.proxy:
-            logger.note(f"Using proxy: [{self.proxy}]")
+            OSLogger().note(f"Using proxy: [{self.proxy}]")
+
+
+enver: OSEnver = OSEnver()
 
 
-enver = OSEnver()

webscout/AIutel.py

@@ -10,14 +10,31 @@ import sys
 import click
 from rich.markdown import Markdown
 from rich.console import Console
-
+import g4f
 appdir = appdirs.AppDirs("AIWEBS", "vortex")
 
 default_path = appdir.user_cache_dir
 
 if not os.path.exists(default_path):
     os.makedirs(default_path)
-
+webai = [
+    "leo",
+    "openai",
+    "opengpt",
+    "koboldai",
+    "gemini",
+    "phind",
+    "blackboxai",
+    "g4fauto",
+    "perplexity",
+    "sean",
+]
+
+gpt4free_providers = [
+    provider.__name__ for provider in g4f.Provider.__providers__  # if provider.working
+]
+
+available_providers = webai + gpt4free_providers
 
 def run_system_command(
     command: str,
@@ -468,7 +485,6 @@ print("The essay is about...")
 ```
 """
 
-    # Idea borrowed from https://github.com/AbanteAI/rawdog
 
     def __init__(
         self,

webscout/__init__.py

@@ -1,7 +1,6 @@
 """Webscout.
 
-Search for words, documents, images, videos, news, maps and text translation 
-using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 """
 import g4f
 import logging

webscout/g4f.py

@@ -2,7 +2,7 @@ import g4f
 from webscout.AIutel import Optimizers
 from webscout.AIutel import Conversation
 from webscout.AIutel import AwesomePrompts
-from webscout.AIutel import Provider
+from webscout.AIbase import Provider
 from webscout.AIutel import available_providers
 
 

webscout/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.3.3"
+__version__ = "1.3.4"
 

webscout/webai.py

@@ -29,6 +29,7 @@ from webscout.AIutel import Optimizers
 from webscout.AIutel import default_path
 from webscout.AIutel import AwesomePrompts
 from webscout.AIutel import RawDog
+from webscout import available_providers
 from colorama import Fore
 from colorama import init as init_colorama
 from dotenv import load_dotenv
@@ -41,7 +42,7 @@ init_colorama(autoreset=True)
 load_dotenv()  # loads .env variables
 
 logging.basicConfig(
-    format="%(asctime)s - %(levelname)s : %(message)s ",  # [%(module)s,%(lineno)s]", # for debug purposes
+    format="%(asctime)s - %(levelname)s : %(message)s ",  
     datefmt="%H:%M:%S",
     level=logging.INFO,
 )
@@ -61,7 +62,7 @@ class this:
 
     rich_code_themes = ["monokai", "paraiso-dark", "igor", "vs", "fruity", "xcode"]
 
-    default_provider = "sean"
+    default_provider = "phind"
 
     getExc = lambda e: e.args[1] if len(e.args) > 1 else str(e)
 
@@ -1077,7 +1078,7 @@ class EntryGroup:
         pass
 
 
-
+import webscout
 class Chatwebai:
     """webai command"""
 
@@ -1194,7 +1195,7 @@ class Chatwebai:
     @click.option(
         "-p",
         "--provider",
-        type=click.Choice(webscout.available_providers),
+        type=click.Choice(available_providers),
         default=this.default_provider,
         help="Name of LLM provider.",
         metavar=(

{webscout-1.3.3.dist-info → webscout-1.3.4.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 1.3.3
-Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos and have TTS support
+Version: 1.3.4
+Summary: Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 Author: OEvortex
 Author-email: helpingai5@gmail.com
 License: HelpingAI Simplified Universal License
@@ -56,7 +56,7 @@ Requires-Dist: pytest >=7.4.2 ; extra == 'dev'
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 
 
 ## Table of Contents
@@ -100,6 +100,8 @@ Search for words, documents, images, videos, news, maps and text translation usi
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
     - [`LLM` with internet](#llm-with-internet)
+  - [`Webai` - terminal gpt and a open interpeter](#webai---terminal-gpt-and-a-open-interpeter)
+    - [for using as terminal gpt](#for-using-as-terminal-gpt)
 
 ## Install
 ```python
@@ -801,3 +803,49 @@ if __name__ == "__main__":
         else:
             print("No response")
 ```
+## `Webai` - terminal gpt and a open interpeter
+
+```python
+from webscout.webai import Main
+
+def use_rawdog_with_webai(prompt):
+    """
+    Wrap the webscout default method in a try-except block to catch any unhandled
+    exceptions and print a helpful message.
+    """
+    try:
+        webai_bot = Main(
+            max_tokens=500, 
+            provider="phind",
+            temperature=0.7,  
+            top_k=40,          
+            top_p=0.95,        
+            model="Phind Model",  # Replace with your desired model
+            auth=None,       # Replace with your auth key/value (if needed)
+            timeout=30,
+            disable_conversation=True,
+            filepath=None,
+            update_file=True,
+            intro=None,
+            rawdog=True,
+            history_offset=10250,
+            awesome_prompt=None,
+            proxy_path=None,
+            quiet=True
+        )
+        webai_response = webai_bot.default(prompt) 
+    except Exception as e:
+        print("Unexpected error:", e)
+
+
+if __name__ == "__main__":
+    user_prompt = input("Enter your prompt: ")
+    use_rawdog_with_webai(user_prompt)
+```
+```shell
+python -m webscout.webai webai --provider "phind" --rawdog
+```
+### for using as terminal gpt
+```python
+python -m webscout.webai webai --provider "sean"
+```

{webscout-1.3.3.dist-info → webscout-1.3.4.dist-info}/RECORD

@@ -6,32 +6,32 @@ DeepWEBS/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 DeepWEBS/networks/filepath_converter.py,sha256=JKMBew1TYe4TVoGTqgTWerq2Pam49_9u9TVUFCTDQyk,3183
 DeepWEBS/networks/google_searcher.py,sha256=-AdIpVkRgemsARnOt8WPkF2Id1baVlqDHyqX2qz8Aew,1966
 DeepWEBS/networks/network_configs.py,sha256=-Hb78_7SBx32h219FnU14qcHTvBdDUf_QAU6-RTL_e0,726
-DeepWEBS/networks/webpage_fetcher.py,sha256=d5paDTB3wa_w6YWmLV7RkpAj8Lh8ztuUuyfe8RuTjQg,3846
+DeepWEBS/networks/webpage_fetcher.py,sha256=vRB9T3o-nMgrMkG2NPHTDctNeXaPSKCmBXqu189h2ZI,3590
 DeepWEBS/utilsdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/utilsdw/enver.py,sha256=vstxg_5P3Rwo1en6oPcuc2SBiATJqxi4C7meGmw5w0M,1754
+DeepWEBS/utilsdw/enver.py,sha256=vpI7s4_o_VL9govSryOv-z1zYK3pTEW3-H9QNN8JYtc,2472
 DeepWEBS/utilsdw/logger.py,sha256=Z0nFUcEGyU8r28yKiIyvEtO26xxpmJgbvNToTfwZecc,8174
 webscout/AI.py,sha256=9ZEctvGx558mmCva6c9lB5c0pbU-of5azle2F4Mpqhg,93054
 webscout/AIbase.py,sha256=vQi2ougu5bG-QdmoYmxCQsOg7KTEgG7EF6nZh5qqUGw,2343
-webscout/AIutel.py,sha256=j8NY4AgJbq3YLosX2R3QShtmjIM0UivXS_iCoMSIZiY,24126
+webscout/AIutel.py,sha256=rVbQwPSqrsUc0vZDMn2bXO6n0w479QUOljBYFRNH76Q,24413
 webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
 webscout/HelpingAI.py,sha256=YeZw0zYVHMcBFFPNdd3_Ghpm9ebt_EScQjHO_IIs4lg,8103
 webscout/LLM.py,sha256=CiDz0okZNEoXuxMwadZnwRGSLpqk2zg0vzvXSxQZjcE,1910
-webscout/__init__.py,sha256=rUXt0AOHdvrt53bAswMWToZ2bvqQ4of93xKd4xGl0uY,957
+webscout/__init__.py,sha256=0RRjdP5ZM25pFJW8RslilZyQOLZtUUIjL-NqTJjePZA,1002
 webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
 webscout/cli.py,sha256=F888fdrFUQgczMBN4yMOSf6Nh-IbvkqpPhDsbnA2FtQ,17059
 webscout/exceptions.py,sha256=4AOO5wexeL96nvUS-badcckcwrPS7UpZyAgB9vknHZE,276
-webscout/g4f.py,sha256=9ovf-gLMDC3Bt6zGrwmZ3_PJh5fSVR4ipOlsaYxbgU0,16358
+webscout/g4f.py,sha256=NEZbXOoVfmHiKcSjVpBMNKZzHgbTJLsd8xOXjtn4js4,16358
 webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
 webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
 webscout/utils.py,sha256=c_98M4oqpb54pUun3fpGGlCerFD6ZHUbghyp5b7Mwgo,2605
-webscout/version.py,sha256=EBB5uHyxxWvfz74zmklK5pUUb8DUYMAu8yaJy9KwDEU,25
+webscout/version.py,sha256=Bw-PmcKUlfRVmyOF25uf1CLYn5NfqviwtSLQ0KmE2W8,25
 webscout/voice.py,sha256=1Ids_2ToPBMX0cH_UyPMkY_6eSE9H4Gazrl0ujPmFag,941
-webscout/webai.py,sha256=DdBuz2B_V1M-OIt6dPAn5LV0eNFBWn2hTP2cJy2qklI,73868
+webscout/webai.py,sha256=ApxRQ6Us7-u_6kBACb1RSXbBFNEPHQhVdQobvns1Qkw,73868
 webscout/webscout_search.py,sha256=3_lli-hDb8_kCGwscK29xuUcOS833ROgpNhDzrxh0dk,3085
 webscout/webscout_search_async.py,sha256=Y5frH0k3hLqBCR-8dn7a_b7EvxdYxn6wHiKl3jWosE0,40670
-webscout-1.3.3.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
-webscout-1.3.3.dist-info/METADATA,sha256=VguyWlPTREI2JlEi3pMK1CPD1-JJjGabRqYmdxSd73k,30440
-webscout-1.3.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-webscout-1.3.3.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
-webscout-1.3.3.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
-webscout-1.3.3.dist-info/RECORD,,
+webscout-1.3.4.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
+webscout-1.3.4.dist-info/METADATA,sha256=t6lU_0POPrMQnuCKyA04SjkhxKS0cMtHr0-oYW-2cPg,31961
+webscout-1.3.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+webscout-1.3.4.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
+webscout-1.3.4.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
+webscout-1.3.4.dist-info/RECORD,,