webscout 1.3.3__tar.gz → 1.3.4__tar.gz

{webscout-1.3.3 → webscout-1.3.4}/DeepWEBS/networks/webpage_fetcher.py

@@ -78,20 +78,18 @@ class BatchWebpageFetcher:
         self.urls = urls
         self.total_count = len(self.urls)
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+        with concurrent.futures.ProcessPoolExecutor() as executor:
             futures = [
-                executor.submit(self.fetch_single_webpage, url, overwrite, output_parent)
+                executor.submit(WebpageFetcher().fetch, url, overwrite, output_parent)
                 for url in urls
             ]
             concurrent.futures.wait(futures)
 
+        self.url_and_html_path_list = [
+            {"url": future.result().url, "html_path": str(future.result().html_path)}
+            for future in futures
+        ]
+
         return self.url_and_html_path_list
 
-if __name__ == "__main__":
-    urls = [
-        "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename",
-        "https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528",
-        "https://docs.python.org/zh-cn/3/tutorial/interpreter.html",
-    ]
-    batch_webpage_fetcher = BatchWebpageFetcher()
-    batch_webpage_fetcher.fetch(urls=urls, overwrite=True, output_parent="python tutorials")
+

webscout-1.3.4/DeepWEBS/utilsdw/enver.py

@@ -0,0 +1,78 @@
+import json
+import os
+from pathlib import Path
+from typing import Dict, Optional
+
+from DeepWEBS.utilsdw.logger import OSLogger
+
+
+class OSEnver:
+    """Manages the OS environment variables."""
+
+    def __init__(self) -> None:
+        """Initializes the OSEnver object."""
+        self.envs_stack: list[Dict[str, str]] = []
+        self.envs: Dict[str, str] = os.environ.copy()
+
+    def store_envs(self) -> None:
+        """Stores a copy of the current environment variables on a stack."""
+        self.envs_stack.append(self.envs.copy())
+
+    def restore_envs(self) -> None:
+        """Restores environment variables from the top of the stack."""
+        self.envs = self.envs_stack.pop()
+
+    def set_envs(
+        self,
+        secrets: bool = True,
+        proxies: Optional[str] = None,
+        store_envs: bool = True,
+    ) -> None:
+        """Sets environment variables based on the contents of secrets.json.
+
+        Args:
+            secrets (bool): Whether to load secrets from secrets.json.
+            proxies (Optional[str]): Proxy URL to set as environment variable.
+            store_envs (bool): Whether to store a copy of the environment variables
+                on the stack.
+        """
+        if store_envs:
+            self.store_envs()
+
+        if secrets:
+            secrets_path = Path(__file__).parents[1] / "secrets.json"
+            if secrets_path.exists():
+                with open(secrets_path, "r") as rf:
+                    secrets = json.load(rf)
+            else:
+                secrets = {}
+
+        if proxies:
+            for proxy_env in ["http_proxy", "https_proxy"]:
+                if isinstance(proxies, str):
+                    self.envs[proxy_env] = proxies
+                elif "http_proxy" in secrets.keys():
+                    self.envs[proxy_env] = secrets["http_proxy"]
+                elif os.getenv("http_proxy"):
+                    self.envs[proxy_env] = os.getenv("http_proxy")
+                else:
+                    continue
+
+        self.proxy = (
+            self.envs.get("all_proxy")
+            or self.envs.get("http_proxy")
+            or self.envs.get("https_proxy")
+            or None
+        )
+        self.requests_proxies = {
+            "http": self.proxy,
+            "https": self.proxy,
+        }
+
+        if self.proxy:
+            OSLogger().note(f"Using proxy: [{self.proxy}]")
+
+
+enver: OSEnver = OSEnver()
+
+

{webscout-1.3.3 → webscout-1.3.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 1.3.3
-Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos and have TTS support
+Version: 1.3.4
+Summary: Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 Author: OEvortex
 Author-email: helpingai5@gmail.com
 License: HelpingAI Simplified Universal License
@@ -56,7 +56,7 @@ Requires-Dist: pytest>=7.4.2; extra == "dev"
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 
 
 ## Table of Contents
@@ -100,6 +100,8 @@ Search for words, documents, images, videos, news, maps and text translation usi
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
     - [`LLM` with internet](#llm-with-internet)
+  - [`Webai` - terminal gpt and a open interpeter](#webai---terminal-gpt-and-a-open-interpeter)
+    - [for using as terminal gpt](#for-using-as-terminal-gpt)
 
 ## Install
 ```python
@@ -801,3 +803,49 @@ if __name__ == "__main__":
         else:
             print("No response")
 ```
+## `Webai` - terminal gpt and a open interpeter
+
+```python
+from webscout.webai import Main
+
+def use_rawdog_with_webai(prompt):
+    """
+    Wrap the webscout default method in a try-except block to catch any unhandled
+    exceptions and print a helpful message.
+    """
+    try:
+        webai_bot = Main(
+            max_tokens=500, 
+            provider="phind",
+            temperature=0.7,  
+            top_k=40,          
+            top_p=0.95,        
+            model="Phind Model",  # Replace with your desired model
+            auth=None,       # Replace with your auth key/value (if needed)
+            timeout=30,
+            disable_conversation=True,
+            filepath=None,
+            update_file=True,
+            intro=None,
+            rawdog=True,
+            history_offset=10250,
+            awesome_prompt=None,
+            proxy_path=None,
+            quiet=True
+        )
+        webai_response = webai_bot.default(prompt) 
+    except Exception as e:
+        print("Unexpected error:", e)
+
+
+if __name__ == "__main__":
+    user_prompt = input("Enter your prompt: ")
+    use_rawdog_with_webai(user_prompt)
+```
+```shell
+python -m webscout.webai webai --provider "phind" --rawdog
+```
+### for using as terminal gpt
+```python
+python -m webscout.webai webai --provider "sean"
+```

{webscout-1.3.3 → webscout-1.3.4}/README.md

@@ -4,7 +4,7 @@
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 
 
 ## Table of Contents
@@ -48,6 +48,8 @@ Search for words, documents, images, videos, news, maps and text translation usi
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
     - [`LLM` with internet](#llm-with-internet)
+  - [`Webai` - terminal gpt and a open interpeter](#webai---terminal-gpt-and-a-open-interpeter)
+    - [for using as terminal gpt](#for-using-as-terminal-gpt)
 
 ## Install
 ```python
@@ -748,4 +750,50 @@ if __name__ == "__main__":
             print("AI:", response)
         else:
             print("No response")
+```
+## `Webai` - terminal gpt and a open interpeter
+
+```python
+from webscout.webai import Main
+
+def use_rawdog_with_webai(prompt):
+    """
+    Wrap the webscout default method in a try-except block to catch any unhandled
+    exceptions and print a helpful message.
+    """
+    try:
+        webai_bot = Main(
+            max_tokens=500, 
+            provider="phind",
+            temperature=0.7,  
+            top_k=40,          
+            top_p=0.95,        
+            model="Phind Model",  # Replace with your desired model
+            auth=None,       # Replace with your auth key/value (if needed)
+            timeout=30,
+            disable_conversation=True,
+            filepath=None,
+            update_file=True,
+            intro=None,
+            rawdog=True,
+            history_offset=10250,
+            awesome_prompt=None,
+            proxy_path=None,
+            quiet=True
+        )
+        webai_response = webai_bot.default(prompt) 
+    except Exception as e:
+        print("Unexpected error:", e)
+
+
+if __name__ == "__main__":
+    user_prompt = input("Enter your prompt: ")
+    use_rawdog_with_webai(user_prompt)
+```
+```shell
+python -m webscout.webai webai --provider "phind" --rawdog
+```
+### for using as terminal gpt
+```python
+python -m webscout.webai webai --provider "sean"
 ```

{webscout-1.3.3 → webscout-1.3.4}/setup.py

@@ -5,8 +5,8 @@ with open("README.md", encoding="utf-8") as f:
 
 setup(
     name="webscout",
-    version="1.3.3", 
-    description="Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos and have TTS support",
+    version="1.3.4", 
+    description="Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support",
     long_description=README,
     long_description_content_type="text/markdown",
     author="OEvortex",

{webscout-1.3.3 → webscout-1.3.4}/webscout/AIutel.py

@@ -10,14 +10,31 @@ import sys
 import click
 from rich.markdown import Markdown
 from rich.console import Console
-
+import g4f
 appdir = appdirs.AppDirs("AIWEBS", "vortex")
 
 default_path = appdir.user_cache_dir
 
 if not os.path.exists(default_path):
     os.makedirs(default_path)
-
+webai = [
+    "leo",
+    "openai",
+    "opengpt",
+    "koboldai",
+    "gemini",
+    "phind",
+    "blackboxai",
+    "g4fauto",
+    "perplexity",
+    "sean",
+]
+
+gpt4free_providers = [
+    provider.__name__ for provider in g4f.Provider.__providers__  # if provider.working
+]
+
+available_providers = webai + gpt4free_providers
 
 def run_system_command(
     command: str,
@@ -468,7 +485,6 @@ print("The essay is about...")
 ```
 """
 
-    # Idea borrowed from https://github.com/AbanteAI/rawdog
 
     def __init__(
         self,

{webscout-1.3.3 → webscout-1.3.4}/webscout/__init__.py

@@ -1,7 +1,6 @@
 """Webscout.
 
-Search for words, documents, images, videos, news, maps and text translation 
-using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 """
 import g4f
 import logging

{webscout-1.3.3 → webscout-1.3.4}/webscout/g4f.py

@@ -2,7 +2,7 @@ import g4f
 from webscout.AIutel import Optimizers
 from webscout.AIutel import Conversation
 from webscout.AIutel import AwesomePrompts
-from webscout.AIutel import Provider
+from webscout.AIbase import Provider
 from webscout.AIutel import available_providers
 
 

webscout-1.3.4/webscout/version.py

@@ -0,0 +1,2 @@
+__version__ = "1.3.4"
+

{webscout-1.3.3 → webscout-1.3.4}/webscout/webai.py

@@ -29,6 +29,7 @@ from webscout.AIutel import Optimizers
 from webscout.AIutel import default_path
 from webscout.AIutel import AwesomePrompts
 from webscout.AIutel import RawDog
+from webscout import available_providers
 from colorama import Fore
 from colorama import init as init_colorama
 from dotenv import load_dotenv
@@ -41,7 +42,7 @@ init_colorama(autoreset=True)
 load_dotenv()  # loads .env variables
 
 logging.basicConfig(
-    format="%(asctime)s - %(levelname)s : %(message)s ",  # [%(module)s,%(lineno)s]", # for debug purposes
+    format="%(asctime)s - %(levelname)s : %(message)s ",  
     datefmt="%H:%M:%S",
     level=logging.INFO,
 )
@@ -61,7 +62,7 @@ class this:
 
     rich_code_themes = ["monokai", "paraiso-dark", "igor", "vs", "fruity", "xcode"]
 
-    default_provider = "sean"
+    default_provider = "phind"
 
     getExc = lambda e: e.args[1] if len(e.args) > 1 else str(e)
 
@@ -1077,7 +1078,7 @@ class EntryGroup:
         pass
 
 
-
+import webscout
 class Chatwebai:
     """webai command"""
 
@@ -1194,7 +1195,7 @@ class Chatwebai:
     @click.option(
         "-p",
         "--provider",
-        type=click.Choice(webscout.available_providers),
+        type=click.Choice(available_providers),
         default=this.default_provider,
         help="Name of LLM provider.",
         metavar=(

{webscout-1.3.3 → webscout-1.3.4}/webscout.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 1.3.3
-Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos and have TTS support
+Version: 1.3.4
+Summary: Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 Author: OEvortex
 Author-email: helpingai5@gmail.com
 License: HelpingAI Simplified Universal License
@@ -56,7 +56,7 @@ Requires-Dist: pytest>=7.4.2; extra == "dev"
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
+Search for anything using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models, can transcribe yt videos, have TTS support and now has webai(terminal gpt and open interpeter) support
 
 
 ## Table of Contents
@@ -100,6 +100,8 @@ Search for words, documents, images, videos, news, maps and text translation usi
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
     - [`LLM` with internet](#llm-with-internet)
+  - [`Webai` - terminal gpt and a open interpeter](#webai---terminal-gpt-and-a-open-interpeter)
+    - [for using as terminal gpt](#for-using-as-terminal-gpt)
 
 ## Install
 ```python
@@ -801,3 +803,49 @@ if __name__ == "__main__":
         else:
             print("No response")
 ```
+## `Webai` - terminal gpt and a open interpeter
+
+```python
+from webscout.webai import Main
+
+def use_rawdog_with_webai(prompt):
+    """
+    Wrap the webscout default method in a try-except block to catch any unhandled
+    exceptions and print a helpful message.
+    """
+    try:
+        webai_bot = Main(
+            max_tokens=500, 
+            provider="phind",
+            temperature=0.7,  
+            top_k=40,          
+            top_p=0.95,        
+            model="Phind Model",  # Replace with your desired model
+            auth=None,       # Replace with your auth key/value (if needed)
+            timeout=30,
+            disable_conversation=True,
+            filepath=None,
+            update_file=True,
+            intro=None,
+            rawdog=True,
+            history_offset=10250,
+            awesome_prompt=None,
+            proxy_path=None,
+            quiet=True
+        )
+        webai_response = webai_bot.default(prompt) 
+    except Exception as e:
+        print("Unexpected error:", e)
+
+
+if __name__ == "__main__":
+    user_prompt = input("Enter your prompt: ")
+    use_rawdog_with_webai(user_prompt)
+```
+```shell
+python -m webscout.webai webai --provider "phind" --rawdog
+```
+### for using as terminal gpt
+```python
+python -m webscout.webai webai --provider "sean"
+```

webscout-1.3.3/DeepWEBS/utilsdw/enver.py

@@ -1,60 +0,0 @@
-import json
-import os
-
-from pathlib import Path
-from DeepWEBS.utilsdw.logger import logger
-
-
-class OSEnver:
-    def __init__(self):
-        self.envs_stack = []
-        self.envs = os.environ.copy()
-
-    def store_envs(self):
-        self.envs_stack.append(self.envs)
-
-    def restore_envs(self):
-        self.envs = self.envs_stack.pop()
-
-    def set_envs(self, secrets=True, proxies=None, store_envs=True):
-        # caller_info = inspect.stack()[1]
-        # logger.back(f"OS Envs is set by: {caller_info.filename}")
-
-        if store_envs:
-            self.store_envs()
-
-        if secrets:
-            secrets_path = Path(__file__).parents[1] / "secrets.json"
-            if secrets_path.exists():
-                with open(secrets_path, "r") as rf:
-                    secrets = json.load(rf)
-            else:
-                secrets = {}
-
-        if proxies:
-            for proxy_env in ["http_proxy", "https_proxy"]:
-                if isinstance(proxies, str):
-                    self.envs[proxy_env] = proxies
-                elif "http_proxy" in secrets.keys():
-                    self.envs[proxy_env] = secrets["http_proxy"]
-                elif os.getenv("http_proxy"):
-                    self.envs[proxy_env] = os.getenv("http_proxy")
-                else:
-                    continue
-
-        self.proxy = (
-            self.envs.get("all_proxy")
-            or self.envs.get("http_proxy")
-            or self.envs.get("https_proxy")
-            or None
-        )
-        self.requests_proxies = {
-            "http": self.proxy,
-            "https": self.proxy,
-        }
-
-        if self.proxy:
-            logger.note(f"Using proxy: [{self.proxy}]")
-
-
-enver = OSEnver()

webscout-1.3.3/webscout/version.py

@@ -1,2 +0,0 @@
-__version__ = "1.3.3"
-