webscout 1.2.4__tar.gz → 1.2.6__tar.gz

{webscout-1.2.4 → webscout-1.2.6}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 1.2.4
-Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
+Version: 1.2.6
+Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
 Author: OEvortex
 Author-email: helpingai5@gmail.com
 License: HelpingAI Simplified Universal License
@@ -45,7 +45,6 @@ Requires-Dist: sse_starlette
 Requires-Dist: termcolor
 Requires-Dist: tiktoken
 Requires-Dist: tldextract
-Requires-Dist: gpt4all
 Requires-Dist: orjson
 Provides-Extra: dev
 Requires-Dist: ruff>=0.1.6; extra == "dev"
@@ -57,8 +56,7 @@ Requires-Dist: pytest>=7.4.2; extra == "dev"
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
-Also containes AI models that you can use
+Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
 
 
 ## Table of Contents
@@ -69,6 +67,7 @@ Also containes AI models that you can use
   - [CLI version of webscout.AI](#cli-version-of-webscoutai)
   - [CLI to use LLM](#cli-to-use-llm)
   - [Regions](#regions)
+  - [Transcriber](#transcriber)
   - [DeepWEBS: Advanced Web Searches](#deepwebs-advanced-web-searches)
     - [Activating DeepWEBS](#activating-deepwebs)
     - [Point to remember before using `DeepWEBS`](#point-to-remember-before-using-deepwebs)
@@ -94,7 +93,7 @@ Also containes AI models that you can use
     - [6. `BlackBox` - Search/chat With BlackBox](#6-blackbox---searchchat-with-blackbox)
     - [7. `PERPLEXITY` - Search With PERPLEXITY](#7-perplexity---search-with-perplexity)
     - [8. `OpenGPT` - chat With OPENGPT](#8-opengpt---chat-with-opengpt)
-    - [9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout](#9-gpt4all---chat-offline-with-language-models-using-gpt4all-from-webscout)
+    - [9. `KOBOLDIA` -](#9-koboldia--)
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
 
@@ -219,7 +218,55 @@ ___
 [Go To TOP](#TOP)
 
 
+## Transcriber
+The transcriber function in webscout is a handy tool that transcribes YouTube videos. Here's an example code demonstrating its usage:
+```python
+import sys
+from webscout import transcriber
+
+def extract_transcript(video_id):
+    """Extracts the transcript from a YouTube video."""
+    try:
+        transcript_list = transcriber.list_transcripts(video_id)
+        for transcript in transcript_list:
+            transcript_text_list = transcript.fetch()
+            lang = transcript.language
+            transcript_text = ""
+            if transcript.language_code == 'en':
+                for line in transcript_text_list:
+                    transcript_text += " " + line["text"]
+                return transcript_text
+            elif transcript.is_translatable:
+                english_transcript_list = transcript.translate('en').fetch()
+                for line in english_transcript_list:
+                    transcript_text += " " + line["text"]
+                return transcript_text
+        print("Transcript extraction failed. Please check the video URL.")
+    except Exception as e:
+        print(f"Error: {e}")
 
+def main():
+    video_url = input("Enter the video link: ")
+
+    if video_url:
+        video_id = video_url.split("=")[1]
+        print("Video URL:", video_url)
+        submit = input("Press 'Enter' to get the transcript or type 'exit' to quit: ")
+        if submit == '':
+            print("Extracting Transcript...")
+            transcript = extract_transcript(video_id)
+            print('Transcript:')
+            print(transcript)
+            print("__________________________________________________________________________________")
+        elif submit.lower() == 'exit':
+            print("Exiting...")
+            sys.exit()
+        else:
+            print("Invalid input. Please try again.")
+
+if __name__ == "__main__":
+    main()
+```
 ## DeepWEBS: Advanced Web Searches
 
 `DeepWEBS` is a standalone feature designed to perform advanced web searches with enhanced capabilities. It is particularly powerful in extracting relevant information directly from webpages and Search engine, focusing exclusively on text (web) searches. Unlike the `WEBS` , which provides a broader range of search functionalities, `DeepWEBS` is specifically tailored for in-depth web searches.
@@ -608,36 +655,23 @@ prompt = "tell me about india"
 response_str = opengpt.chat(prompt)
 print(response_str)
 ```
-### 9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout
+### 9. `KOBOLDIA` - 
 ```python
-from webscout import GPT4ALL
+from webscout.AI import KOBOLDAI
 
-# Initialize the GPT4ALL class with your model path and other optional parameters
-gpt4all_instance = GPT4ALL(
-    model="path/to/your/model/file", # Replace with the actual path to your model file
-    is_conversation=True,
-    max_tokens=800,
-    temperature=0.7,
-    presence_penalty=0,
-    frequency_penalty=1.18,
-    top_p=0.4,
-    intro="Hello, how can I assist you today?",
-    filepath="path/to/conversation/history/file", # Optional, for conversation history
-    update_file=True,
-    history_offset=10250,
-    act=None # Optional, for using an awesome prompt as intro
-)
+# Instantiate the KOBOLDAI class with default parameters
+koboldai = KOBOLDAI()
 
-# Generate a response from the AI model
-response = gpt4all_instance.chat(
-    prompt="What is the weather like today?",
-    stream=False, # Set to True if you want to stream the response
-    optimizer=None, # Optional, specify an optimizer if needed
-    conversationally=False # Set to True for conversationally generated responses
-)
+# Define a prompt to send to the AI
+prompt = "What is the capital of France?"
+
+# Use the 'ask' method to get a response from the AI
+response = koboldai.ask(prompt)
+
+# Extract and print the message from the response
+message = koboldai.get_message(response)
+print(message)
 
-# Print the generated response
-print(response)
 ```
 
 ## usage of special .LLM file from webscout (webscout.LLM)

{webscout-1.2.4 → webscout-1.2.6}/README.md

@@ -4,8 +4,7 @@
 <a href="#"><img alt="Python version" src="https://img.shields.io/pypi/pyversions/webscout"/></a>
 <a href="https://pepy.tech/project/webscout"><img alt="Downloads" src="https://static.pepy.tech/badge/webscout"></a>
 
-Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
-Also containes AI models that you can use
+Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
 
 
 ## Table of Contents
@@ -16,6 +15,7 @@ Also containes AI models that you can use
   - [CLI version of webscout.AI](#cli-version-of-webscoutai)
   - [CLI to use LLM](#cli-to-use-llm)
   - [Regions](#regions)
+  - [Transcriber](#transcriber)
   - [DeepWEBS: Advanced Web Searches](#deepwebs-advanced-web-searches)
     - [Activating DeepWEBS](#activating-deepwebs)
     - [Point to remember before using `DeepWEBS`](#point-to-remember-before-using-deepwebs)
@@ -41,7 +41,7 @@ Also containes AI models that you can use
     - [6. `BlackBox` - Search/chat With BlackBox](#6-blackbox---searchchat-with-blackbox)
     - [7. `PERPLEXITY` - Search With PERPLEXITY](#7-perplexity---search-with-perplexity)
     - [8. `OpenGPT` - chat With OPENGPT](#8-opengpt---chat-with-opengpt)
-    - [9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout](#9-gpt4all---chat-offline-with-language-models-using-gpt4all-from-webscout)
+    - [9. `KOBOLDIA` -](#9-koboldia--)
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
 
@@ -166,7 +166,55 @@ ___
 [Go To TOP](#TOP)
 
 
+## Transcriber
+The transcriber function in webscout is a handy tool that transcribes YouTube videos. Here's an example code demonstrating its usage:
+```python
+import sys
+from webscout import transcriber
+
+def extract_transcript(video_id):
+    """Extracts the transcript from a YouTube video."""
+    try:
+        transcript_list = transcriber.list_transcripts(video_id)
+        for transcript in transcript_list:
+            transcript_text_list = transcript.fetch()
+            lang = transcript.language
+            transcript_text = ""
+            if transcript.language_code == 'en':
+                for line in transcript_text_list:
+                    transcript_text += " " + line["text"]
+                return transcript_text
+            elif transcript.is_translatable:
+                english_transcript_list = transcript.translate('en').fetch()
+                for line in english_transcript_list:
+                    transcript_text += " " + line["text"]
+                return transcript_text
+        print("Transcript extraction failed. Please check the video URL.")
+    except Exception as e:
+        print(f"Error: {e}")
 
+def main():
+    video_url = input("Enter the video link: ")
+
+    if video_url:
+        video_id = video_url.split("=")[1]
+        print("Video URL:", video_url)
+        submit = input("Press 'Enter' to get the transcript or type 'exit' to quit: ")
+        if submit == '':
+            print("Extracting Transcript...")
+            transcript = extract_transcript(video_id)
+            print('Transcript:')
+            print(transcript)
+            print("__________________________________________________________________________________")
+        elif submit.lower() == 'exit':
+            print("Exiting...")
+            sys.exit()
+        else:
+            print("Invalid input. Please try again.")
+
+if __name__ == "__main__":
+    main()
+```
 ## DeepWEBS: Advanced Web Searches
 
 `DeepWEBS` is a standalone feature designed to perform advanced web searches with enhanced capabilities. It is particularly powerful in extracting relevant information directly from webpages and Search engine, focusing exclusively on text (web) searches. Unlike the `WEBS` , which provides a broader range of search functionalities, `DeepWEBS` is specifically tailored for in-depth web searches.
@@ -555,36 +603,23 @@ prompt = "tell me about india"
 response_str = opengpt.chat(prompt)
 print(response_str)
 ```
-### 9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout
+### 9. `KOBOLDIA` - 
 ```python
-from webscout import GPT4ALL
+from webscout.AI import KOBOLDAI
 
-# Initialize the GPT4ALL class with your model path and other optional parameters
-gpt4all_instance = GPT4ALL(
-    model="path/to/your/model/file", # Replace with the actual path to your model file
-    is_conversation=True,
-    max_tokens=800,
-    temperature=0.7,
-    presence_penalty=0,
-    frequency_penalty=1.18,
-    top_p=0.4,
-    intro="Hello, how can I assist you today?",
-    filepath="path/to/conversation/history/file", # Optional, for conversation history
-    update_file=True,
-    history_offset=10250,
-    act=None # Optional, for using an awesome prompt as intro
-)
+# Instantiate the KOBOLDAI class with default parameters
+koboldai = KOBOLDAI()
 
-# Generate a response from the AI model
-response = gpt4all_instance.chat(
-    prompt="What is the weather like today?",
-    stream=False, # Set to True if you want to stream the response
-    optimizer=None, # Optional, specify an optimizer if needed
-    conversationally=False # Set to True for conversationally generated responses
-)
+# Define a prompt to send to the AI
+prompt = "What is the capital of France?"
+
+# Use the 'ask' method to get a response from the AI
+response = koboldai.ask(prompt)
+
+# Extract and print the message from the response
+message = koboldai.get_message(response)
+print(message)
 
-# Print the generated response
-print(response)
 ```
 
 ## usage of special .LLM file from webscout (webscout.LLM)

{webscout-1.2.4 → webscout-1.2.6}/setup.py

@@ -9,8 +9,8 @@ with open("README.md", encoding="utf-8") as f:
 
 setup(
     name="webscout",
-    version="1.2.4", # Use the version variable from the version.py file
-    description="Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models",
+    version="1.2.6", 
+    description="Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos",
     long_description=README,
     long_description_content_type="text/markdown",
     author="OEvortex",
@@ -53,7 +53,6 @@ setup(
         "termcolor",
         "tiktoken",
         "tldextract",
-        "gpt4all",
         "orjson",
     ],
     entry_points={

{webscout-1.2.4 → webscout-1.2.6}/webscout/AI.py

@@ -21,9 +21,199 @@ import yaml
 from webscout.AIutel import Optimizers
 from webscout.AIutel import Conversation
 from webscout.AIutel import AwesomePrompts
+from webscout.AIbase import Provider
 from Helpingai_T2 import Perplexity
 from typing import Any
 import logging
+#------------------------------------------------------KOBOLDAI-----------------------------------------------------------
+class KOBOLDAI(Provider):
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 600,
+        temperature: float = 1,
+        top_p: float = 1,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+    ):
+        """Instantiate TGPT
+
+        Args:
+            is_conversation (str, optional): Flag for chatting conversationally. Defaults to True.
+            max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
+            temperature (float, optional): Charge of the generated text's randomness. Defaults to 0.2.
+            top_p (float, optional): Sampling threshold during inference time. Defaults to 0.999.
+            timeout (int, optional): Http requesting timeout. Defaults to 30
+            intro (str, optional): Conversation introductory prompt. Defaults to `Conversation.intro`.
+            filepath (str, optional): Path to file containing conversation history. Defaults to None.
+            update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
+            proxies (dict, optional) : Http reqiuest proxies (socks). Defaults to {}.
+            history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
+            act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
+        """
+        self.session = requests.Session()
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.temperature = temperature
+        self.top_p = top_p
+        self.chat_endpoint = (
+            "https://koboldai-koboldcpp-tiefighter.hf.space/api/extra/generate/stream"
+        )
+        self.stream_chunk_size = 64
+        self.timeout = timeout
+        self.last_response = {}
+        self.headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        self.session.headers.update(self.headers)
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> dict:
+        """Chat with AI
+
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            raw (bool, optional): Stream back raw response as received. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+           dict : {}
+        ```json
+        {
+           "token" : "How may I assist you today?"
+        }
+        ```
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        self.session.headers.update(self.headers)
+        payload = {
+            "prompt": conversation_prompt,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+        }
+
+        def for_stream():
+            response = self.session.post(
+                self.chat_endpoint, json=payload, stream=True, timeout=self.timeout
+            )
+            if not response.ok:
+                raise Exception(
+                    f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
+                )
+
+            message_load = ""
+            for value in response.iter_lines(
+                decode_unicode=True,
+                delimiter="" if raw else "event: message\ndata:",
+                chunk_size=self.stream_chunk_size,
+            ):
+                try:
+                    resp = json.loads(value)
+                    message_load += self.get_message(resp)
+                    resp["token"] = message_load
+                    self.last_response.update(resp)
+                    yield value if raw else resp
+                except json.decoder.JSONDecodeError:
+                    pass
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+
+        def for_non_stream():
+            # let's make use of stream
+            for _ in for_stream():
+                pass
+            return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """Generate response `str`
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+            str: Response generated
+        """
+
+        def for_stream():
+            for response in self.ask(
+                prompt, True, optimizer=optimizer, conversationally=conversationally
+            ):
+                yield self.get_message(response)
+
+        def for_non_stream():
+            return self.get_message(
+                self.ask(
+                    prompt,
+                    False,
+                    optimizer=optimizer,
+                    conversationally=conversationally,
+                )
+            )
+
+        return for_stream() if stream else for_non_stream()
+
+    def get_message(self, response: dict) -> str:
+        """Retrieves message only from response
+
+        Args:
+            response (dict): Response generated by `self.ask`
+
+        Returns:
+            str: Message extracted
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response.get("token")
 #------------------------------------------------------OpenGPT-----------------------------------------------------------
 class OPENGPT:
     def __init__(
@@ -1237,5 +1427,19 @@ def opengpt(prompt, stream):
     else:
         response_str = opengpt.chat(prompt)
         print(response_str)
+        
+@cli.command()
+@click.option('--prompt', prompt='Enter your prompt', help='The prompt to send.')
+@click.option('--stream', is_flag=True, help='Flag for streaming response.')
+@click.option('--raw', is_flag=True, help='Stream back raw response as received.')
+@click.option('--optimizer', type=str, help='Prompt optimizer name.')
+@click.option('--conversationally', is_flag=True, help='Chat conversationally when using optimizer.')
+def koboldai_cli(prompt, stream, raw, optimizer, conversationally):
+    """Chat with KOBOLDAI using the provided prompt."""
+    koboldai_instance = KOBOLDAI() # Initialize a KOBOLDAI instance
+    response = koboldai_instance.ask(prompt, stream, raw, optimizer, conversationally)
+    processed_response = koboldai_instance.get_message(response) # Process the response
+    print(processed_response)
+    
 if __name__ == '__main__':
     cli()

{webscout-1.2.4 → webscout-1.2.6}/webscout/AIutel.py

@@ -11,7 +11,7 @@ import click
 from rich.markdown import Markdown
 from rich.console import Console
 
-appdir = appdirs.AppDirs("pytgpt", "Smartwa")
+appdir = appdirs.AppDirs("AIWEBS", "vortex")
 
 default_path = appdir.user_cache_dir
 

{webscout-1.2.4 → webscout-1.2.6}/webscout/__init__.py

@@ -1,7 +1,7 @@
 """Webscout.
 
 Search for words, documents, images, videos, news, maps and text translation 
-using the DuckDuckGo.com search engine.
+using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
 """
 
 import logging
@@ -9,7 +9,10 @@ from .webscout_search import WEBS
 from .webscout_search_async import AsyncWEBS
 from .version import __version__
 from .DWEBS import DeepWEBS
-from .offlineAI import GPT4ALL
+from .AIutel import appdir
+from .transcriber import transcriber
+
+
 __all__ = ["WEBS", "AsyncWEBS", "__version__", "cli"]
 
 logging.getLogger("webscout").addHandler(logging.NullHandler())