webscout 6.3__py3-none-any.whl → 6.5__py3-none-any.whl

webscout/conversation.py

@@ -0,0 +1,227 @@
+"""
+>>> from conversation import Conversation
+>>> chat = Conversation(max_tokens=500)
+>>> chat.add_message("user", "Hello!")
+>>> chat.add_message("llm", "Hi there!")
+>>> prompt = chat.gen_complete_prompt("What's up?")
+"""
+
+import os
+import logging
+from typing import Optional
+from .Litlogger import LitLogger, LogFormat, ColorScheme
+
+# Create a logger instance for this module
+logger = LitLogger(
+    name="Conversation",
+    format=LogFormat.MODERN_EMOJI,
+    color_scheme=ColorScheme.CYBERPUNK
+)
+
+class Conversation:
+    """Handles prompt generation based on history and maintains chat context.
+    
+    This class is responsible for managing chat conversations, including:
+    - Maintaining chat history
+    - Loading/saving conversations from/to files
+    - Generating prompts based on context
+    - Managing token limits and history pruning
+    
+    Examples:
+        >>> chat = Conversation(max_tokens=500)
+        >>> chat.add_message("user", "Hello!")
+        >>> chat.add_message("llm", "Hi there!")
+        >>> prompt = chat.gen_complete_prompt("What's up?")
+    """
+
+    intro = (
+        "You're a Large Language Model for chatting with people. "
+        "Assume role of the LLM and give your response."
+    )
+
+    def __init__(
+        self,
+        status: bool = True,
+        max_tokens: int = 600,
+        filepath: Optional[str] = None,
+        update_file: bool = True,
+    ):
+        """Initialize a new Conversation manager.
+
+        Args:
+            status (bool): Flag to control history tracking. Defaults to True.
+            max_tokens (int): Maximum tokens for completion response. Defaults to 600.
+            filepath (str, optional): Path to save/load conversation history. Defaults to None.
+            update_file (bool): Whether to append new messages to file. Defaults to True.
+
+        Examples:
+            >>> chat = Conversation(max_tokens=500)
+            >>> chat = Conversation(filepath="chat_history.txt")
+        """
+        self.status = status
+        self.max_tokens_to_sample = max_tokens
+        self.chat_history = ""  # Initialize as empty string
+        self.history_format = "\nUser : %(user)s\nLLM :%(llm)s"
+        self.file = filepath
+        self.update_file = update_file
+        self.history_offset = 10250
+        self.prompt_allowance = 10
+        
+        if filepath:
+            self.load_conversation(filepath, False)
+
+    def load_conversation(self, filepath: str, exists: bool = True) -> None:
+        """Load conversation history from a text file.
+
+        Args:
+            filepath (str): Path to the history file
+            exists (bool): Flag for file availability. Defaults to True.
+
+        Raises:
+            AssertionError: If filepath is not str or file doesn't exist
+        """
+        assert isinstance(
+            filepath, str
+        ), f"Filepath needs to be of str datatype not {type(filepath)}"
+        assert (
+            os.path.isfile(filepath) if exists else True
+        ), f"File '{filepath}' does not exist"
+
+        if not os.path.isfile(filepath):
+            logging.debug(f"Creating new chat-history file - '{filepath}'")
+            with open(filepath, "w", encoding="utf-8") as fh:
+                fh.write(self.intro)
+        else:
+            logging.debug(f"Loading conversation from '{filepath}'")
+            with open(filepath, encoding="utf-8") as fh:
+                file_contents = fh.readlines()
+                if file_contents:
+                    self.intro = file_contents[0]  # First line is intro
+                    self.chat_history = "\n".join(file_contents[1:])
+    
+    def __trim_chat_history(self, chat_history: str, intro: str) -> str:
+        """Keep the chat history fresh by trimming it when it gets too long! 
+
+        This method makes sure we don't exceed our token limits by:
+        - Calculating total length (intro + history)
+        - Trimming older messages if needed
+        - Keeping the convo smooth and within limits
+
+        Args:
+            chat_history (str): The current chat history to trim
+            intro (str): The conversation's intro/system prompt
+
+        Returns:
+            str: The trimmed chat history, ready to use! 
+
+        Examples:
+            >>> chat = Conversation(max_tokens=500)
+            >>> trimmed = chat._Conversation__trim_chat_history("Hello! Hi!", "Intro")
+        """
+        len_of_intro = len(intro)
+        len_of_chat_history = len(chat_history)
+        total = self.max_tokens_to_sample + len_of_intro + len_of_chat_history
+
+        if total > self.history_offset:
+            truncate_at = (total - self.history_offset) + self.prompt_allowance
+            trimmed_chat_history = chat_history[truncate_at:]
+            return "... " + trimmed_chat_history
+        return chat_history
+
+    def gen_complete_prompt(self, prompt: str, intro: Optional[str] = None) -> str:
+        """Generate a complete prompt that's ready to go! 
+
+        This method:
+        - Combines the intro, history, and new prompt
+        - Trims history if needed
+        - Keeps everything organized and flowing
+
+        Args:
+            prompt (str): Your message to add to the chat
+            intro (str, optional): Custom intro to use. Default: None (uses class intro)
+
+        Returns:
+            str: The complete conversation prompt, ready for the LLM! 
+
+        Examples:
+            >>> chat = Conversation()
+            >>> prompt = chat.gen_complete_prompt("What's good?")
+        """
+        if not self.status:
+            return prompt
+
+        intro = intro or self.intro or (
+            "You're a Large Language Model for chatting with people. "
+            "Assume role of the LLM and give your response."
+        )
+        
+        incomplete_chat_history = self.chat_history + self.history_format % {
+            "user": prompt,
+            "llm": ""
+        }
+        return intro + self.__trim_chat_history(incomplete_chat_history, intro)
+
+    def update_chat_history(
+        self, prompt: str, response: str, force: bool = False
+    ) -> None:
+        """Keep the conversation flowing by updating the chat history! 
+
+        This method:
+        - Adds new messages to the history
+        - Updates the file if needed
+        - Keeps everything organized
+
+        Args:
+            prompt (str): Your message to add
+            response (str): The LLM's response
+            force (bool): Force update even if history is off. Default: False
+
+        Examples:
+            >>> chat = Conversation()
+            >>> chat.update_chat_history("Hi!", "Hello there!")
+        """
+        if not self.status and not force:
+            return
+
+        new_history = self.history_format % {"user": prompt, "llm": response}
+        
+        if self.file and self.update_file:
+            # Create file if it doesn't exist
+            if not os.path.exists(self.file):
+                with open(self.file, "w", encoding="utf-8") as fh:
+                    fh.write(self.intro + "\n")
+            
+            # Append new history
+            with open(self.file, "a", encoding="utf-8") as fh:
+                fh.write(new_history)
+        
+        self.chat_history += new_history
+
+    def add_message(self, role: str, content: str) -> None:
+        """Add a new message to the chat - simple and clean! 
+
+        This method:
+        - Validates the message role
+        - Adds the message to history
+        - Updates file if needed
+
+        Args:
+            role (str): Who's sending? ('user', 'llm', 'tool', or 'reasoning')
+            content (str): What's the message?
+
+        Examples:
+            >>> chat = Conversation()
+            >>> chat.add_message("user", "Hey there!")
+            >>> chat.add_message("llm", "Hi! How can I help?")
+        """
+        role_formats = {
+            "user": "User",
+            "llm": "LLM",
+            "tool": "Tool",
+            "reasoning": "Reasoning"
+        }
+
+        if role in role_formats:
+            self.chat_history += f"\n{role_formats[role]} : {content}"
+        else:
+            logger.warning(f"Unknown role '{role}' for message: {content}")

webscout/exceptions.py

@@ -1,29 +1,161 @@
-class WebscoutE(Exception):
-    """Base exception class for search."""
-
-
-class RatelimitE(Exception):
-    """Raised for rate limit exceeded errors during API requests."""
-
-class ConversationLimitException(Exception):
-    """Raised for conversation limit exceeded errors during API requests."""
-    pass
-class TimeoutE(Exception):
-    """Raised for timeout errors during API requests."""
-    
-class FailedToGenerateResponseError(Exception):
-    
-    """Provider failed to fetch response"""
-class AllProvidersFailure(Exception):
-    """None of the providers generated response successfully"""
-    pass
-
-class FacebookInvalidCredentialsException(Exception):
-    pass
-
-
-class FacebookRegionBlocked(Exception):
-    pass
-
-class ModelUnloadedException(Exception):
-    pass
+class WebscoutE(Exception):
+    """Base exception class for search."""
+
+
+class APIConnectionError(Exception):
+    """Raised when there are connection issues with an API."""
+    pass
+
+
+class RatelimitE(Exception):
+    """Raised for rate limit exceeded errors during API requests."""
+
+
+class ConversationLimitException(Exception):
+    """Raised for conversation limit exceeded errors during API requests."""
+    pass
+
+
+class TimeoutE(Exception):
+    """Raised for timeout errors during API requests."""
+
+
+class FailedToGenerateResponseError(Exception):
+    """Provider failed to fetch response"""
+
+
+class AllProvidersFailure(Exception):
+    """None of the providers generated response successfully"""
+    pass
+
+
+class FacebookInvalidCredentialsException(Exception):
+    pass
+
+
+class FacebookRegionBlocked(Exception):
+    pass
+
+
+class ModelUnloadedException(Exception):
+    pass
+
+
+class TranscriptRetrievalError(WebscoutE):
+    """Base class for transcript retrieval errors."""
+
+    def __init__(self, video_id, message):
+        super().__init__(message.format(video_url=WATCH_URL.format(video_id=video_id)))
+        self.video_id = video_id
+
+
+class YouTubeRequestFailedError(TranscriptRetrievalError):
+    """Raised when a request to YouTube fails."""
+
+    def __init__(self, video_id, http_error):
+        message = 'Request to YouTube failed: {reason}'
+        super().__init__(video_id, message.format(reason=str(http_error)))
+
+
+class VideoUnavailableError(TranscriptRetrievalError):
+    """Raised when the video is unavailable."""
+
+    def __init__(self, video_id):
+        message = 'The video is no longer available'
+        super().__init__(video_id, message)
+
+
+class InvalidVideoIdError(TranscriptRetrievalError):
+    """Raised when an invalid video ID is provided."""
+
+    def __init__(self, video_id):
+        message = (
+            'You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n'
+            'Do NOT run: `YTTranscriber.get_transcript("https://www.youtube.com/watch?v=1234")`\n'
+            'Instead run: `YTTranscriber.get_transcript("1234")`'
+        )
+        super().__init__(video_id, message)
+
+
+class TooManyRequestsError(TranscriptRetrievalError):
+    """Raised when YouTube rate limits the requests."""
+
+    def __init__(self, video_id):
+        message = (
+            'YouTube is receiving too many requests from this IP and now requires solving a captcha to continue. '
+            'One of the following things can be done to work around this:\n\
+            - Manually solve the captcha in a browser and export the cookie. '
+            '- Use a different IP address\n\
+            - Wait until the ban on your IP has been lifted'
+        )
+        super().__init__(video_id, message)
+
+
+class TranscriptsDisabledError(TranscriptRetrievalError):
+    """Raised when transcripts are disabled for the video."""
+
+    def __init__(self, video_id):
+        message = 'Subtitles are disabled for this video'
+        super().__init__(video_id, message)
+
+
+class NoTranscriptAvailableError(TranscriptRetrievalError):
+    """Raised when no transcripts are available for the video."""
+
+    def __init__(self, video_id):
+        message = 'No transcripts are available for this video'
+        super().__init__(video_id, message)
+
+
+class NotTranslatableError(TranscriptRetrievalError):
+    """Raised when the transcript is not translatable."""
+
+    def __init__(self, video_id):
+        message = 'The requested language is not translatable'
+        super().__init__(video_id, message)
+
+
+class TranslationLanguageNotAvailableError(TranscriptRetrievalError):
+    """Raised when the requested translation language is not available."""
+
+    def __init__(self, video_id):
+        message = 'The requested translation language is not available'
+        super().__init__(video_id, message)
+
+
+class CookiePathInvalidError(TranscriptRetrievalError):
+    """Raised when the cookie path is invalid."""
+
+    def __init__(self, video_id):
+        message = 'The provided cookie path is invalid'
+        super().__init__(video_id, message)
+
+
+class CookiesInvalidError(TranscriptRetrievalError):
+    """Raised when the provided cookies are invalid."""
+
+    def __init__(self, video_id):
+        message = 'The cookies provided are not valid (may have expired)'
+        super().__init__(video_id, message)
+
+
+class FailedToCreateConsentCookieError(TranscriptRetrievalError):
+    """Raised when consent cookie creation fails."""
+
+    def __init__(self, video_id):
+        message = 'Failed to automatically give consent to saving cookies'
+        super().__init__(video_id, message)
+
+
+class NoTranscriptFoundError(TranscriptRetrievalError):
+    """Raised when no transcript is found for the requested language codes."""
+
+    def __init__(self, video_id, requested_language_codes, transcript_data):
+        message = (
+            'No transcripts were found for any of the requested language codes: {requested_language_codes}\n\n'
+            '{transcript_data}'
+        )
+        super().__init__(video_id, message.format(
+            requested_language_codes=requested_language_codes,
+            transcript_data=str(transcript_data)
+        ))

webscout/litagent/__init__.py

@@ -0,0 +1,172 @@
+"""
+LitAgent - A lit user agent generator with infinite possibilities! 🔥
+
+Examples:
+>>> from webscout import LitAgent
+>>> agent = LitAgent()
+>>> 
+>>> # Get random user agents
+>>> agent.random()  # Random agent from any browser
+>>> agent.mobile()  # Random mobile device agent
+>>> agent.desktop()  # Random desktop agent
+>>>
+>>> # Browser specific agents
+>>> agent.chrome()  # Latest Chrome browser
+>>> agent.firefox()  # Latest Firefox browser
+>>> agent.safari()  # Latest Safari browser
+>>> agent.edge()  # Latest Edge browser
+>>> agent.opera()  # Latest Opera browser
+>>>
+>>> # Refresh your agents
+>>> agent.refresh()  # Get fresh new agents
+"""
+
+import random
+from typing import Optional, List, Dict
+from ..Litlogger import LitLogger, LogFormat, ColorScheme
+
+logger = LitLogger(
+    name="LitAgent",
+    format=LogFormat.MODERN_EMOJI,
+    color_scheme=ColorScheme.CYBERPUNK
+)
+
+class LitAgent:
+    """A lit user agent generator that keeps it fresh! 🌟"""
+    
+    # Browser versions we support
+    BROWSERS = {
+        "chrome": (48, 120),
+        "firefox": (48, 121),
+        "safari": (605, 617),
+        "edge": (79, 120),
+        "opera": (48, 104)
+    }
+    
+    # OS versions
+    OS_VERSIONS = {
+        "windows": ["10.0", "11.0"],
+        "mac": ["10_15_7", "11_0", "12_0", "13_0", "14_0"],
+        "linux": ["x86_64", "i686"],
+        "android": ["10", "11", "12", "13", "14"],
+        "ios": ["14_0", "15_0", "16_0", "17_0"]
+    }
+    
+    # Device types
+    DEVICES = {
+        "mobile": [
+            "iPhone", "iPad", "Samsung Galaxy", "Google Pixel",
+            "OnePlus", "Xiaomi", "Huawei", "OPPO", "Vivo"
+        ],
+        "desktop": ["Windows PC", "MacBook", "iMac", "Linux Desktop"],
+        "tablet": ["iPad", "Samsung Galaxy Tab", "Microsoft Surface"],
+        "console": ["PlayStation 5", "Xbox Series X", "Nintendo Switch"],
+        "tv": ["Samsung Smart TV", "LG WebOS", "Android TV", "Apple TV"]
+    }
+
+    def __init__(self):
+        """Initialize LitAgent with style! 💫"""
+        self.agents = self._generate_agents(100)  # Keep 100 agents in memory
+
+    def _generate_agents(self, count: int) -> List[str]:
+        """Generate some lit user agents! 🛠️"""
+        agents = []
+        for _ in range(count):
+            browser = random.choice(list(self.BROWSERS.keys()))
+            version = random.randint(*self.BROWSERS[browser])
+            
+            if browser in ['chrome', 'firefox', 'edge', 'opera']:
+                os_type = random.choice(['windows', 'mac', 'linux'])
+                os_ver = random.choice(self.OS_VERSIONS[os_type])
+                
+                if os_type == 'windows':
+                    platform = f"Windows NT {os_ver}"
+                elif os_type == 'mac':
+                    platform = f"Macintosh; Intel Mac OS X {os_ver}"
+                else:
+                    platform = f"X11; Linux {os_ver}"
+                
+                agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
+                if browser == 'chrome':
+                    agent += f"Chrome/{version}.0.0.0 Safari/537.36"
+                elif browser == 'firefox':
+                    agent += f"Firefox/{version}.0"
+                elif browser == 'edge':
+                    agent += f"Edge/{version}.0.0.0"
+                elif browser == 'opera':
+                    agent += f"OPR/{version}.0.0.0"
+            
+            elif browser == 'safari':
+                device = random.choice(['mac', 'ios'])
+                if device == 'mac':
+                    ver = random.choice(self.OS_VERSIONS['mac'])
+                    agent = f"Mozilla/5.0 (Macintosh; Intel Mac OS X {ver}) "
+                else:
+                    ver = random.choice(self.OS_VERSIONS['ios'])
+                    device = random.choice(['iPhone', 'iPad'])
+                    agent = f"Mozilla/5.0 ({device}; CPU OS {ver} like Mac OS X) "
+                agent += f"AppleWebKit/{version}.1.15 (KHTML, like Gecko) Version/{version//100}.0 Safari/{version}.1.15"
+            
+            agents.append(agent)
+        
+        return list(set(agents))  # Remove any duplicates
+
+    def random(self) -> str:
+        """Get a random user agent! 🎲"""
+        return random.choice(self.agents)
+
+    def browser(self, name: str) -> str:
+        """Get a browser-specific agent! 🌐"""
+        name = name.lower()
+        if name not in self.BROWSERS:
+            logger.warning(f"Unknown browser: {name} - Using random browser")
+            return self.random()
+        
+        agents = [a for a in self.agents if name in a.lower()]
+        return random.choice(agents) if agents else self.random()
+
+    def mobile(self) -> str:
+        """Get a mobile device agent! 📱"""
+        agents = [a for a in self.agents if any(d in a for d in self.DEVICES['mobile'])]
+        return random.choice(agents) if agents else self.random()
+
+    def desktop(self) -> str:
+        """Get a desktop agent! 💻"""
+        agents = [a for a in self.agents if 'Windows' in a or 'Macintosh' in a or 'Linux' in a]
+        return random.choice(agents) if agents else self.random()
+
+    def chrome(self) -> str:
+        """Get a Chrome agent! 🌐"""
+        return self.browser('chrome')
+
+    def firefox(self) -> str:
+        """Get a Firefox agent! 🦊"""
+        return self.browser('firefox')
+
+    def safari(self) -> str:
+        """Get a Safari agent! 🧭"""
+        return self.browser('safari')
+
+    def edge(self) -> str:
+        """Get an Edge agent! 📐"""
+        return self.browser('edge')
+
+    def opera(self) -> str:
+        """Get an Opera agent! 🎭"""
+        return self.browser('opera')
+
+    def refresh(self) -> None:
+        """Refresh the agents with new ones! 🔄"""
+        self.agents = self._generate_agents(100)
+
+agent = LitAgent()
+
+if __name__ == "__main__":
+    # Test it out! 🧪
+
+    print("Random:", agent.random())
+    print("Chrome:", agent.chrome())
+    print("Firefox:", agent.firefox())
+    print("Safari:", agent.safari())
+    print("Mobile:", agent.mobile())
+    print("Desktop:", agent.desktop())