webscout 7.9__py3-none-any.whl → 8.0__py3-none-any.whl

webscout/Provider/typefully.py

@@ -0,0 +1,280 @@
+from typing import Union, Any, Dict
+import requests
+import re
+from uuid import uuid4
+
+from webscout.AIutel import Optimizers
+from webscout.AIutel import Conversation
+from webscout.AIutel import AwesomePrompts
+from webscout.AIbase import Provider
+from webscout import exceptions
+from webscout.litagent import LitAgent
+
+class TypefullyAI(Provider):
+    """
+    A class to interact with the Typefully AI API.
+
+    Attributes:
+        system_prompt (str): The system prompt to define the assistant's role.
+        model (str): The model identifier to use for completions.
+        output_length (int): Maximum length of the generated output.
+
+    Examples:
+        >>> from webscout.Provider.typefully import TypefullyAI
+        >>> ai = TypefullyAI()
+        >>> response = ai.chat("What's the weather today?")
+        >>> print(response)
+        'The weather today is sunny with a high of 75°F.'
+    """
+    AVAILABLE_MODELS = ["openai:gpt-4o-mini", "openai:gpt-4o", "anthropic:claude-3-5-haiku-20241022", "groq:llama-3.3-70b-versatile"]
+
+    def __init__(
+        self,
+        is_conversation: bool = True,
+        max_tokens: int = 600,
+        timeout: int = 30,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        proxies: dict = {},
+        history_offset: int = 10250,
+        act: str = None,
+        system_prompt: str = "You're a helpful assistant.",
+        model: str = "openai:gpt-4o-mini",
+    ):
+        """
+        Initializes the TypefullyAI API with given parameters.
+
+        Args:
+            is_conversation (bool): Whether the provider is in conversation mode.
+            max_tokens (int): Maximum number of tokens to sample.
+            timeout (int): Timeout for API requests.
+            intro (str): Introduction message for the conversation.
+            filepath (str): Filepath for storing conversation history.
+            update_file (bool): Whether to update the conversation history file.
+            proxies (dict): Proxies for the API requests.
+            history_offset (int): Offset for conversation history.
+            act (str): Act for the conversation.
+            system_prompt (str): The system prompt to define the assistant's role.
+            model (str): The model identifier to use.
+
+        Examples:
+            >>> ai = TypefullyAI(system_prompt="You are a friendly assistant.")
+            >>> print(ai.system_prompt)
+            'You are a friendly assistant.'
+        """
+        self.session = requests.Session()
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.api_endpoint = "https://typefully.com/tools/ai/api/completion"
+        self.timeout = timeout
+        self.last_response = {}
+        self.system_prompt = system_prompt
+        self.model = model
+        self.output_length = max_tokens
+
+        # Initialize LitAgent for user agent generation
+        self.agent = LitAgent()
+
+        self.headers = {
+            "authority": "typefully.com",
+            "accept": "*/*",
+            "accept-encoding": "gzip, deflate, br, zstd",
+            "accept-language": "en-US,en;q=0.9",
+            "content-type": "application/json",
+            "dnt": "1",
+            "origin": "https://typefully.com",
+            "referer": "https://typefully.com/tools/ai/chat-gpt-alternative",
+            "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Windows"',
+            "user-agent": self.agent.random()  # Use LitAgent to generate a random user agent
+        }
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        self.session.headers.update(self.headers)
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+        self.session.proxies = proxies
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Sends a prompt to the Typefully AI API and returns the response.
+
+        Args:
+            prompt (str): The prompt to send to the API.
+            stream (bool): Whether to stream the response.
+            raw (bool): Whether to return the raw response.
+            optimizer (str): Optimizer to use for the prompt.
+            conversationally (bool): Whether to generate the prompt conversationally.
+
+        Returns:
+            Dict[str, Any]: The API response.
+
+        Examples:
+            >>> ai = TypefullyAI()
+            >>> response = ai.ask("Tell me a joke!")
+            >>> print(response)
+            {'text': 'Why did the scarecrow win an award? Because he was outstanding in his field!'}
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        payload = {
+            "prompt": conversation_prompt,
+            "systemPrompt": self.system_prompt,
+            "modelIdentifier": self.model,
+            "outputLength": self.output_length
+        }
+
+        def for_stream():
+            response = self.session.post(self.api_endpoint, headers=self.headers, json=payload, stream=True, timeout=self.timeout)
+            if not response.ok:
+                raise exceptions.FailedToGenerateResponseError(
+                    f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
+                )
+            streaming_response = ""
+            for line in response.iter_lines(decode_unicode=True):
+                if line:
+                    match = re.search(r'0:"(.*?)"', line)
+                    if match:
+                        content = match.group(1)
+                        streaming_response += content
+                        yield content if raw else dict(text=content)
+                    elif line.startswith('e:') or line.startswith('d:'):
+                        # End of response
+                        break
+            self.last_response.update(dict(text=streaming_response))
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+
+        def for_non_stream():
+            for _ in for_stream():
+                pass
+            return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """
+        Generates a response from the Typefully AI API.
+
+        Args:
+            prompt (str): The prompt to send to the API.
+            stream (bool): Whether to stream the response.
+            optimizer (str): Optimizer to use for the prompt.
+            conversationally (bool): Whether to generate the prompt conversationally.
+
+        Returns:
+            str: The API response.
+
+        Examples:
+            >>> ai = TypefullyAI()
+            >>> response = ai.chat("What's the weather today?")
+            >>> print(response)
+            'The weather today is sunny with a high of 75°F.'
+        """
+
+        def for_stream():
+            for response in self.ask(
+                prompt, True, optimizer=optimizer, conversationally=conversationally
+            ):
+                yield self.get_message(response)
+
+        def for_non_stream():
+            return self.get_message(
+                self.ask(
+                    prompt,
+                    False,
+                    optimizer=optimizer,
+                    conversationally=conversationally,
+                )
+            )
+
+        return for_stream() if stream else for_non_stream()
+
+    def get_message(self, response: dict) -> str:
+        """
+        Extracts the message from the API response.
+
+        Args:
+            response (dict): The API response.
+
+        Returns:
+            str: The message content.
+
+        Examples:
+            >>> ai = TypefullyAI()
+            >>> response = ai.ask("Tell me a joke!")
+            >>> message = ai.get_message(response)
+            >>> print(message)
+            'Why did the scarecrow win an award? Because he was outstanding in his field!'
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        formatted_text = response["text"].replace('\\n', '\n').replace('\\n\\n', '\n\n')
+        return formatted_text
+
+if __name__ == "__main__":
+    print("-" * 80)
+    print(f"{'Model':<50} {'Status':<10} {'Response'}")
+    print("-" * 80)
+    
+    # Test all available models
+    working = 0
+    total = len(TypefullyAI.AVAILABLE_MODELS)
+    
+    for model in TypefullyAI.AVAILABLE_MODELS:
+        try:
+            test_ai = TypefullyAI(model=model, timeout=60)
+            response = test_ai.chat("Say 'Hello' in one word", stream=True)
+            response_text = ""
+            for chunk in response:
+                response_text += chunk
+                print(f"\r{model:<50} {'Testing...':<10}", end="", flush=True)
+            
+            if response_text and len(response_text.strip()) > 0:
+                status = "✓"
+                # Truncate response if too long
+                display_text = response_text.strip()[:50] + "..." if len(response_text.strip()) > 50 else response_text.strip()
+            else:
+                status = "✗"
+                display_text = "Empty or invalid response"
+            print(f"\r{model:<50} {status:<10} {display_text}")
+        except Exception as e:
+            print(f"\r{model:<50} {'✗':<10} {str(e)}")
+    

webscout/version.py

@@ -1,2 +1,2 @@
-__version__ = "7.9"
+__version__ = "8.0"
 __prog__ = "webscout"

webscout/webscout_search.py

@@ -91,7 +91,7 @@ class WEBS:
         if not proxy and proxies:
             warnings.warn("'proxies' is deprecated, use 'proxy' instead.", stacklevel=1)
             self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies
-        
+
         default_headers = {
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.5",
@@ -105,10 +105,10 @@ class WEBS:
             "Sec-Fetch-User": "?1",
             "Referer": "https://duckduckgo.com/",
         }
-        
+
         self.headers = headers if headers else {}
         self.headers.update(default_headers)
-        
+
         self.client = primp.Client(
             headers=self.headers,
             proxy=self.proxy,
@@ -192,18 +192,36 @@ class WEBS:
         resp_content = self._get_url("GET", "https://duckduckgo.com", params={"q": keywords}).content
         return _extract_vqd(resp_content, keywords)
 
-    def chat_yield(self, keywords: str, model: str = "gpt-4o-mini", timeout: int = 30) -> Iterator[str]:
+    def chat_yield(self, keywords: str, model: str = "gpt-4o-mini", timeout: int = 30, max_retries: int = 3) -> Iterator[str]:
         """Initiates a chat session with webscout AI.
 
         Args:
             keywords (str): The initial message or question to send to the AI.
             model (str): The model to use: "gpt-4o-mini", "llama-3.3-70b", "claude-3-haiku",
                 "o3-mini", "mistral-small-3". Defaults to "gpt-4o-mini".
-            timeout (int): Timeout value for the HTTP client. Defaults to 20.
+            timeout (int): Timeout value for the HTTP client. Defaults to 30.
+            max_retries (int): Maximum number of retry attempts for rate limited requests. Defaults to 3.
 
         Yields:
             str: Chunks of the response from the AI.
         """
+        # Get Cloudflare Turnstile token
+        def get_turnstile_token():
+            try:
+                # Visit the DuckDuckGo chat page to get the Turnstile token
+                resp_content = self._get_url(
+                    method="GET",
+                    url="https://duckduckgo.com/?q=DuckDuckGo+AI+Chat&ia=chat&duckai=1",
+                ).content
+
+                # Extract the Turnstile token if available
+                if b'cf-turnstile-response' in resp_content:
+                    token = resp_content.split(b'cf-turnstile-response="', maxsplit=1)[1].split(b'"', maxsplit=1)[0].decode()
+                    return token
+                return ""
+            except Exception:
+                return ""
+
         # x-fe-version
         if not self._chat_xfe:
             resp_content = self._get_url(
@@ -231,55 +249,100 @@ class WEBS:
         if model not in self._chat_models:
             warnings.warn(f"{model=} is unavailable. Using 'gpt-4o-mini'", stacklevel=1)
             model = "gpt-4o-mini"
+
+        # Get Cloudflare Turnstile token
+        turnstile_token = get_turnstile_token()
+
         json_data = {
             "model": self._chat_models[model],
             "messages": self._chat_messages,
         }
-        resp = self._get_url(
-            method="POST",
-            url="https://duckduckgo.com/duckchat/v1/chat",
-            headers={
-                "x-fe-version": self._chat_xfe,
-                "x-vqd-4": self._chat_vqd,
-                "x-vqd-hash-1": "",
-            },
-            json=json_data,
-            timeout=timeout,
-        )
-        self._chat_vqd = resp.headers.get("x-vqd-4", "")
-        self._chat_vqd_hash = resp.headers.get("x-vqd-hash-1", "")
-        chunks = []
-        try:
-            for chunk in resp.stream():
-                lines = chunk.split(b"data:")
-                for line in lines:
-                    if line := line.strip():
-                        if line == b"[DONE]":
-                            break
-                        if line == b"[DONE][LIMIT_CONVERSATION]":
-                            raise ConversationLimitException("ERR_CONVERSATION_LIMIT")
-                        x = json_loads(line)
-                        if isinstance(x, dict):
-                            if x.get("action") == "error":
-                                err_message = x.get("type", "")
-                                if x.get("status") == 429:
-                                    raise (
-                                        ConversationLimitException(err_message)
-                                        if err_message == "ERR_CONVERSATION_LIMIT"
-                                        else RatelimitE(err_message)
-                                    )
-                                raise WebscoutE(err_message)
-                            elif message := x.get("message"):
-                                chunks.append(message)
-                                yield message
-        except Exception as ex:
-            raise WebscoutE(f"chat_yield() {type(ex).__name__}: {ex}") from ex
 
-        result = "".join(chunks)
-        self._chat_messages.append({"role": "assistant", "content": result})
-        self._chat_tokens_count += len(result)
+        # Add Turnstile token if available
+        if turnstile_token:
+            json_data["cf-turnstile-response"] = turnstile_token
+
+        # Enhanced headers to better mimic a real browser
+        chat_headers = {
+            "x-fe-version": self._chat_xfe,
+            "x-vqd-4": self._chat_vqd,
+            "x-vqd-hash-1": "",
+            "Accept": "text/event-stream",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Cache-Control": "no-cache",
+            "Content-Type": "application/json",
+            "DNT": "1",
+            "Origin": "https://duckduckgo.com",
+            "Referer": "https://duckduckgo.com/",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": self.client.headers.get("User-Agent", "")
+        }
 
-    def chat(self, keywords: str, model: str = "gpt-4o-mini", timeout: int = 30) -> str:
+        # Retry logic for rate limited requests
+        retry_count = 0
+        while retry_count <= max_retries:
+            try:
+                resp = self._get_url(
+                    method="POST",
+                    url="https://duckduckgo.com/duckchat/v1/chat",
+                    headers=chat_headers,
+                    json=json_data,
+                    timeout=timeout,
+                )
+
+                self._chat_vqd = resp.headers.get("x-vqd-4", "")
+                self._chat_vqd_hash = resp.headers.get("x-vqd-hash-1", "")
+                chunks = []
+
+                for chunk in resp.stream():
+                    lines = chunk.split(b"data:")
+                    for line in lines:
+                        if line := line.strip():
+                            if line == b"[DONE]":
+                                break
+                            if line == b"[DONE][LIMIT_CONVERSATION]":
+                                raise ConversationLimitException("ERR_CONVERSATION_LIMIT")
+                            x = json_loads(line)
+                            if isinstance(x, dict):
+                                if x.get("action") == "error":
+                                    err_message = x.get("type", "")
+                                    if x.get("status") == 429:
+                                        raise (
+                                            ConversationLimitException(err_message)
+                                            if err_message == "ERR_CONVERSATION_LIMIT"
+                                            else RatelimitE(err_message)
+                                        )
+                                    raise WebscoutE(err_message)
+                                elif message := x.get("message"):
+                                    chunks.append(message)
+                                    yield message
+
+                # If we get here, the request was successful
+                result = "".join(chunks)
+                self._chat_messages.append({"role": "assistant", "content": result})
+                self._chat_tokens_count += len(result)
+                return
+
+            except RatelimitE as ex:
+                retry_count += 1
+                if retry_count > max_retries:
+                    raise WebscoutE(f"chat_yield() Rate limit exceeded after {max_retries} retries: {ex}") from ex
+
+                # Get a fresh Turnstile token for the retry
+                turnstile_token = get_turnstile_token()
+                if turnstile_token:
+                    json_data["cf-turnstile-response"] = turnstile_token
+
+                # Exponential backoff
+                sleep_time = 2 ** retry_count
+                sleep(sleep_time)
+
+            except Exception as ex:
+                raise WebscoutE(f"chat_yield() {type(ex).__name__}: {ex}") from ex
+
+    def chat(self, keywords: str, model: str = "gpt-4o-mini", timeout: int = 30, max_retries: int = 3) -> str:
         """Initiates a chat session with webscout AI.
 
         Args:
@@ -287,11 +350,12 @@ class WEBS:
             model (str): The model to use: "gpt-4o-mini", "llama-3.3-70b", "claude-3-haiku",
                 "o3-mini", "mistral-small-3". Defaults to "gpt-4o-mini".
             timeout (int): Timeout value for the HTTP client. Defaults to 30.
+            max_retries (int): Maximum number of retry attempts for rate limited requests. Defaults to 3.
 
         Returns:
             str: The response from the AI.
         """
-        answer_generator = self.chat_yield(keywords, model, timeout)
+        answer_generator = self.chat_yield(keywords, model, timeout, max_retries)
         return "".join(answer_generator)
 
     def text(
@@ -1225,22 +1289,22 @@ class WEBS:
         assert location, "location is mandatory"
         lang = language.split('-')[0]
         url = f"https://duckduckgo.com/js/spice/forecast/{quote(location)}/{lang}"
-        
+
         resp = self._get_url("GET", url).content
         resp_text = resp.decode('utf-8')
-        
+
         if "ddg_spice_forecast(" not in resp_text:
             raise WebscoutE(f"No weather data found for {location}")
-        
+
         json_text = resp_text[resp_text.find('(') + 1:resp_text.rfind(')')]
         try:
             result = json.loads(json_text)
         except Exception as e:
             raise WebscoutE(f"Error parsing weather JSON: {e}")
-        
+
         if not result or 'currentWeather' not in result or 'forecastDaily' not in result:
             raise WebscoutE(f"Invalid weather data format for {location}")
-        
+
         formatted_data = {
             "location": result["currentWeather"]["metadata"].get("ddg-location", "Unknown"),
             "current": {