webscout 8.2.4__py3-none-any.whl → 8.2.6__py3-none-any.whl

webscout/Extra/autocoder/autocoder.py

@@ -7,19 +7,16 @@ import queue
 import tempfile
 import threading
 import subprocess
+from typing import Optional, Generator, List, Tuple, Dict, Any, NamedTuple
 from rich.panel import Panel
 from rich.syntax import Syntax
-from rich.console import Console, Group
+from rich.console import Console
 from rich.markdown import Markdown
 from rich.table import Table
 from rich.theme import Theme
 from rich.live import Live
-from rich.rule import Rule
 from rich.box import ROUNDED
-from typing import Optional, Generator, List, Tuple, Dict, Any
-from webscout.AIutel import run_system_command
 from .autocoder_utiles import get_intro_prompt
-
 # Initialize LitLogger with custom format and colors 
 default_path = tempfile.mkdtemp(prefix="webscout_autocoder")
 
@@ -34,6 +31,70 @@ CUSTOM_THEME = Theme({
 })
 
 console = Console(theme=CUSTOM_THEME)
+class CommandResult(NamedTuple):
+    """Result of a system command execution."""
+    success: bool
+    stdout: str
+    stderr: str
+    
+def run_system_command(
+    command: str,
+    exit_on_error: bool = False,
+    stdout_error: bool = False,
+    help: Optional[str] = None
+) -> Tuple[bool, CommandResult]:
+    """Execute a system command and return the result.
+    
+    Args:
+        command (str): Command to execute
+        exit_on_error (bool): Whether to exit on error. Defaults to False.
+        stdout_error (bool): Whether to include stdout in error messages. Defaults to False.
+        help (str, optional): Help message for errors. Defaults to None.
+        
+    Returns:
+        Tuple[bool, CommandResult]: Success status and command result containing stdout/stderr
+    """
+    try:
+        # Execute command and capture output
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=True,
+            text=True
+        )
+        
+        # Get stdout and stderr
+        stdout, stderr = process.communicate()
+        success = process.returncode == 0
+        
+        # Create result object
+        result = CommandResult(
+            success=success,
+            stdout=stdout.strip() if stdout else "",
+            stderr=stderr.strip() if stderr else ""
+        )
+        
+        # Handle errors if needed
+        if not success and exit_on_error:
+            error_msg = stderr if stderr else stdout if stdout_error else "Command failed"
+            if help:
+                error_msg += f"\n{help}"
+            sys.exit(error_msg)
+            
+        return success, result
+        
+    except Exception as e:
+        # Handle execution errors
+        error_msg = str(e)
+        if help:
+            error_msg += f"\n{help}"
+            
+        if exit_on_error:
+            sys.exit(error_msg)
+            
+        return False, CommandResult(success=False, stdout="", stderr=error_msg)
+
 
 class AutoCoder:
     """Generate and auto-execute Python scripts in the CLI with advanced error handling and retry logic.

webscout/Extra/gguf.py

@@ -1,6 +1,8 @@
 """
 Convert Hugging Face models to GGUF format with advanced features.
 
+For detailed documentation, see: webscout/Extra/gguf.md
+
 >>> python -m webscout.Extra.gguf convert -m "OEvortex/HelpingAI-Lite-1.5T" -q "q4_k_m,q5_k_m"
 >>> # With upload options:
 >>> python -m webscout.Extra.gguf convert -m "your-model" -u "username" -t "token" -q "q4_k_m"

webscout/Provider/AISEARCH/scira_search.py

@@ -69,17 +69,15 @@ class Scira(AISearch):
     """
 
     AVAILABLE_MODELS = {
-        "scira-default": "Grok3",
-        "scira-grok-3-mini": "Grok3-mini", # thinking model
+        "scira-default": "Grok3-mini", # thinking model
+        "scira-grok-3": "Grok3",
+        "scira-anthropic": "Sonnet 3.7 thinking",
         "scira-vision" : "Grok2-Vision", # vision model
         "scira-4.1-mini": "GPT4.1-mini",
         "scira-qwq": "QWQ-32B",
         "scira-o4-mini": "o4-mini",
         "scira-google": "gemini 2.5 flash"
-
-
     }
-
     def __init__(
         self,
         timeout: int = 60,

webscout/Provider/Aitopia.py

@@ -1,4 +1,5 @@
-import requests
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
 import json
 import uuid
 import time
@@ -6,8 +7,8 @@ import hashlib
 from typing import Any, Dict, Optional, Generator, Union
 
 from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
+from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream
+from webscout.AIutel import AwesomePrompts
 from webscout.AIbase import Provider, AsyncProvider
 from webscout import exceptions
 from webscout.litagent import LitAgent
@@ -67,9 +68,9 @@ class Aitopia(Provider):
             "user-agent": self.fingerprint["user_agent"]
         }
         
-        self.session = requests.Session()
+        self.session = Session() # Use curl_cffi Session
         self.session.headers.update(self.headers)
-        self.session.proxies.update(proxies)
+        self.session.proxies = proxies # Assign proxies directly
 
         self.is_conversation = is_conversation
         self.max_tokens_to_sample = max_tokens
@@ -129,6 +130,20 @@ class Aitopia(Provider):
         random_str = str(uuid.uuid4()) + str(time.time())
         return hashlib.md5(random_str.encode()).hexdigest()
 
+    @staticmethod
+    def _aitopia_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]:
+        """Extracts content from Aitopia stream JSON objects."""
+        if isinstance(chunk, dict):
+            # Handle Claude 3 Haiku response format
+            if "delta" in chunk and "text" in chunk["delta"]:
+                return chunk["delta"]["text"]
+            # Handle GPT-4o Mini response format
+            elif "choices" in chunk and "0" in chunk["choices"]:
+                return chunk["choices"]["0"]["delta"].get("content")
+            # Add other potential formats here if needed
+        return None
+
+
     def ask(
         self,
         prompt: str,
@@ -184,63 +199,72 @@ class Aitopia(Provider):
         }
 
         def for_stream():
+            streaming_text = "" # Initialize outside try block
             try:
-                with requests.post(self.url, headers=self.headers, json=payload, stream=True, timeout=self.timeout) as response:
-                    if response.status_code != 200:
-                        raise exceptions.FailedToGenerateResponseError(
-                            f"Request failed with status code {response.status_code}"
-                        )
-                    
-                    streaming_text = ""
-                    for line in response.iter_lines():
-                        if line:
-                            line = line.decode('utf-8')
-                            if line.startswith('data: '):
-                                data = line[6:]
-                                if data == '[DONE]':
-                                    break
-                                try:
-                                    json_data = json.loads(data)
-                                    
-                                    # Handle Claude 3 Haiku response format
-                                    if "delta" in json_data and "text" in json_data["delta"]:
-                                        content = json_data["delta"]["text"]
-                                        if content:
-                                            streaming_text += content
-                                            resp = dict(text=content)
-                                            yield resp if raw else resp
-                                    # Handle GPT-4o Mini response format
-                                    elif "choices" in json_data and "0" in json_data["choices"]:
-                                        content = json_data["choices"]["0"]["delta"].get("content", "")
-                                        if content:
-                                            streaming_text += content
-                                            resp = dict(text=content)
-                                            yield resp if raw else resp
-                                except json.JSONDecodeError:
-                                    continue
-                    
+                response = self.session.post(
+                    self.url, headers=self.headers, json=payload, stream=True, timeout=self.timeout,
+                    impersonate="chrome120" # Add impersonate
+                )
+                response.raise_for_status()
+
+                # Use sanitize_stream
+                processed_stream = sanitize_stream(
+                    data=response.iter_content(chunk_size=None), # Pass byte iterator
+                    intro_value="data:",
+                    to_json=True,     # Stream sends JSON
+                    skip_markers=["[DONE]"],
+                    content_extractor=self._aitopia_extractor, # Use the specific extractor
+                    yield_raw_on_error=False # Skip non-JSON lines or lines where extractor fails
+                )
+
+                for content_chunk in processed_stream:
+                    # content_chunk is the string extracted by _aitopia_extractor
+                    if content_chunk and isinstance(content_chunk, str):
+                        streaming_text += content_chunk
+                        resp = dict(text=content_chunk)
+                        yield resp if not raw else content_chunk
+
+            except CurlError as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e
+            except Exception as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed: {str(e)}")
+            finally:
+                # Update history after stream finishes or fails
+                if streaming_text:
                     self.last_response = {"text": streaming_text}
                     self.conversation.update_chat_history(prompt, streaming_text)
-                    
-            except requests.RequestException as e:
-                raise exceptions.FailedToGenerateResponseError(f"Request failed: {str(e)}")
 
         def for_non_stream():
             try:
-                response = requests.post(self.url, headers=self.headers, json=payload, timeout=self.timeout)
-                if response.status_code != 200:
-                    raise exceptions.FailedToGenerateResponseError(
-                        f"Request failed with status code {response.status_code}"
-                    )
+                response = self.session.post(
+                    self.url, headers=self.headers, json=payload, timeout=self.timeout,
+                    impersonate="chrome120" # Add impersonate
+                )
+                response.raise_for_status()
+
+                response_text_raw = response.text # Get raw text
+
+                # Use sanitize_stream to parse the non-streaming JSON response
+                # Assuming non-stream uses the GPT format based on original code
+                processed_stream = sanitize_stream(
+                    data=response_text_raw,
+                    to_json=True, # Parse the whole text as JSON
+                    intro_value=None,
+                    content_extractor=lambda chunk: chunk.get("choices", [{}])[0].get("message", {}).get("content") if isinstance(chunk, dict) else None,
+                    yield_raw_on_error=False
+                )
+                # Extract the single result
+                content = next(processed_stream, None)
+                content = content if isinstance(content, str) else "" # Ensure it's a string
 
-                response_data = response.json()
-                if 'choices' in response_data and len(response_data['choices']) > 0:
-                    content = response_data['choices'][0].get('message', {}).get('content', '')
+                if content: # Check if content was successfully extracted
                     self.last_response = {"text": content}
                     self.conversation.update_chat_history(prompt, content)
                     return {"text": content}
                 else:
-                    raise exceptions.FailedToGenerateResponseError("No response content found")
+                    raise exceptions.FailedToGenerateResponseError("No response content found or failed to parse")
+            except CurlError as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}") from e
             except Exception as e:
                 raise exceptions.FailedToGenerateResponseError(f"Request failed: {e}")
 

webscout/Provider/AllenAI.py

@@ -156,19 +156,15 @@ class AllenAI(Provider):
             err_text = getattr(e, 'response', None) and getattr(e.response, 'text', '')
             return {"client": temp_id, "error": f"{type(e).__name__}: {e} - {err_text}"}
     
-
-    def parse_stream(self, raw_data):
-        """Parse the raw streaming data according to the JS implementation"""
-        result = ""
-        for line in raw_data.splitlines():
-            try:
-                parsed = json.loads(line)
-                # Check if message starts with msg_ pattern
-                if parsed.get("message", "").startswith("msg_"):
-                    result += parsed.get("content", "")
-            except:
-                continue
-        return result
+    @staticmethod
+    def _allenai_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]:
+        """Extracts content from AllenAI stream JSON objects."""
+        if isinstance(chunk, dict):
+            if chunk.get("message", "").startswith("msg_") and "content" in chunk:
+                return chunk.get("content")
+            elif "message" in chunk and chunk.get("content"): # Legacy handling
+                return chunk.get("content")
+        return None
 
     def ask(
         self,
@@ -269,57 +265,49 @@ class AllenAI(Provider):
             )
             response.raise_for_status() # Check for HTTP errors
             
-            # Iterate over bytes and decode manually
-            for chunk_bytes in response.iter_content(chunk_size=None): # Process byte chunks
-                if not chunk_bytes:
-                    continue
-                    
-                try:
-                    decoded = chunk_bytes.decode('utf-8')
-                    for line in decoded.splitlines(): # Process lines within the chunk
-                        line = line.strip()
-                        if not line:
-                            continue
-                        
-                        data = json.loads(line)
-                        
-                        # Check for message pattern from JS implementation
-                        if isinstance(data, dict):
-                            content_found = False
-                            if data.get("message", "").startswith("msg_") and "content" in data:
-                                content = data.get("content", "")
-                                content_found = True
-                            # Legacy handling for older API
-                            elif "message" in data and data.get("content"):
-                                content = data.get("content")
-                                content_found = True
-                            
-                            if content_found and content:
-                                streaming_text += content
-                                resp = dict(text=content)
-                                yield resp if not raw else content
-                            
-                            # Update parent ID if present
-                            if data.get("id"):
-                                current_parent = data.get("id")
-                            elif data.get("children"):
-                                for child in data["children"]:
-                                    if child.get("role") == "assistant":
-                                        current_parent = child.get("id")
-                                        break
-                            
-                            # Handle completion
-                            if data.get("final") or data.get("finish_reason") == "stop":
-                                if current_parent:
-                                    self.parent = current_parent
-                                
-                                # Update conversation history
-                                self.conversation.update_chat_history(prompt, streaming_text)
-                                self.last_response = {"text": streaming_text}
-                                return # End the generator
+            # Use sanitize_stream
+            processed_stream = sanitize_stream(
+                data=response.iter_content(chunk_size=None), # Pass byte iterator
+                intro_value=None, # No prefix
+                to_json=True,     # Stream sends JSON lines
+                content_extractor=self._allenai_extractor, # Use the specific extractor
+                yield_raw_on_error=False # Skip non-JSON lines or lines where extractor fails
+            )
+
+            for content_chunk in processed_stream:
+                # content_chunk is the string extracted by _allenai_extractor
+                if content_chunk and isinstance(content_chunk, str):
+                    streaming_text += content_chunk
+                    resp = dict(text=content_chunk)
+                    yield resp if not raw else content_chunk
 
-                except (json.JSONDecodeError, UnicodeDecodeError):
-                    continue # Ignore lines/chunks that are not valid JSON or cannot be decoded
+            # Try to extract parent ID from the *last* raw line (less reliable than before)
+            # This part is tricky as sanitize_stream consumes the raw lines.
+            # We might need to re-fetch or adjust if parent ID is critical per stream.
+            # For now, we'll rely on the non-stream request to update parent ID more reliably.
+            # Example placeholder logic (might not work reliably):
+            try:
+                last_line_data = json.loads(response.text.splitlines()[-1]) # Get last line if possible
+                if last_line_data.get("id"):
+                    current_parent = last_line_data.get("id")
+                elif last_line_data.get("children"):
+                    for child in last_line_data["children"]: # Use last_line_data here
+                        if child.get("role") == "assistant":
+                            current_parent = child.get("id")
+                            break
+                
+                # Handle completion
+                if last_line_data.get("final") or last_line_data.get("finish_reason") == "stop":
+                    if current_parent:
+                        self.parent = current_parent
+                    
+                    # Update conversation history
+                    self.conversation.update_chat_history(prompt, streaming_text)
+                    self.last_response = {"text": streaming_text} # Update last response here
+                    return # End the generator
+            except Exception as e:
+                # Log the error but continue with the rest of the function
+                print(f"Error processing response data: {str(e)}")
             
             # If loop finishes without returning (e.g., no final message), update history
             if current_parent:
@@ -348,10 +336,19 @@ class AllenAI(Provider):
             )
             response.raise_for_status() # Check for HTTP errors
             
-            # Parse the response as per JS implementation
-            raw_response = response.text
-            parsed_response = self.parse_stream(raw_response) # Use existing parser
-            
+            raw_response = response.text # Get raw text
+
+            # Process the full text using sanitize_stream line by line
+            processed_stream = sanitize_stream(
+                data=raw_response.splitlines(), # Split into lines
+                intro_value=None,
+                to_json=True,
+                content_extractor=self._allenai_extractor,
+                yield_raw_on_error=False
+            )
+            # Aggregate the results
+            parsed_response = "".join(list(processed_stream))
+
             # Update parent ID from the full response if possible (might need adjustment based on actual non-stream response structure)
             # This part is speculative as the non-stream structure isn't fully clear from the stream logic
             try:

webscout/Provider/ChatGPTClone.py

@@ -10,7 +10,7 @@ from datetime import date
 
 from webscout.AIutel import Optimizers
 from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts
+from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
 from webscout.AIbase import Provider
 from webscout import WEBS, exceptions
 # from webscout.litagent import LitAgent
@@ -105,6 +105,17 @@ class ChatGPTClone(Provider):
         self.session.headers.update(self.headers)
         return self.impersonate
 
+    @staticmethod
+    def _chatgptclone_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]:
+        """Extracts content from the ChatGPTClone stream format '0:"..."'."""
+        if isinstance(chunk, str):
+            match = re.search(r'0:"((?:[^\\"]|\\.)*)"', chunk) # Use the existing regex
+            if match:
+                content = match.group(1)
+                # Decode JSON string escapes and then unicode escapes
+                decoded_content = json.loads(f'"{content}"').encode().decode('unicode_escape')
+                return decoded_content
+        return None
     def ask(
         self,
         prompt: str,
@@ -154,44 +165,31 @@ class ChatGPTClone(Provider):
 
         def for_stream():
             response = _make_request()
-            streaming_text = ""
-            buffer = ""
+            streaming_text = "" # Initialize outside try block
             try:
-                for line in response.iter_content():
-                    if line:
-                        # decode bytes to str before concatenation
-                        if isinstance(line, bytes):
-                            line = line.decode("utf-8", errors="replace")
-                        buffer += line
-                        match = re.search(r'0:"((?:[^\\"]|\\.)*)"', buffer)
-                        if match:
-                            content = match.group(1)
-                            try:
-                                decoded_content = json.loads(f'"{content}"')
-                            except json.JSONDecodeError:
-                                decoded_content = content.encode().decode('unicode_escape')
-                            streaming_text += decoded_content
-                            yield decoded_content if raw else dict(text=decoded_content)
-                            buffer = ""
-                        elif len(buffer) > 1024:
-                            buffer = ""
-                if buffer:
-                    match = re.search(r'0:"((?:[^\\"]|\\.)*)"', buffer)
-                    if match:
-                        content = match.group(1)
-                        try:
-                            decoded_content = json.loads(f'"{content}"')
-                        except json.JSONDecodeError:
-                            decoded_content = content.encode().decode('unicode_escape')
-                        streaming_text += decoded_content
-                        yield decoded_content if raw else dict(text=decoded_content)
-                self.last_response.update(dict(text=streaming_text))
-                self.conversation.update_chat_history(prompt, streaming_text)
+                # Use sanitize_stream
+                processed_stream = sanitize_stream(
+                    data=response.iter_content(chunk_size=None), # Pass byte iterator
+                    intro_value=None, # No simple prefix
+                    to_json=False,    # Content is text after extraction
+                    content_extractor=self._chatgptclone_extractor, # Use the specific extractor
+                    yield_raw_on_error=True # Yield even if extractor fails (might get metadata lines)
+                )
+
+                for content_chunk in processed_stream:
+                    # content_chunk is the string extracted by _chatgptclone_extractor
+                    if content_chunk and isinstance(content_chunk, str):
+                        streaming_text += content_chunk
+                        yield content_chunk if raw else dict(text=content_chunk)
+
             except RequestsError as e:
                 raise exceptions.FailedToGenerateResponseError(f"Stream interrupted by request error: {e}") from e
             except Exception as e:
                 raise exceptions.FailedToGenerateResponseError(f"Error processing stream: {e}") from e
             finally:
+                # Update history after stream finishes or fails
+                self.last_response.update(dict(text=streaming_text))
+                self.conversation.update_chat_history(prompt, streaming_text)
                 response.close()
 
         def for_non_stream():
@@ -227,7 +225,8 @@ class ChatGPTClone(Provider):
         assert isinstance(response, dict)
         if not isinstance(response, dict) or "text" not in response:
             return str(response)
-        formatted_text = response["text"]
+        # Extractor handles formatting
+        formatted_text = response.get("text", "")
         return formatted_text
 
 if __name__ == "__main__":