webscout 8.2.3__py3-none-any.whl → 8.2.5__py3-none-any.whl

webscout/DWEBS.py

@@ -4,7 +4,7 @@ DWEBS - A Google search library with advanced features
 import random
 from time import sleep
 from webscout.scout import Scout
-from requests import get
+from curl_cffi.requests import Session
 from urllib.parse import unquote, urlencode
 from typing import List, Dict, Optional, Union, Iterator, Any
 from concurrent.futures import ThreadPoolExecutor
@@ -35,34 +35,55 @@ class SearchResult:
 
 class GoogleSearch:
     """Google search implementation with configurable parameters and advanced features."""
-    
+
     _executor: ThreadPoolExecutor = ThreadPoolExecutor()
-    
+
     def __init__(
         self,
         timeout: int = 10,
         proxies: Optional[Dict[str, str]] = None,
         verify: bool = True,
         lang: str = "en",
-        sleep_interval: float = 0.0
+        sleep_interval: float = 0.0,
+        impersonate: str = "chrome110"
     ):
         """
         Initialize GoogleSearch with custom settings.
-        
+
         Args:
             timeout: Request timeout in seconds
             proxies: Proxy configuration for requests
             verify: Whether to verify SSL certificates
             lang: Search language
             sleep_interval: Sleep time between pagination requests
+            impersonate: Browser profile for curl_cffi. Defaults to "chrome110".
         """
-        self.timeout = timeout
+        self.timeout = timeout # Keep timeout for potential non-session uses or reference
         self.proxies = proxies if proxies else {}
         self.verify = verify
         self.lang = lang
         self.sleep_interval = sleep_interval
         self.base_url = "https://www.google.com/search"
-    
+        # Initialize curl_cffi session
+        self.session = Session(
+            proxies=self.proxies,
+            verify=self.verify,
+            timeout=self.timeout,
+            impersonate=impersonate
+        )
+        # Set common headers for the session
+        self.session.headers = {
+            "User-Agent": self._get_useragent(),
+            "Accept-Language": self.lang,
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        }
+        # Set default cookies for the session
+        self.session.cookies.update({
+            'CONSENT': 'PENDING+987',
+            'SOCS': 'CAESHAgBEhIaAB',
+        })
+
     def _get_useragent(self) -> str:
         """
         Generate a random user agent string.
@@ -99,29 +120,23 @@ class GoogleSearch:
         # Add search type if specified
         if search_type:
             params["tbm"] = search_type
-            
+        
         try:
-            resp = get(
+            # Use the curl_cffi session
+            resp = self.session.get(
                 url=self.base_url,
-                headers={
-                    "User-Agent": self._get_useragent(),
-                    "Accept-Language": self.lang,
-                    "Accept-Encoding": "gzip, deflate, br",
-                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-                },
                 params=params,
-                proxies=self.proxies if any(self.proxies) else None,
-                timeout=self.timeout,
-                verify=self.verify,
-                cookies={
-                    'CONSENT': 'PENDING+987',
-                    'SOCS': 'CAESHAgBEhIaAB',
-                }
+                # Headers and cookies are now part of the session
+                # proxies, timeout, verify are handled by the session
             )
             resp.raise_for_status()
             return resp.text
         except Exception as e:
-            raise RuntimeError(f"Search request failed: {str(e)}")
+            # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise RuntimeError(f"Search request failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise RuntimeError(f"Search request failed: {str(e)}")
     
     def _extract_url(self, raw_link: str) -> Optional[str]:
         """
@@ -283,8 +298,15 @@ class GoogleSearch:
         start = start_num
         
         while len(fetched_results) < max_results:
+            # Add safe search parameter to the request
+            # Note: This modifies the session params for this specific request type
+            # It might be better to pass params directly to session.get if mixing search types
+            term_with_safe = f"{keywords} safe:{safe}"
+            if region and region.lower() != "all":
+                 term_with_safe += f" location:{region}" # Example of adding region, adjust as needed
+
             response_html = self._make_request(
-                term=keywords, 
+                term=term_with_safe, # Pass term with safe search
                 results=max_results - len(fetched_results), 
                 start=start
             )
@@ -304,7 +326,7 @@ class GoogleSearch:
             if len(fetched_results) >= max_results:
                 break
             
-            start += 10
+            start += 10 # Google typically uses increments of 10
             sleep(self.sleep_interval)
         
         return fetched_results[:max_results]
@@ -340,11 +362,15 @@ class GoogleSearch:
         safe = safe_map.get(safesearch.lower(), "moderate")
         
         # Keep track of unique results
-        fetched_results = []
         fetched_links = set()
-        
+
+        # Add safe search parameter
+        term_with_safe = f"{keywords} safe:{safe}"
+        if region and region.lower() != "all":
+             term_with_safe += f" location:{region}" # Example
+
         response_html = self._make_request(
-            term=keywords, 
+            term=term_with_safe, # Pass term with safe search
             results=max_results,
             search_type="nws"
         )
@@ -353,7 +379,7 @@ class GoogleSearch:
             html=response_html,
             num_results=max_results,
             fetched_links=fetched_links,
-            unique=True
+            unique=True # News results are generally unique per request
         )
         
         return results[:max_results]
@@ -384,17 +410,19 @@ class GoogleSearch:
             
             url = f"https://www.google.com/complete/search?{urlencode(params)}"
             
+            # Use a simpler header set for the suggestions API
             headers = {
                 "User-Agent": self._get_useragent(),
                 "Accept": "application/json, text/javascript, */*",
                 "Accept-Language": self.lang,
             }
             
-            response = get(
+            # Use session.get but override headers for this specific request
+            response = self.session.get(
                 url=url,
                 headers=headers,
-                timeout=self.timeout,
-                verify=self.verify
+                params=params # Pass params directly
+                # timeout and verify are handled by session
             )
             response.raise_for_status()
             
@@ -405,25 +433,40 @@ class GoogleSearch:
             return []
             
         except Exception as e:
+            # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 # Log error or handle differently if needed
+                 print(f"Suggestions request failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 print(f"Suggestions request failed: {str(e)}")
             # Return empty list on error instead of raising exception
             return []
 
 
 # Legacy function support for backward compatibility
-def search(term, num_results=10, lang="en", proxy=None, advanced=False, sleep_interval=0, timeout=5, safe="active", ssl_verify=True, region=None, start_num=0, unique=False):
+def search(term, num_results=10, lang="en", proxy=None, advanced=False, sleep_interval=0, timeout=5, safe="active", ssl_verify=True, region=None, start_num=0, unique=False, impersonate="chrome110"): # Added impersonate
     """Legacy function for backward compatibility."""
     google_search = GoogleSearch(
         timeout=timeout,
         proxies={"https": proxy, "http": proxy} if proxy else None,
         verify=ssl_verify,
         lang=lang,
-        sleep_interval=sleep_interval
+        sleep_interval=sleep_interval,
+        impersonate=impersonate # Pass impersonate
     )
     
+    # Map legacy safe values
+    safe_search_map = {
+        "active": "on",
+        "moderate": "moderate",
+        "off": "off"
+    }
+    safesearch_val = safe_search_map.get(safe, "moderate")
+
     results = google_search.text(
         keywords=term,
         region=region,
-        safesearch="on" if safe == "active" else "moderate" if safe == "moderate" else "off",
+        safesearch=safesearch_val,
         max_results=num_results,
         start_num=start_num,
         unique=unique

webscout/Extra/gguf.py

@@ -1,6 +1,8 @@
 """
 Convert Hugging Face models to GGUF format with advanced features.
 
+For detailed documentation, see: webscout/Extra/gguf.md
+
 >>> python -m webscout.Extra.gguf convert -m "OEvortex/HelpingAI-Lite-1.5T" -q "q4_k_m,q5_k_m"
 >>> # With upload options:
 >>> python -m webscout.Extra.gguf convert -m "your-model" -u "username" -t "token" -q "q4_k_m"

webscout/Extra/tempmail/base.py

@@ -148,7 +148,7 @@ def get_random_email(provider_name: str = "mailtm") -> Tuple[str, TempMailProvid
         except Exception as e:
             raise RuntimeError(f"Failed to initialize provider: {e}")
 
-    # Create the account
+    # Create the account (auto-generates a random email)
     success = provider.create_account()
     if not success:
         raise RuntimeError(f"Failed to create account with provider {provider_name}")

webscout/Provider/AISEARCH/hika_search.py

@@ -31,12 +31,15 @@ class Hika(AISearch):
         timeout: int = 60,
         proxies: Optional[dict] = None,
         language: str = "en",
+        model: str = "deepseek-r1",
+        
     ):
         self.session = requests.Session()
         self.base_url = "https://api.hika.fyi/api/"
         self.endpoint = "kbase/web"
         self.timeout = timeout
         self.language = language
+        self.model = model
         self.last_response = {}
         
         self.headers = {
@@ -104,6 +107,7 @@ class Hika(AISearch):
         # Prepare payload
         payload = {
             "keyword": prompt,
+            "model": self.model,
             "language": self.language,
             "stream": True  # Always request streaming for consistent handling
         }

webscout/Provider/AISEARCH/scira_search.py

@@ -69,17 +69,14 @@ class Scira(AISearch):
     """
 
     AVAILABLE_MODELS = {
-        "scira-default": "Grok3",
-        "scira-grok-3-mini": "Grok3-mini", # thinking model
+        "scira-default": "Grok3-mini", # thinking model
+        "scira-grok-3": "Grok3", 
         "scira-vision" : "Grok2-Vision", # vision model
         "scira-4.1-mini": "GPT4.1-mini",
         "scira-qwq": "QWQ-32B",
         "scira-o4-mini": "o4-mini",
         "scira-google": "gemini 2.5 flash"
-
-
     }
-
     def __init__(
         self,
         timeout: int = 60,

webscout/Provider/Aitopia.py

@@ -1,4 +1,5 @@
-import requests
+from curl_cffi import CurlError
+from curl_cffi.requests import Session
 import json
 import uuid
 import time
@@ -6,8 +7,8 @@ import hashlib
 from typing import Any, Dict, Optional, Generator, Union
 
 from webscout.AIutel import Optimizers
-from webscout.AIutel import Conversation
-from webscout.AIutel import AwesomePrompts, sanitize_stream
+from webscout.AIutel import Conversation, sanitize_stream # Import sanitize_stream
+from webscout.AIutel import AwesomePrompts
 from webscout.AIbase import Provider, AsyncProvider
 from webscout import exceptions
 from webscout.litagent import LitAgent
@@ -67,9 +68,9 @@ class Aitopia(Provider):
             "user-agent": self.fingerprint["user_agent"]
         }
         
-        self.session = requests.Session()
+        self.session = Session() # Use curl_cffi Session
         self.session.headers.update(self.headers)
-        self.session.proxies.update(proxies)
+        self.session.proxies = proxies # Assign proxies directly
 
         self.is_conversation = is_conversation
         self.max_tokens_to_sample = max_tokens
@@ -129,6 +130,20 @@ class Aitopia(Provider):
         random_str = str(uuid.uuid4()) + str(time.time())
         return hashlib.md5(random_str.encode()).hexdigest()
 
+    @staticmethod
+    def _aitopia_extractor(chunk: Union[str, Dict[str, Any]]) -> Optional[str]:
+        """Extracts content from Aitopia stream JSON objects."""
+        if isinstance(chunk, dict):
+            # Handle Claude 3 Haiku response format
+            if "delta" in chunk and "text" in chunk["delta"]:
+                return chunk["delta"]["text"]
+            # Handle GPT-4o Mini response format
+            elif "choices" in chunk and "0" in chunk["choices"]:
+                return chunk["choices"]["0"]["delta"].get("content")
+            # Add other potential formats here if needed
+        return None
+
+
     def ask(
         self,
         prompt: str,
@@ -184,63 +199,72 @@ class Aitopia(Provider):
         }
 
         def for_stream():
+            streaming_text = "" # Initialize outside try block
             try:
-                with requests.post(self.url, headers=self.headers, json=payload, stream=True, timeout=self.timeout) as response:
-                    if response.status_code != 200:
-                        raise exceptions.FailedToGenerateResponseError(
-                            f"Request failed with status code {response.status_code}"
-                        )
-                    
-                    streaming_text = ""
-                    for line in response.iter_lines():
-                        if line:
-                            line = line.decode('utf-8')
-                            if line.startswith('data: '):
-                                data = line[6:]
-                                if data == '[DONE]':
-                                    break
-                                try:
-                                    json_data = json.loads(data)
-                                    
-                                    # Handle Claude 3 Haiku response format
-                                    if "delta" in json_data and "text" in json_data["delta"]:
-                                        content = json_data["delta"]["text"]
-                                        if content:
-                                            streaming_text += content
-                                            resp = dict(text=content)
-                                            yield resp if raw else resp
-                                    # Handle GPT-4o Mini response format
-                                    elif "choices" in json_data and "0" in json_data["choices"]:
-                                        content = json_data["choices"]["0"]["delta"].get("content", "")
-                                        if content:
-                                            streaming_text += content
-                                            resp = dict(text=content)
-                                            yield resp if raw else resp
-                                except json.JSONDecodeError:
-                                    continue
-                    
+                response = self.session.post(
+                    self.url, headers=self.headers, json=payload, stream=True, timeout=self.timeout,
+                    impersonate="chrome120" # Add impersonate
+                )
+                response.raise_for_status()
+
+                # Use sanitize_stream
+                processed_stream = sanitize_stream(
+                    data=response.iter_content(chunk_size=None), # Pass byte iterator
+                    intro_value="data:",
+                    to_json=True,     # Stream sends JSON
+                    skip_markers=["[DONE]"],
+                    content_extractor=self._aitopia_extractor, # Use the specific extractor
+                    yield_raw_on_error=False # Skip non-JSON lines or lines where extractor fails
+                )
+
+                for content_chunk in processed_stream:
+                    # content_chunk is the string extracted by _aitopia_extractor
+                    if content_chunk and isinstance(content_chunk, str):
+                        streaming_text += content_chunk
+                        resp = dict(text=content_chunk)
+                        yield resp if not raw else content_chunk
+
+            except CurlError as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {str(e)}") from e
+            except Exception as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed: {str(e)}")
+            finally:
+                # Update history after stream finishes or fails
+                if streaming_text:
                     self.last_response = {"text": streaming_text}
                     self.conversation.update_chat_history(prompt, streaming_text)
-                    
-            except requests.RequestException as e:
-                raise exceptions.FailedToGenerateResponseError(f"Request failed: {str(e)}")
 
         def for_non_stream():
             try:
-                response = requests.post(self.url, headers=self.headers, json=payload, timeout=self.timeout)
-                if response.status_code != 200:
-                    raise exceptions.FailedToGenerateResponseError(
-                        f"Request failed with status code {response.status_code}"
-                    )
+                response = self.session.post(
+                    self.url, headers=self.headers, json=payload, timeout=self.timeout,
+                    impersonate="chrome120" # Add impersonate
+                )
+                response.raise_for_status()
+
+                response_text_raw = response.text # Get raw text
+
+                # Use sanitize_stream to parse the non-streaming JSON response
+                # Assuming non-stream uses the GPT format based on original code
+                processed_stream = sanitize_stream(
+                    data=response_text_raw,
+                    to_json=True, # Parse the whole text as JSON
+                    intro_value=None,
+                    content_extractor=lambda chunk: chunk.get("choices", [{}])[0].get("message", {}).get("content") if isinstance(chunk, dict) else None,
+                    yield_raw_on_error=False
+                )
+                # Extract the single result
+                content = next(processed_stream, None)
+                content = content if isinstance(content, str) else "" # Ensure it's a string
 
-                response_data = response.json()
-                if 'choices' in response_data and len(response_data['choices']) > 0:
-                    content = response_data['choices'][0].get('message', {}).get('content', '')
+                if content: # Check if content was successfully extracted
                     self.last_response = {"text": content}
                     self.conversation.update_chat_history(prompt, content)
                     return {"text": content}
                 else:
-                    raise exceptions.FailedToGenerateResponseError("No response content found")
+                    raise exceptions.FailedToGenerateResponseError("No response content found or failed to parse")
+            except CurlError as e:
+                raise exceptions.FailedToGenerateResponseError(f"Request failed (CurlError): {e}") from e
             except Exception as e:
                 raise exceptions.FailedToGenerateResponseError(f"Request failed: {e}")