webscout 8.2.3__py3-none-any.whl → 8.2.4__py3-none-any.whl

webscout/webscout_search_async.py

@@ -11,7 +11,7 @@ from time import time
 from types import TracebackType
 from typing import Any, Dict, List, Optional, Type, Union, cast, AsyncIterator
 
-import httpx
+import curl_cffi.requests
 from lxml.etree import _Element
 from lxml.html import HTMLParser as LHTMLParser
 from lxml.html import document_fromstring
@@ -31,18 +31,14 @@ from .utils import (
 class AsyncWEBS:
     """Asynchronous webscout class to get search results."""
 
+    # curl_cffi supports different browser versions than httpx
     _impersonates = (
-        "chrome_100", "chrome_101", "chrome_104", "chrome_105", "chrome_106", "chrome_107",
-        "chrome_108", "chrome_109", "chrome_114", "chrome_116", "chrome_117", "chrome_118",
-        "chrome_119", "chrome_120", "chrome_123", "chrome_124", "chrome_126", "chrome_127",
-        "chrome_128", "chrome_129", "chrome_130", "chrome_131", "chrome_133",
-        "safari_ios_16.5", "safari_ios_17.2", "safari_ios_17.4.1", "safari_ios_18.1.1",
-        "safari_15.3", "safari_15.5", "safari_15.6.1", "safari_16", "safari_16.5",
-        "safari_17.0", "safari_17.2.1", "safari_17.4.1", "safari_17.5",
-        "safari_18", "safari_18.2",
-        "safari_ipad_18",
-        "edge_101", "edge_122", "edge_127", "edge_131",
-        "firefox_109", "firefox_117", "firefox_128", "firefox_133", "firefox_135",
+        "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
+        "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133a",
+        "chrome99_android", "chrome131_android",
+        "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios", "safari18_0", "safari18_0_ios",
+        "edge99", "edge101",
+        "firefox133", "firefox135",
     )
     _impersonates_os = ("android", "ios", "linux", "macos", "windows")
     _chat_models = {
@@ -94,11 +90,14 @@ class AsyncWEBS:
         self.headers = headers if headers else {}
         self.headers.update(default_headers)
 
-        self.client = httpx.AsyncClient(
+        # Use curl_cffi AsyncSession instead of httpx.AsyncClient
+        impersonate_browser = choice(self._impersonates)
+        self.timeout = timeout
+        self.client = curl_cffi.requests.AsyncSession(
             headers=self.headers,
-            proxies=self.proxy,
+            proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None,
             timeout=timeout,
-            follow_redirects=False,
+            impersonate=impersonate_browser,
             verify=verify,
         )
         self.sleep_timestamp = 0.0
@@ -147,17 +146,33 @@ class AsyncWEBS:
         """Make HTTP request with proper rate limiting."""
         await self._sleep()
         try:
-            resp = await self.client.request(
-                method,
-                url,
-                params=params,
-                content=content,
-                data=data,
-                headers=headers,
-                cookies=cookies,
-                json=json,
-                timeout=timeout or self.timeout,
-            )
+            # curl_cffi doesn't accept cookies=True in request methods
+            request_kwargs = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+            }
+
+            # Add cookies if they're a dict, not a bool
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+
+            if method == "GET":
+                # curl_cffi uses data instead of content
+                if content:
+                    request_kwargs["data"] = content
+                resp = await self.client.get(url, **request_kwargs)
+            elif method == "POST":
+                # handle both data and content
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = await self.client.post(url, **request_kwargs)
+            else:
+                # handle both data and content
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = await self.client.request(method, url, **request_kwargs)
         except Exception as ex:
             if "time" in str(ex).lower():
                 raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
@@ -278,7 +293,8 @@ class AsyncWEBS:
                 self._chat_vqd_hash = resp.headers.get("x-vqd-hash-1", "")
                 chunks = []
 
-                async for chunk in resp.aiter_bytes():
+                # curl_cffi uses aiter_content instead of aiter_bytes
+                async for chunk in resp.aiter_content(chunk_size=1024):
                     lines = chunk.split(b"data:")
                     for line in lines:
                         if line := line.strip():
@@ -286,20 +302,24 @@ class AsyncWEBS:
                                 break
                             if line == b"[DONE][LIMIT_CONVERSATION]":
                                 raise ConversationLimitException("ERR_CONVERSATION_LIMIT")
-                            x = json_loads(line)
-                            if isinstance(x, dict):
-                                if x.get("action") == "error":
-                                    err_message = x.get("type", "")
-                                    if x.get("status") == 429:
-                                        raise (
-                                            ConversationLimitException(err_message)
-                                            if err_message == "ERR_CONVERSATION_LIMIT"
-                                            else RatelimitE(err_message)
-                                        )
-                                    raise WebscoutE(err_message)
-                                elif message := x.get("message"):
-                                    chunks.append(message)
-                                    yield message
+                            try:
+                                x = json_loads(line)
+                                if isinstance(x, dict):
+                                    if x.get("action") == "error":
+                                        err_message = x.get("type", "")
+                                        if x.get("status") == 429:
+                                            raise (
+                                                ConversationLimitException(err_message)
+                                                if err_message == "ERR_CONVERSATION_LIMIT"
+                                                else RatelimitE(err_message)
+                                            )
+                                        raise WebscoutE(err_message)
+                                    elif message := x.get("message"):
+                                        chunks.append(message)
+                                        yield message
+                            except Exception:
+                                # Skip invalid JSON data
+                                continue
 
                 # If we get here, the request was successful
                 result = "".join(chunks)
@@ -421,7 +441,8 @@ class AsyncWEBS:
             if b"No  results." in resp_content:
                 return results
 
-            tree = document_fromstring(resp_content, self.parser)
+            # curl_cffi returns bytes, not a file-like object
+            tree = document_fromstring(resp_content)
             elements = tree.xpath("//div[h2]")
             if not isinstance(elements, list):
                 return results
@@ -494,7 +515,8 @@ class AsyncWEBS:
             if b"No more results." in resp_content:
                 return results
 
-            tree = document_fromstring(resp_content, self.parser)
+            # curl_cffi returns bytes, not a file-like object
+            tree = document_fromstring(resp_content)
             elements = tree.xpath("//table[last()]//tr")
             if not isinstance(elements, list):
                 return results
@@ -583,21 +605,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().images,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            size,
-            color,
-            type_image,
-            layout,
-            license_image,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aimages method is not implemented yet")
 
     async def avideos(
         self,
@@ -630,19 +639,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().videos,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            resolution,
-            duration,
-            license_videos,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("avideos method is not implemented yet")
 
     async def anews(
         self,
@@ -669,16 +667,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().news,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("anews method is not implemented yet")
 
     async def aanswers(
         self,
@@ -697,12 +687,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().answers,
-            keywords,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aanswers method is not implemented yet")
 
     async def asuggestions(
         self,
@@ -723,13 +709,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().suggestions,
-            keywords,
-            region,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("asuggestions method is not implemented yet")
 
     async def amaps(
         self,
@@ -771,23 +752,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().maps,
-            keywords,
-            place,
-            street,
-            city,
-            county,
-            state,
-            country,
-            postalcode,
-            latitude,
-            longitude,
-            radius,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("amaps method is not implemented yet")
 
     async def atranslate(
         self,
@@ -810,14 +776,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().translate,
-            keywords,
-            from_,
-            to,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("atranslate method is not implemented yet")
 
     async def aweather(
         self,
@@ -868,10 +828,5 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().weather,
-            location,
-            language,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aweather method is not implemented yet")

webscout/yep_search.py

@@ -1,12 +1,13 @@
-import cloudscraper
+from curl_cffi.requests import Session
 from urllib.parse import urlencode
 from webscout.litagent import LitAgent
 from typing import List, Dict, Optional, Tuple
 from concurrent.futures import ThreadPoolExecutor
 import json
+
 class YepSearch:
     """Yep.com search class to get search results."""
-    
+
     _executor: ThreadPoolExecutor = ThreadPoolExecutor()
 
     def __init__(
@@ -14,47 +15,54 @@ class YepSearch:
         timeout: int = 20,
         proxies: Dict[str, str] | None = None,
         verify: bool = True,
+        impersonate: str = "chrome110"
     ):
         """Initialize YepSearch.
-        
+
         Args:
             timeout: Timeout value for the HTTP client. Defaults to 20.
             proxies: Proxy configuration for requests. Defaults to None.
             verify: Verify SSL certificates. Defaults to True.
+            impersonate: Browser profile to impersonate for curl_cffi. Defaults to "chrome110".
         """
         self.base_url = "https://api.yep.com/fs/2/search"
         self.timeout = timeout
-        self.session = cloudscraper.create_scraper()
+        # Initialize curl_cffi session
+        self.session = Session(
+            proxies=proxies,
+            verify=verify,
+            impersonate=impersonate,
+            timeout=timeout # Set timeout directly in session
+        )
         self.session.headers.update({
             "Accept": "*/*",
             "Accept-Language": "en-US,en;q=0.9,en-IN;q=0.8",
             "DNT": "1",
             "Origin": "https://yep.com",
             "Referer": "https://yep.com/",
+            # Sec-Ch-Ua headers are often handled by impersonate, but keeping them might be safer
             "Sec-Ch-Ua": '"Not(A:Brand";v="99", "Microsoft Edge";v="133", "Chromium";v="133"',
             "Sec-Ch-Ua-Mobile": "?0",
             "Sec-Ch-Ua-Platform": '"Windows"',
             "Sec-Fetch-Dest": "empty",
             "Sec-Fetch-Mode": "cors",
             "Sec-Fetch-Site": "same-site",
-            "User-Agent": LitAgent().random()
+            "User-Agent": LitAgent().random() # Keep custom User-Agent or rely on impersonate
         })
-        if proxies:
-            self.session.proxies.update(proxies)
-        self.session.verify = verify
+        # Proxies and verify are handled by the Session constructor now
 
     def _remove_html_tags(self, text: str) -> str:
         """Remove HTML tags from text using simple string manipulation.
-        
+
         Args:
             text: String containing HTML tags
-            
+
         Returns:
             Clean text without HTML tags
         """
         result = ""
         in_tag = False
-        
+
         for char in text:
             if char == '<':
                 in_tag = True
@@ -62,7 +70,7 @@ class YepSearch:
                 in_tag = False
             elif not in_tag:
                 result += char
-                
+
         # Replace common HTML entities
         replacements = {
             '&nbsp;': ' ',
@@ -72,21 +80,21 @@ class YepSearch:
             '&quot;': '"',
             '&apos;': "'",
         }
-        
+
         for entity, replacement in replacements.items():
             result = result.replace(entity, replacement)
-            
+
         return result.strip()
 
     def format_results(self, raw_results: dict) -> List[Dict]:
         """Format raw API results into a consistent structure."""
         formatted_results = []
-        
+
         if not raw_results or len(raw_results) < 2:
             return formatted_results
 
         results = raw_results[1].get('results', [])
-        
+
         for result in results:
             formatted_result = {
                 "title": self._remove_html_tags(result.get("title", "")),
@@ -97,7 +105,7 @@ class YepSearch:
                 "type": result.get("type", "organic"),
                 "first_seen": result.get("first_seen", None)
             }
-            
+
             # Add sitelinks if they exist
             if "sitelinks" in result:
                 sitelinks = []
@@ -105,7 +113,7 @@ class YepSearch:
                     sitelinks.extend(result["sitelinks"]["full"])
                 if "short" in result["sitelinks"]:
                     sitelinks.extend(result["sitelinks"]["short"])
-                
+
                 if sitelinks:
                     formatted_result["sitelinks"] = [
                         {
@@ -114,9 +122,9 @@ class YepSearch:
                         }
                         for link in sitelinks
                     ]
-            
+
             formatted_results.append(formatted_result)
-        
+
         return formatted_results
 
     def text(
@@ -154,20 +162,25 @@ class YepSearch:
             "safeSearch": safe_setting,
             "type": "web"
         }
-        
+
         url = f"{self.base_url}?{urlencode(params)}"
         try:
-            response = self.session.get(url, timeout=self.timeout)
+            # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             raw_results = response.json()
-            
+
             formatted_results = self.format_results(raw_results)
-            
+
             if max_results:
                 return formatted_results[:max_results]
             return formatted_results
         except Exception as e:
-            raise Exception(f"Yep search failed: {str(e)}")
+            # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep search failed: {str(e)}")
 
     def images(
         self,
@@ -210,23 +223,24 @@ class YepSearch:
             "safeSearch": safe_setting,
             "type": "images"
         }
-        
+
         url = f"{self.base_url}?{urlencode(params)}"
         try:
-            response = self.session.get(url, timeout=self.timeout)
+            # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             raw_results = response.json()
-            
+
             if not raw_results or len(raw_results) < 2:
                 return []
 
             formatted_results = []
             results = raw_results[1].get('results', [])
-            
+
             for result in results:
                 if result.get("type") != "Image":
                     continue
-                
+
                 formatted_result = {
                     "title": self._remove_html_tags(result.get("title", "")),
                     "image": result.get("image_id", ""),
@@ -236,19 +250,23 @@ class YepSearch:
                     "width": result.get("width", 0),
                     "source": result.get("visual_url", "")
                 }
-                
+
                 # Add high-res thumbnail if available
                 if "srcset" in result:
                     formatted_result["thumbnail_hd"] = result["srcset"].split(",")[1].strip().split(" ")[0]
-                
+
                 formatted_results.append(formatted_result)
-            
+
             if max_results:
                 return formatted_results[:max_results]
             return formatted_results
-        
+
         except Exception as e:
-            raise Exception(f"Yep image search failed: {str(e)}")
+             # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep image search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep image search failed: {str(e)}")
 
     def suggestions(
         self,
@@ -275,23 +293,55 @@ class YepSearch:
             "type": "web",
             "gl": region
         }
-        
+
         url = f"https://api.yep.com/ac/?{urlencode(params)}"
-        
+
         try:
-            response = self.session.get(url, timeout=self.timeout)
+             # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             data = response.json()
             # Return suggestions list if response format is valid
             if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
                 return data[1]
             return []
-        
+
         except Exception as e:
-            raise Exception(f"Yep suggestions failed: {str(e)}")
+             # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep suggestions failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep suggestions failed: {str(e)}")
 
 
 if __name__ == "__main__":
-    yep = YepSearch()
-    r = yep.suggestions("hi", region="all")
-    print(r)
+    from rich import print
+    yep = YepSearch(
+        timeout=20,  # Optional: Set custom timeout
+        proxies=None,  # Optional: Use proxies
+        verify=True   # Optional: SSL verification
+    )
+
+    # Text Search
+    text_results = yep.text(
+        keywords="artificial intelligence",
+        region="all",           # Optional: Region for results
+        safesearch="moderate",  # Optional: "on", "moderate", "off"
+        max_results=10          # Optional: Limit number of results
+    )
+
+    # Image Search
+    image_results = yep.images(
+        keywords="nature photography",
+        region="all",
+        safesearch="moderate",
+        max_results=10
+    )
+
+    # Get search suggestions
+    suggestions = yep.suggestions("hist")
+    print(text_results)
+    print("---" * 30)
+    print(image_results)
+    print("---" * 30)
+    print(suggestions)

{webscout-8.2.3.dist-info → webscout-8.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: webscout
-Version: 8.2.3
+Version: 8.2.4
 Summary: Search for anything using Google, DuckDuckGo, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
 Author: OEvortex
 Author-email: helpingai5@gmail.com
@@ -41,7 +41,7 @@ Requires-Dist: pip
 Requires-Dist: nodriver
 Requires-Dist: mistune
 Requires-Dist: tenacity
-Requires-Dist: curl-cffi
+Requires-Dist: curl_cffi
 Requires-Dist: nest-asyncio
 Requires-Dist: websocket-client
 Requires-Dist: colorama
@@ -61,23 +61,35 @@ Requires-Dist: html5lib
 Requires-Dist: aiofiles
 Requires-Dist: openai
 Requires-Dist: prompt-toolkit
-Requires-Dist: primp
-Requires-Dist: pyreqwest-impersonate
-Requires-Dist: gradio-client
+Requires-Dist: pyreqwest_impersonate
+Requires-Dist: gradio_client
 Requires-Dist: psutil
 Requires-Dist: aiohttp
+Provides-Extra: dev
+Requires-Dist: ruff>=0.1.6; extra == "dev"
+Requires-Dist: pytest>=7.4.2; extra == "dev"
 Provides-Extra: local
 Requires-Dist: llama-cpp-python; extra == "local"
 Requires-Dist: fastapi; extra == "local"
 Requires-Dist: uvicorn; extra == "local"
 Requires-Dist: rich; extra == "local"
 Requires-Dist: typer; extra == "local"
-Requires-Dist: huggingface-hub; extra == "local"
+Requires-Dist: huggingface_hub; extra == "local"
 Requires-Dist: pydantic; extra == "local"
 Requires-Dist: requests; extra == "local"
-Provides-Extra: dev
-Requires-Dist: ruff>=0.1.6; extra == "dev"
-Requires-Dist: pytest>=7.4.2; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 <div align="center">
   <a href="https://github.com/OEvortex/Webscout">