webscout 8.2.2__py3-none-any.whl → 8.2.7__py3-none-any.whl

webscout/webscout_search_async.py

@@ -11,7 +11,7 @@ from time import time
 from types import TracebackType
 from typing import Any, Dict, List, Optional, Type, Union, cast, AsyncIterator
 
-import httpx
+import curl_cffi.requests
 from lxml.etree import _Element
 from lxml.html import HTMLParser as LHTMLParser
 from lxml.html import document_fromstring
@@ -31,18 +31,14 @@ from .utils import (
 class AsyncWEBS:
     """Asynchronous webscout class to get search results."""
 
+    # curl_cffi supports different browser versions than httpx
     _impersonates = (
-        "chrome_100", "chrome_101", "chrome_104", "chrome_105", "chrome_106", "chrome_107",
-        "chrome_108", "chrome_109", "chrome_114", "chrome_116", "chrome_117", "chrome_118",
-        "chrome_119", "chrome_120", "chrome_123", "chrome_124", "chrome_126", "chrome_127",
-        "chrome_128", "chrome_129", "chrome_130", "chrome_131", "chrome_133",
-        "safari_ios_16.5", "safari_ios_17.2", "safari_ios_17.4.1", "safari_ios_18.1.1",
-        "safari_15.3", "safari_15.5", "safari_15.6.1", "safari_16", "safari_16.5",
-        "safari_17.0", "safari_17.2.1", "safari_17.4.1", "safari_17.5",
-        "safari_18", "safari_18.2",
-        "safari_ipad_18",
-        "edge_101", "edge_122", "edge_127", "edge_131",
-        "firefox_109", "firefox_117", "firefox_128", "firefox_133", "firefox_135",
+        "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
+        "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133a",
+        "chrome99_android", "chrome131_android",
+        "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios", "safari18_0", "safari18_0_ios",
+        "edge99", "edge101",
+        "firefox133", "firefox135",
     )
     _impersonates_os = ("android", "ios", "linux", "macos", "windows")
     _chat_models = {
@@ -94,11 +90,14 @@ class AsyncWEBS:
         self.headers = headers if headers else {}
         self.headers.update(default_headers)
 
-        self.client = httpx.AsyncClient(
+        # Use curl_cffi AsyncSession instead of httpx.AsyncClient
+        impersonate_browser = choice(self._impersonates)
+        self.timeout = timeout
+        self.client = curl_cffi.requests.AsyncSession(
             headers=self.headers,
-            proxies=self.proxy,
+            proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None,
             timeout=timeout,
-            follow_redirects=False,
+            impersonate=impersonate_browser,
             verify=verify,
         )
         self.sleep_timestamp = 0.0
@@ -147,17 +146,33 @@ class AsyncWEBS:
         """Make HTTP request with proper rate limiting."""
         await self._sleep()
         try:
-            resp = await self.client.request(
-                method,
-                url,
-                params=params,
-                content=content,
-                data=data,
-                headers=headers,
-                cookies=cookies,
-                json=json,
-                timeout=timeout or self.timeout,
-            )
+            # curl_cffi doesn't accept cookies=True in request methods
+            request_kwargs = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+            }
+
+            # Add cookies if they're a dict, not a bool
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+
+            if method == "GET":
+                # curl_cffi uses data instead of content
+                if content:
+                    request_kwargs["data"] = content
+                resp = await self.client.get(url, **request_kwargs)
+            elif method == "POST":
+                # handle both data and content
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = await self.client.post(url, **request_kwargs)
+            else:
+                # handle both data and content
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = await self.client.request(method, url, **request_kwargs)
         except Exception as ex:
             if "time" in str(ex).lower():
                 raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
@@ -278,7 +293,8 @@ class AsyncWEBS:
                 self._chat_vqd_hash = resp.headers.get("x-vqd-hash-1", "")
                 chunks = []
 
-                async for chunk in resp.aiter_bytes():
+                # curl_cffi uses aiter_content instead of aiter_bytes
+                async for chunk in resp.aiter_content(chunk_size=1024):
                     lines = chunk.split(b"data:")
                     for line in lines:
                         if line := line.strip():
@@ -286,20 +302,24 @@ class AsyncWEBS:
                                 break
                             if line == b"[DONE][LIMIT_CONVERSATION]":
                                 raise ConversationLimitException("ERR_CONVERSATION_LIMIT")
-                            x = json_loads(line)
-                            if isinstance(x, dict):
-                                if x.get("action") == "error":
-                                    err_message = x.get("type", "")
-                                    if x.get("status") == 429:
-                                        raise (
-                                            ConversationLimitException(err_message)
-                                            if err_message == "ERR_CONVERSATION_LIMIT"
-                                            else RatelimitE(err_message)
-                                        )
-                                    raise WebscoutE(err_message)
-                                elif message := x.get("message"):
-                                    chunks.append(message)
-                                    yield message
+                            try:
+                                x = json_loads(line)
+                                if isinstance(x, dict):
+                                    if x.get("action") == "error":
+                                        err_message = x.get("type", "")
+                                        if x.get("status") == 429:
+                                            raise (
+                                                ConversationLimitException(err_message)
+                                                if err_message == "ERR_CONVERSATION_LIMIT"
+                                                else RatelimitE(err_message)
+                                            )
+                                        raise WebscoutE(err_message)
+                                    elif message := x.get("message"):
+                                        chunks.append(message)
+                                        yield message
+                            except Exception:
+                                # Skip invalid JSON data
+                                continue
 
                 # If we get here, the request was successful
                 result = "".join(chunks)
@@ -421,7 +441,8 @@ class AsyncWEBS:
             if b"No  results." in resp_content:
                 return results
 
-            tree = document_fromstring(resp_content, self.parser)
+            # curl_cffi returns bytes, not a file-like object
+            tree = document_fromstring(resp_content)
             elements = tree.xpath("//div[h2]")
             if not isinstance(elements, list):
                 return results
@@ -494,7 +515,8 @@ class AsyncWEBS:
             if b"No more results." in resp_content:
                 return results
 
-            tree = document_fromstring(resp_content, self.parser)
+            # curl_cffi returns bytes, not a file-like object
+            tree = document_fromstring(resp_content)
             elements = tree.xpath("//table[last()]//tr")
             if not isinstance(elements, list):
                 return results
@@ -583,21 +605,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().images,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            size,
-            color,
-            type_image,
-            layout,
-            license_image,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aimages method is not implemented yet")
 
     async def avideos(
         self,
@@ -630,19 +639,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().videos,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            resolution,
-            duration,
-            license_videos,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("avideos method is not implemented yet")
 
     async def anews(
         self,
@@ -669,16 +667,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().news,
-            keywords,
-            region,
-            safesearch,
-            timelimit,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("anews method is not implemented yet")
 
     async def aanswers(
         self,
@@ -697,12 +687,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().answers,
-            keywords,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aanswers method is not implemented yet")
 
     async def asuggestions(
         self,
@@ -723,13 +709,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().suggestions,
-            keywords,
-            region,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("asuggestions method is not implemented yet")
 
     async def amaps(
         self,
@@ -771,23 +752,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().maps,
-            keywords,
-            place,
-            street,
-            city,
-            county,
-            state,
-            country,
-            postalcode,
-            latitude,
-            longitude,
-            radius,
-            max_results,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("amaps method is not implemented yet")
 
     async def atranslate(
         self,
@@ -810,14 +776,8 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().translate,
-            keywords,
-            from_,
-            to,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("atranslate method is not implemented yet")
 
     async def aweather(
         self,
@@ -868,10 +828,5 @@ class AsyncWEBS:
             RatelimitE: Inherits from WebscoutE, raised for exceeding API request rate limits.
             TimeoutE: Inherits from WebscoutE, raised for API request timeouts.
         """
-        result = await self._loop.run_in_executor(
-            self._executor,
-            super().weather,
-            location,
-            language,
-        )
-        return result
+        # These methods are not implemented in the async version yet
+        raise NotImplementedError("aweather method is not implemented yet")

webscout/yep_search.py

@@ -1,12 +1,13 @@
-import cloudscraper
+from curl_cffi.requests import Session
 from urllib.parse import urlencode
 from webscout.litagent import LitAgent
 from typing import List, Dict, Optional, Tuple
 from concurrent.futures import ThreadPoolExecutor
 import json
+
 class YepSearch:
     """Yep.com search class to get search results."""
-    
+
     _executor: ThreadPoolExecutor = ThreadPoolExecutor()
 
     def __init__(
@@ -14,47 +15,54 @@ class YepSearch:
         timeout: int = 20,
         proxies: Dict[str, str] | None = None,
         verify: bool = True,
+        impersonate: str = "chrome110"
     ):
         """Initialize YepSearch.
-        
+
         Args:
             timeout: Timeout value for the HTTP client. Defaults to 20.
             proxies: Proxy configuration for requests. Defaults to None.
             verify: Verify SSL certificates. Defaults to True.
+            impersonate: Browser profile to impersonate for curl_cffi. Defaults to "chrome110".
         """
         self.base_url = "https://api.yep.com/fs/2/search"
         self.timeout = timeout
-        self.session = cloudscraper.create_scraper()
+        # Initialize curl_cffi session
+        self.session = Session(
+            proxies=proxies,
+            verify=verify,
+            impersonate=impersonate,
+            timeout=timeout # Set timeout directly in session
+        )
         self.session.headers.update({
             "Accept": "*/*",
             "Accept-Language": "en-US,en;q=0.9,en-IN;q=0.8",
             "DNT": "1",
             "Origin": "https://yep.com",
             "Referer": "https://yep.com/",
+            # Sec-Ch-Ua headers are often handled by impersonate, but keeping them might be safer
             "Sec-Ch-Ua": '"Not(A:Brand";v="99", "Microsoft Edge";v="133", "Chromium";v="133"',
             "Sec-Ch-Ua-Mobile": "?0",
             "Sec-Ch-Ua-Platform": '"Windows"',
             "Sec-Fetch-Dest": "empty",
             "Sec-Fetch-Mode": "cors",
             "Sec-Fetch-Site": "same-site",
-            "User-Agent": LitAgent().random()
+            "User-Agent": LitAgent().random() # Keep custom User-Agent or rely on impersonate
         })
-        if proxies:
-            self.session.proxies.update(proxies)
-        self.session.verify = verify
+        # Proxies and verify are handled by the Session constructor now
 
     def _remove_html_tags(self, text: str) -> str:
         """Remove HTML tags from text using simple string manipulation.
-        
+
         Args:
             text: String containing HTML tags
-            
+
         Returns:
             Clean text without HTML tags
         """
         result = ""
         in_tag = False
-        
+
         for char in text:
             if char == '<':
                 in_tag = True
@@ -62,7 +70,7 @@ class YepSearch:
                 in_tag = False
             elif not in_tag:
                 result += char
-                
+
         # Replace common HTML entities
         replacements = {
             '&nbsp;': ' ',
@@ -72,21 +80,21 @@ class YepSearch:
             '&quot;': '"',
             '&apos;': "'",
         }
-        
+
         for entity, replacement in replacements.items():
             result = result.replace(entity, replacement)
-            
+
         return result.strip()
 
     def format_results(self, raw_results: dict) -> List[Dict]:
         """Format raw API results into a consistent structure."""
         formatted_results = []
-        
+
         if not raw_results or len(raw_results) < 2:
             return formatted_results
 
         results = raw_results[1].get('results', [])
-        
+
         for result in results:
             formatted_result = {
                 "title": self._remove_html_tags(result.get("title", "")),
@@ -97,7 +105,7 @@ class YepSearch:
                 "type": result.get("type", "organic"),
                 "first_seen": result.get("first_seen", None)
             }
-            
+
             # Add sitelinks if they exist
             if "sitelinks" in result:
                 sitelinks = []
@@ -105,7 +113,7 @@ class YepSearch:
                     sitelinks.extend(result["sitelinks"]["full"])
                 if "short" in result["sitelinks"]:
                     sitelinks.extend(result["sitelinks"]["short"])
-                
+
                 if sitelinks:
                     formatted_result["sitelinks"] = [
                         {
@@ -114,9 +122,9 @@ class YepSearch:
                         }
                         for link in sitelinks
                     ]
-            
+
             formatted_results.append(formatted_result)
-        
+
         return formatted_results
 
     def text(
@@ -154,20 +162,25 @@ class YepSearch:
             "safeSearch": safe_setting,
             "type": "web"
         }
-        
+
         url = f"{self.base_url}?{urlencode(params)}"
         try:
-            response = self.session.get(url, timeout=self.timeout)
+            # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             raw_results = response.json()
-            
+
             formatted_results = self.format_results(raw_results)
-            
+
             if max_results:
                 return formatted_results[:max_results]
             return formatted_results
         except Exception as e:
-            raise Exception(f"Yep search failed: {str(e)}")
+            # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep search failed: {str(e)}")
 
     def images(
         self,
@@ -210,23 +223,24 @@ class YepSearch:
             "safeSearch": safe_setting,
             "type": "images"
         }
-        
+
         url = f"{self.base_url}?{urlencode(params)}"
         try:
-            response = self.session.get(url, timeout=self.timeout)
+            # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             raw_results = response.json()
-            
+
             if not raw_results or len(raw_results) < 2:
                 return []
 
             formatted_results = []
             results = raw_results[1].get('results', [])
-            
+
             for result in results:
                 if result.get("type") != "Image":
                     continue
-                
+
                 formatted_result = {
                     "title": self._remove_html_tags(result.get("title", "")),
                     "image": result.get("image_id", ""),
@@ -236,19 +250,23 @@ class YepSearch:
                     "width": result.get("width", 0),
                     "source": result.get("visual_url", "")
                 }
-                
+
                 # Add high-res thumbnail if available
                 if "srcset" in result:
                     formatted_result["thumbnail_hd"] = result["srcset"].split(",")[1].strip().split(" ")[0]
-                
+
                 formatted_results.append(formatted_result)
-            
+
             if max_results:
                 return formatted_results[:max_results]
             return formatted_results
-        
+
         except Exception as e:
-            raise Exception(f"Yep image search failed: {str(e)}")
+             # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep image search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep image search failed: {str(e)}")
 
     def suggestions(
         self,
@@ -275,23 +293,55 @@ class YepSearch:
             "type": "web",
             "gl": region
         }
-        
+
         url = f"https://api.yep.com/ac/?{urlencode(params)}"
-        
+
         try:
-            response = self.session.get(url, timeout=self.timeout)
+             # Use the session timeout defined in __init__
+            response = self.session.get(url)
             response.raise_for_status()
             data = response.json()
             # Return suggestions list if response format is valid
             if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
                 return data[1]
             return []
-        
+
         except Exception as e:
-            raise Exception(f"Yep suggestions failed: {str(e)}")
+             # Provide more specific error context if possible
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep suggestions failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep suggestions failed: {str(e)}")
 
 
 if __name__ == "__main__":
-    yep = YepSearch()
-    r = yep.suggestions("hi", region="all")
-    print(r)
+    from rich import print
+    yep = YepSearch(
+        timeout=20,  # Optional: Set custom timeout
+        proxies=None,  # Optional: Use proxies
+        verify=True   # Optional: SSL verification
+    )
+
+    # Text Search
+    text_results = yep.text(
+        keywords="artificial intelligence",
+        region="all",           # Optional: Region for results
+        safesearch="moderate",  # Optional: "on", "moderate", "off"
+        max_results=10          # Optional: Limit number of results
+    )
+
+    # Image Search
+    image_results = yep.images(
+        keywords="nature photography",
+        region="all",
+        safesearch="moderate",
+        max_results=10
+    )
+
+    # Get search suggestions
+    suggestions = yep.suggestions("hist")
+    print(text_results)
+    print("---" * 30)
+    print(image_results)
+    print("---" * 30)
+    print(suggestions)