webscout 2025.10.11__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/search/engines/yep/base.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from ....litagent import LitAgent
+from curl_cffi.requests import Session
+
+
+class YepBase:
+    """Base class for Yep search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 20,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        impersonate: str = "chrome110",
+    ):
+        self.base_url = "https://api.yep.com/fs/2/search"
+        self.timeout = timeout
+        self.session = Session(
+            proxies=proxies,
+            verify=verify,
+            impersonate=impersonate,
+            timeout=timeout,
+        )
+        self.session.headers.update(
+            {
+                **LitAgent().generate_fingerprint(),
+                "Origin": "https://yep.com",
+                "Referer": "https://yep.com/",
+            }
+        )
+

webscout/search/engines/yep/images.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepImages(YepBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "images"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            if not raw_results or len(raw_results) < 2:
+                return []
+
+            formatted_results = []
+            results = raw_results[1].get('results', [])
+
+            for result in results:
+                if result.get("type") != "Image":
+                    continue
+
+                formatted_result = {
+                    "title": self._remove_html_tags(result.get("title", "")),
+                    "image": result.get("image_id", ""),
+                    "thumbnail": result.get("src", ""),
+                    "url": result.get("host_page", ""),
+                    "height": result.get("height", 0),
+                    "width": result.get("width", 0),
+                    "source": result.get("visual_url", "")
+                }
+
+                if "srcset" in result:
+                    formatted_result["thumbnail_hd"] = result["srcset"].split(",")[1].strip().split(" ")[0]
+
+                formatted_results.append(formatted_result)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep image search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep image search failed: {str(e)}")
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+

webscout/search/engines/yep/suggestions.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import List
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepSuggestions(YepBase):
+    def run(self, *args, **kwargs) -> List[str]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+
+        params = {
+            "query": keywords,
+            "type": "web",
+            "gl": region
+        }
+
+        url = f"https://api.yep.com/ac/?{urlencode(params)}"
+
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            data = response.json()
+            if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
+                return data[1]
+            return []
+
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep suggestions failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep suggestions failed: {str(e)}")
+

webscout/search/engines/yep/text.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepSearch(YepBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "web"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            formatted_results = self.format_results(raw_results)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep search failed: {str(e)}")
+
+    def format_results(self, raw_results: dict) -> List[Dict]:
+        formatted_results = []
+
+        if not raw_results or len(raw_results) < 2:
+            return formatted_results
+
+        results = raw_results[1].get('results', [])
+
+        for result in results:
+            formatted_result = {
+                "title": self._remove_html_tags(result.get("title", "")),
+                "href": result.get("url", ""),
+                "body": self._remove_html_tags(result.get("snippet", "")),
+                "source": result.get("visual_url", ""),
+                "position": len(formatted_results) + 1,
+                "type": result.get("type", "organic"),
+                "first_seen": result.get("first_seen", None)
+            }
+
+            if "sitelinks" in result:
+                sitelinks = []
+                if "full" in result["sitelinks"]:
+                    sitelinks.extend(result["sitelinks"]["full"])
+                if "short" in result["sitelinks"]:
+                    sitelinks.extend(result["sitelinks"]["short"])
+
+                if sitelinks:
+                    formatted_result["sitelinks"] = [
+                        {
+                            "title": self._remove_html_tags(link.get("title", "")),
+                            "href": link.get("url", "")
+                        }
+                        for link in sitelinks
+                    ]
+
+            formatted_results.append(formatted_result)
+
+        return formatted_results
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+

webscout/search/http_client.py

@@ -0,0 +1,156 @@
+"""HTTP client for search engines."""
+
+from __future__ import annotations
+
+import logging
+from random import choice
+from typing import Any, Literal
+
+try:
+    import trio  # noqa: F401
+except ImportError:
+    pass
+
+import curl_cffi.requests
+
+from ..exceptions import RatelimitE, TimeoutE, WebscoutE
+
+logger = logging.getLogger(__name__)
+
+
+class HttpClient:
+    """HTTP client wrapper for search engines."""
+    
+    # curl_cffi supported browser impersonations
+    _impersonates = (
+        "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
+        "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133a",
+        "chrome99_android", "chrome131_android",
+        "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios", "safari18_0", "safari18_0_ios",
+        "edge99", "edge101",
+        "firefox133", "firefox135",
+    )
+
+    def __init__(
+        self,
+        proxy: str | None = None,
+        timeout: int | None = 10,
+        verify: bool = True,
+        headers: dict[str, str] | None = None,
+    ) -> None:
+        """Initialize HTTP client.
+        
+        Args:
+            proxy: Proxy URL (supports http/https/socks5).
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+            headers: Default headers for requests.
+        """
+        self.proxy = proxy
+        self.timeout = timeout
+        self.verify = verify
+        
+        # Choose random browser to impersonate
+        impersonate_browser = choice(self._impersonates)
+        
+        # Initialize curl_cffi session
+        self.client = curl_cffi.requests.Session(
+            headers=headers or {},
+            proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None,
+            timeout=timeout,
+            impersonate=impersonate_browser,
+            verify=verify,
+        )
+    
+    def request(
+        self,
+        method: Literal["GET", "POST", "HEAD", "OPTIONS", "DELETE", "PUT", "PATCH"],
+        url: str,
+        params: dict[str, Any] | None = None,
+        data: dict[str, Any] | bytes | None = None,
+        json: Any = None,
+        headers: dict[str, str] | None = None,
+        cookies: dict[str, str] | None = None,
+        timeout: int | None = None,
+        **kwargs: Any,
+    ) -> curl_cffi.requests.Response:
+        """Make HTTP request.
+        
+        Args:
+            method: HTTP method.
+            url: Request URL.
+            params: URL parameters.
+            data: Request body data.
+            json: JSON data to send.
+            headers: Request headers.
+            cookies: Request cookies.
+            timeout: Request timeout (overrides default).
+            **kwargs: Additional arguments passed to curl_cffi.
+            
+        Returns:
+            Response object.
+            
+        Raises:
+            TimeoutE: Request timeout.
+            RatelimitE: Rate limit exceeded.
+            WebscoutE: Other request errors.
+        """
+        try:
+            request_kwargs: dict[str, Any] = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+                **kwargs,
+            }
+            
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+            
+            if data is not None:
+                request_kwargs["data"] = data
+            
+            resp = self.client.request(method, url, **request_kwargs)
+            
+            # Check response status
+            if resp.status_code == 200:
+                return resp
+            elif resp.status_code in (202, 301, 403, 400, 429, 418):
+                raise RatelimitE(f"{resp.url} {resp.status_code} Rate limit")
+            else:
+                raise WebscoutE(f"{resp.url} returned {resp.status_code}")
+                
+        except curl_cffi.requests.RequestException as ex:
+            if "time" in str(ex).lower() or "timeout" in str(ex).lower():
+                raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+            raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+    
+    def get(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
+        """Make GET request."""
+        return self.request("GET", url, **kwargs)
+    
+    def post(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
+        """Make POST request."""
+        return self.request("POST", url, **kwargs)
+    
+    def set_cookies(self, url: str, cookies: dict[str, str]) -> None:
+        """Set cookies for a domain.
+        
+        Args:
+            url: URL to set cookies for.
+            cookies: Cookie dictionary.
+        """
+        self.client.cookies.update(cookies)
+    
+    def close(self) -> None:
+        """Close the HTTP client."""
+        if hasattr(self.client, 'close'):
+            self.client.close()
+    
+    def __enter__(self) -> HttpClient:
+        """Context manager entry."""
+        return self
+    
+    def __exit__(self, *args: Any) -> None:
+        """Context manager exit."""
+        self.close()

webscout/search/results.py

@@ -0,0 +1,137 @@
+"""Result models for search engines."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class TextResult:
+    """Text search result."""
+    
+    title: str = ""
+    href: str = ""
+    body: str = ""
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "href": self.href,
+            "body": self.body,
+        }
+
+
+@dataclass
+class ImagesResult:
+    """Images search result."""
+    
+    title: str = ""
+    image: str = ""
+    thumbnail: str = ""
+    url: str = ""
+    height: int = 0
+    width: int = 0
+    source: str = ""
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "image": self.image,
+            "thumbnail": self.thumbnail,
+            "url": self.url,
+            "height": self.height,
+            "width": self.width,
+            "source": self.source,
+        }
+
+
+@dataclass
+class VideosResult:
+    """Videos search result."""
+    
+    content: str = ""
+    description: str = ""
+    duration: str = ""
+    embed_html: str = ""
+    embed_url: str = ""
+    image_token: str = ""
+    images: dict[str, str] = field(default_factory=dict)
+    provider: str = ""
+    published: str = ""
+    publisher: str = ""
+    statistics: dict[str, int] = field(default_factory=dict)
+    title: str = ""
+    uploader: str = ""
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "content": self.content,
+            "description": self.description,
+            "duration": self.duration,
+            "embed_html": self.embed_html,
+            "embed_url": self.embed_url,
+            "image_token": self.image_token,
+            "images": self.images,
+            "provider": self.provider,
+            "published": self.published,
+            "publisher": self.publisher,
+            "statistics": self.statistics,
+            "title": self.title,
+            "uploader": self.uploader,
+        }
+
+
+@dataclass
+class NewsResult:
+    """News search result."""
+    
+    date: str = ""
+    title: str = ""
+    body: str = ""
+    url: str = ""
+    image: str = ""
+    source: str = ""
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "date": self.date,
+            "title": self.title,
+            "body": self.body,
+            "url": self.url,
+            "image": self.image,
+            "source": self.source,
+        }
+
+
+@dataclass
+class BooksResult:
+    """Books search result."""
+    
+    title: str = ""
+    author: str = ""
+    href: str = ""
+    thumbnail: str = ""
+    year: str = ""
+    publisher: str = ""
+    language: str = ""
+    filesize: str = ""
+    extension: str = ""
+    
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "author": self.author,
+            "href": self.href,
+            "thumbnail": self.thumbnail,
+            "year": self.year,
+            "publisher": self.publisher,
+            "language": self.language,
+            "filesize": self.filesize,
+            "extension": self.extension,
+        }

webscout/search/yep_main.py

@@ -0,0 +1,44 @@
+"""Yep unified search interface."""
+
+from __future__ import annotations
+from typing import Dict, List, Optional
+from .base import BaseSearch
+from .engines.yep.text import YepSearch as YepTextSearch
+from .engines.yep.images import YepImages
+from .engines.yep.suggestions import YepSuggestions
+
+
+class YepSearch(BaseSearch):
+    """Unified Yep search interface."""
+
+    def text(self, keywords: str, region: str = "all", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YepTextSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def images(self, keywords: str, region: str = "all", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YepImages()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def suggestions(self, keywords: str, region: str = "all") -> List[str]:
+        search = YepSuggestions()
+        return search.run(keywords, region)
+
+    def videos(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Videos search not supported by Yep."""
+        raise NotImplementedError("Yep does not support video search")
+
+    def news(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """News search not supported by Yep."""
+        raise NotImplementedError("Yep does not support news search")
+
+    def answers(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Instant answers not supported by Yep."""
+        raise NotImplementedError("Yep does not support instant answers")
+
+    def maps(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Maps search not supported by Yep."""
+        raise NotImplementedError("Yep does not support maps search")
+
+    def translate(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Translation not supported by Yep."""
+        raise NotImplementedError("Yep does not support translation")

webscout/version.py

@@ -1,2 +1,2 @@
-__version__ = "2025.10.11"
+__version__ = "2025.10.13"
 __prog__ = "webscout"

webscout/version.py.bak

@@ -0,0 +1,2 @@
+__version__ = "2025.10.11"
+__prog__ = "webscout"

{webscout-2025.10.11.dist-info → webscout-2025.10.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webscout
-Version: 2025.10.11
+Version: 2025.10.13
 Summary: Search for anything using Google, DuckDuckGo, phind.com, Contains AI models, can transcribe yt videos, temporary email and phone number generation, has TTS support, webai (terminal gpt and open interpreter) and offline LLMs and more
 Author-email: OEvortex <helpingai5@gmail.com>
 License: HelpingAI
@@ -477,7 +477,7 @@ results = search("Python programming", num_results=5)
 
 <hr/>
 
-## 🦆 DuckDuckGo Search with WEBS and AsyncWEBS
+## 🦆 DuckDuckGo Search with WEBS
 
 Webscout provides powerful interfaces to DuckDuckGo's search capabilities through the `WEBS` and `AsyncWEBS` classes.
 
@@ -586,8 +586,7 @@ with WEBS() as webs:
 <p>
 
 ```python
-from webscout import WEBS
-import datetime
+from webscout.search import DuckDuckGoSearch
 
 def fetch_formatted_news(keywords, timelimit='d', max_results=20):
     """Fetch and format news articles"""