webscout 8.2.2__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/yep/images.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from webscout.search.results import ImagesResult
+
+from .base import YepBase
+
+
+class YepImages(YepBase):
+    name = "yep"
+    category = "images"
+    def run(self, *args, **kwargs) -> List[ImagesResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "images"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            if not raw_results or len(raw_results) < 2:
+                return []
+
+            formatted_results = []
+            results = raw_results[1].get('results', [])
+
+            for result in results:
+                if result.get("type") != "Image":
+                    continue
+
+                formatted_result = ImagesResult(
+                    title=self._remove_html_tags(result.get("title", "")),
+                    image=result.get("image_id", ""),
+                    thumbnail=result.get("src", ""),
+                    url=result.get("host_page", ""),
+                    height=result.get("height", 0),
+                    width=result.get("width", 0),
+                    source=result.get("visual_url", "")
+                )
+
+                formatted_results.append(formatted_result)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+
+        except Exception as e:
+            resp = getattr(e, 'response', None)
+            if resp is not None:
+                 raise Exception(f"Yep image search failed with status {resp.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep image search failed: {str(e)}")
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+

webscout/search/engines/yep/suggestions.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepSuggestions(YepBase):
+    name = "yep"
+    category = "suggestions"
+    def run(self, *args, **kwargs) -> List[str]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+
+        params = {
+            "query": keywords,
+            "type": "web",
+            "gl": region
+        }
+
+        url = f"https://api.yep.com/ac/?{urlencode(params)}"
+
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            data = response.json()
+            if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
+                return data[1]
+            return []
+
+        except Exception as e:
+            resp = getattr(e, 'response', None)
+            if resp is not None:
+                 raise Exception(f"Yep suggestions failed with status {resp.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep suggestions failed: {str(e)}")
+

webscout/search/engines/yep/text.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from webscout.search.results import TextResult
+
+from .base import YepBase
+
+
+class YepSearch(YepBase):
+    name = "yep"
+    category = "text"
+    def run(self, *args, **kwargs) -> List[TextResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "web"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            formatted_results = self.format_results(raw_results)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+        except Exception as e:
+            resp = getattr(e, 'response', None)
+            if resp is not None:
+                 raise Exception(f"Yep search failed with status {resp.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep search failed: {str(e)}")
+
+    def format_results(self, raw_results: dict) -> List[TextResult]:
+        formatted_results = []
+
+        if not raw_results or len(raw_results) < 2:
+            return formatted_results
+
+        results = raw_results[1].get('results', [])
+
+        for result in results:
+            formatted_result = TextResult(
+                title=self._remove_html_tags(result.get("title", "")),
+                href=result.get("url", ""),
+                body=self._remove_html_tags(result.get("snippet", ""))
+            )
+
+            formatted_results.append(formatted_result)
+
+        return formatted_results
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+

webscout/search/http_client.py

@@ -0,0 +1,172 @@
+"""HTTP client for search engines."""
+
+from __future__ import annotations
+
+from random import choice
+from typing import Any, Literal, Optional, cast
+
+try:
+    import trio  # type: ignore
+except ImportError:
+    pass
+
+import curl_cffi.requests
+
+from ..exceptions import RatelimitE, TimeoutE, WebscoutE
+
+
+class HttpClient:
+    """HTTP client wrapper for search engines."""
+
+    # curl_cffi supported browser impersonations
+    _impersonates = (
+        "chrome99",
+        "chrome100",
+        "chrome101",
+        "chrome104",
+        "chrome107",
+        "chrome110",
+        "chrome116",
+        "chrome119",
+        "chrome120",
+        "chrome123",
+        "chrome124",
+        "chrome131",
+        "chrome133a",
+        "chrome99_android",
+        "chrome131_android",
+        "safari15_3",
+        "safari15_5",
+        "safari17_0",
+        "safari17_2_ios",
+        "safari18_0",
+        "safari18_0_ios",
+        "edge99",
+        "edge101",
+        "firefox133",
+        "firefox135",
+    )
+
+    def __init__(
+        self,
+        proxy: str | None = None,
+        timeout: int | None = 10,
+        verify: bool = True,
+        headers: dict[str, str] | None = None,
+    ) -> None:
+        """Initialize HTTP client.
+
+        Args:
+            proxy: Proxy URL (supports http/https/socks5).
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+            headers: Default headers for requests.
+        """
+        self.proxy = proxy
+        self.timeout = timeout
+        self.verify = verify
+
+        # Choose random browser to impersonate
+        impersonate_browser = choice(self._impersonates)
+
+        # Initialize curl_cffi session
+        self.client = curl_cffi.requests.Session(
+            headers=headers or {},
+            proxies={"http": self.proxy, "https": self.proxy} if self.proxy else None,
+            timeout=cast(Any, timeout),
+            impersonate=impersonate_browser,
+            verify=verify,
+        )
+
+    def request(
+        self,
+        method: Literal["GET", "POST", "HEAD", "OPTIONS", "DELETE", "PUT", "PATCH"],
+        url: str,
+        params: dict[str, Any] | None = None,
+        data: dict[str, Any] | bytes | None = None,
+        json: Any = None,
+        headers: dict[str, str] | None = None,
+        cookies: dict[str, str] | None = None,
+        timeout: int | None = None,
+        **kwargs: Any,
+    ) -> curl_cffi.requests.Response:
+        """Make HTTP request.
+
+        Args:
+            method: HTTP method.
+            url: Request URL.
+            params: URL parameters.
+            data: Request body data.
+            json: JSON data to send.
+            headers: Request headers.
+            cookies: Request cookies.
+            timeout: Request timeout (overrides default).
+            **kwargs: Additional arguments passed to curl_cffi.
+
+        Returns:
+            Response object.
+
+        Raises:
+            TimeoutE: Request timeout.
+            RatelimitE: Rate limit exceeded.
+            WebscoutE: Other request errors.
+        """
+        try:
+            request_kwargs: dict[str, Any] = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+                **kwargs,
+            }
+
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+
+            if data is not None:
+                request_kwargs["data"] = data
+
+            resp = self.client.request(method, url, **request_kwargs)
+
+            # Check response status
+            if resp.status_code == 200:
+                return resp
+            elif resp.status_code in (202, 301, 403, 400, 429, 418):
+                raise RatelimitE(f"{resp.url} {resp.status_code} Rate limit")
+            else:
+                raise WebscoutE(f"{resp.url} returned {resp.status_code}")
+
+        except Exception as ex:
+            if "time" in str(ex).lower() or "timeout" in str(ex).lower():
+                raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+            raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+
+    def get(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
+        """Make GET request."""
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> curl_cffi.requests.Response:
+        """Make POST request."""
+        return self.request("POST", url, **kwargs)
+
+    def set_cookies(self, url: str, cookies: dict[str, str]) -> None:
+        """Set cookies for a domain.
+
+        Args:
+            url: URL to set cookies for.
+            cookies: Cookie dictionary.
+        """
+        self.client.cookies.update(cookies)
+
+    def close(self) -> None:
+        """Close the HTTP client."""
+        if hasattr(self.client, "close"):
+            self.client.close()
+
+    def __enter__(self) -> HttpClient:
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        """Context manager exit."""
+        self.close()

webscout/search/results.py

@@ -0,0 +1,141 @@
+"""Result models for search engines."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+@dataclass
+class TextResult:
+    """Text search result."""
+
+    title: str = ""
+    href: str = ""
+    body: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "href": self.href,
+            "body": self.body,
+        }
+
+
+@dataclass
+class ImagesResult:
+    """Images search result."""
+
+    title: str = ""
+    image: str = ""
+    thumbnail: str = ""
+    url: str = ""
+    height: int = 0
+    width: int = 0
+    source: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "image": self.image,
+            "thumbnail": self.thumbnail,
+            "url": self.url,
+            "height": self.height,
+            "width": self.width,
+            "source": self.source,
+        }
+
+
+@dataclass
+class VideosResult:
+    """Videos search result."""
+
+    content: str = ""
+    description: str = ""
+    duration: str = ""
+    embed_html: str = ""
+    embed_url: str = ""
+    image_token: str = ""
+    images: dict[str, str] = field(default_factory=dict)
+    provider: str = ""
+    published: str = ""
+    publisher: str = ""
+    statistics: dict[str, int] = field(default_factory=dict)
+    title: str = ""
+    uploader: str = ""
+    url: str = ""
+    thumbnail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "content": self.content,
+            "description": self.description,
+            "duration": self.duration,
+            "embed_html": self.embed_html,
+            "embed_url": self.embed_url,
+            "image_token": self.image_token,
+            "images": self.images,
+            "provider": self.provider,
+            "published": self.published,
+            "publisher": self.publisher,
+            "statistics": self.statistics,
+            "title": self.title,
+            "uploader": self.uploader,
+            "url": self.url,
+            "thumbnail": self.thumbnail,
+        }
+
+
+@dataclass
+class NewsResult:
+    """News search result."""
+
+    date: str = ""
+    title: str = ""
+    body: str = ""
+    url: str = ""
+    image: str = ""
+    source: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "date": self.date,
+            "title": self.title,
+            "body": self.body,
+            "url": self.url,
+            "image": self.image,
+            "source": self.source,
+        }
+
+
+@dataclass
+class BooksResult:
+    """Books search result."""
+
+    title: str = ""
+    author: str = ""
+    href: str = ""
+    thumbnail: str = ""
+    year: str = ""
+    publisher: str = ""
+    language: str = ""
+    filesize: str = ""
+    extension: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "title": self.title,
+            "author": self.author,
+            "href": self.href,
+            "thumbnail": self.thumbnail,
+            "year": self.year,
+            "publisher": self.publisher,
+            "language": self.language,
+            "filesize": self.filesize,
+            "extension": self.extension,
+        }

webscout/search/yahoo_main.py

@@ -0,0 +1,57 @@
+"""Yahoo unified search interface."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from .base import BaseSearch
+from .engines.yahoo.answers import YahooAnswers
+from .engines.yahoo.images import YahooImages
+from .engines.yahoo.maps import YahooMaps
+from .engines.yahoo.news import YahooNews
+from .engines.yahoo.suggestions import YahooSuggestions
+from .engines.yahoo.text import YahooText
+from .engines.yahoo.translate import YahooTranslate
+from .engines.yahoo.videos import YahooVideos
+from .engines.yahoo.weather import YahooWeather
+from .results import ImagesResult, NewsResult, TextResult, VideosResult
+
+
+class YahooSearch(BaseSearch):
+    """Unified Yahoo search interface."""
+
+    def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooText()
+        return search.run(keywords=keywords, region=region, safesearch=safesearch, max_results=max_results)
+
+    def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooImages()
+        return search.run(keywords=keywords, region=region, safesearch=safesearch, max_results=max_results)
+
+    def videos(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooVideos()
+        return search.run(keywords=keywords, region=region, safesearch=safesearch, max_results=max_results)
+
+    def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooNews()
+        return search.run(keywords=keywords, region=region, safesearch=safesearch, max_results=max_results)
+
+    def suggestions(self, keywords: str, region: str = "us") -> List[str]:
+        search = YahooSuggestions()
+        return search.run(keywords, region)
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        search = YahooAnswers()
+        return search.run(keywords)
+
+    def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooMaps()
+        return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        search = YahooTranslate()
+        return search.run(keywords, from_lang, to_lang)
+
+    def weather(self, keywords: str) -> List[Dict[str, str]]:
+        search = YahooWeather()
+        return search.run(keywords)

webscout/search/yep_main.py

@@ -0,0 +1,48 @@
+"""Yep unified search interface."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from .base import BaseSearch
+from .engines.yep.images import YepImages
+from .engines.yep.suggestions import YepSuggestions
+from .engines.yep.text import YepSearch as YepTextSearch
+
+
+class YepSearch(BaseSearch):
+    """Unified Yep search interface."""
+
+    def text(self, keywords: str, region: str = "all", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YepTextSearch()
+        results = search.run(keywords, region, safesearch, max_results)
+        return [r.to_dict() for r in results]
+
+    def images(self, keywords: str, region: str = "all", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YepImages()
+        results = search.run(keywords, region, safesearch, max_results)
+        return [r.to_dict() for r in results]
+
+    def suggestions(self, keywords: str, region: str = "all") -> List[str]:
+        search = YepSuggestions()
+        return search.run(keywords, region)
+
+    def videos(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Videos search not supported by Yep."""
+        raise NotImplementedError("Yep does not support video search")
+
+    def news(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """News search not supported by Yep."""
+        raise NotImplementedError("Yep does not support news search")
+
+    def answers(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Instant answers not supported by Yep."""
+        raise NotImplementedError("Yep does not support instant answers")
+
+    def maps(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Maps search not supported by Yep."""
+        raise NotImplementedError("Yep does not support maps search")
+
+    def translate(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Translation not supported by Yep."""
+        raise NotImplementedError("Yep does not support translation")