webscout 8.2.2__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/base.py

@@ -0,0 +1,201 @@
+"""Base class for search engines."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from functools import cached_property
+from typing import Any, Generic, Literal, Optional, TypeVar
+
+from litprinter import ic
+
+try:
+    from lxml import html
+    from lxml.etree import HTMLParser as LHTMLParser  # type: ignore
+    LXML_AVAILABLE = True
+except ImportError:
+    LXML_AVAILABLE = False
+    from typing import Any
+    html: Any = None
+    LHTMLParser: Any = None
+
+from .http_client import HttpClient
+from .results import BooksResult, ImagesResult, NewsResult, TextResult, VideosResult
+
+T = TypeVar("T")
+
+
+class BaseSearchEngine(ABC, Generic[T]):
+    """Abstract base class for all search engine backends."""
+
+    name: str  # unique key, e.g. "google"
+    category: Literal["text", "images", "videos", "news", "books", "suggestions", "weather", "maps", "translate"]
+    provider: str  # source of the search results (e.g. "google", "bing", etc.)
+    disabled: bool = False  # if True, the engine is disabled
+    priority: float = 1
+
+    search_url: str
+    search_method: str  # GET or POST
+    search_headers: Mapping[str, str] = {}
+    items_xpath: str = ""
+    elements_xpath: Mapping[str, str] = {}
+    elements_replace: Mapping[str, str] = {}
+
+    def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
+        """Initialize search engine.
+
+        Args:
+            proxy: Proxy URL (supports http/https/socks5).
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+        """
+        self.http_client = HttpClient(proxy=proxy, timeout=timeout, verify=verify)
+        self.results: list[T] = []
+
+    @property
+    def result_type(self) -> type[T]:
+        """Get result type based on category."""
+        categories = {
+            "text": TextResult,
+            "images": ImagesResult,
+            "videos": VideosResult,
+            "news": NewsResult,
+            "books": BooksResult,
+        }
+        return categories[self.category]  # type: ignore
+
+    @abstractmethod
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        raise NotImplementedError
+
+    def request(self, method: str, url: str, **kwargs: Any) -> str | None:
+        """Make a request to the search engine."""
+        try:
+            response = self.http_client.request(method, url, **kwargs)  # type: ignore
+            return response.text
+        except Exception as ex:
+            ic.configureOutput(prefix='ERROR| ')
+            ic(f"Error in {self.name} request: {ex}")
+            return None
+
+    @cached_property
+    def parser(self) -> Any:
+        """Get HTML parser."""
+        if not LXML_AVAILABLE:
+            ic.configureOutput(prefix='WARNING| ')
+            ic("lxml not available, HTML parsing disabled")
+            return None
+        return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False) if LHTMLParser else None
+
+    def extract_tree(self, html_text: str) -> Any:
+        """Extract html tree from html text."""
+        if not LXML_AVAILABLE or not self.parser:
+            raise ImportError("lxml is required for HTML parsing")
+        assert self.parser is not None
+        assert html is not None
+        return html.fromstring(html_text, parser=self.parser)
+
+    def pre_process_html(self, html_text: str) -> str:
+        """Pre-process html_text before extracting results."""
+        return html_text
+
+    def extract_results(self, html_text: str) -> list[T]:
+        """Extract search results from html text."""
+        if not LXML_AVAILABLE:
+            raise ImportError("lxml is required for result extraction")
+
+        html_text = self.pre_process_html(html_text)
+        tree = self.extract_tree(html_text)
+
+        results = []
+        items = tree.xpath(self.items_xpath) if self.items_xpath else []
+
+        for item in items:
+            result = self.result_type()
+            for key, xpath in self.elements_xpath.items():
+                try:
+                    data = item.xpath(xpath)
+                    if data:
+                        # Join text nodes or get first attribute
+                        value = "".join(data) if isinstance(data, list) else data
+                        setattr(result, key, value.strip() if isinstance(value, str) else value)
+                except Exception as ex:
+                    ic.configureOutput(prefix='DEBUG| ')
+                    ic(f"Error extracting {key}: {ex}")
+            results.append(result)
+
+        return results
+
+    def post_extract_results(self, results: list[T]) -> list[T]:
+        """Post-process search results."""
+        return results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> list[T] | None:
+        """Search the engine."""
+        payload = self.build_payload(
+            query=query, region=region, safesearch=safesearch, timelimit=timelimit, page=page, **kwargs
+        )
+        if self.search_method == "GET":
+            html_text = self.request(self.search_method, self.search_url, params=payload, headers=self.search_headers)
+        else:
+            html_text = self.request(self.search_method, self.search_url, data=payload, headers=self.search_headers)
+        if not html_text:
+            return None
+        results = self.extract_results(html_text)
+        return self.post_extract_results(results)
+
+
+# Legacy base class for backwards compatibility
+class BaseSearch(ABC):
+    """Base class for synchronous search engines (legacy)."""
+
+    @abstractmethod
+    def text(self, *args, **kwargs) -> list[Any]:
+        """Text search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def images(self, *args, **kwargs) -> list[Any]:
+        """Images search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def videos(self, *args, **kwargs) -> list[Any]:
+        """Videos search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def news(self, *args, **kwargs) -> list[Any]:
+        """News search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def answers(self, *args, **kwargs) -> list[Any]:
+        """Instant answers."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def suggestions(self, *args, **kwargs) -> list[Any]:
+        """Suggestions."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def maps(self, *args, **kwargs) -> list[Any]:
+        """Maps search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def translate(self, *args, **kwargs) -> list[Any]:
+        """Translate."""
+        raise NotImplementedError

webscout/search/bing_main.py

@@ -0,0 +1,45 @@
+"""Bing unified search interface."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from .base import BaseSearch
+from .engines.bing.images import BingImagesSearch
+from .engines.bing.news import BingNewsSearch
+from .engines.bing.suggestions import BingSuggestionsSearch
+from .engines.bing.text import BingTextSearch
+from .results import ImagesResult, NewsResult, TextResult
+
+
+class BingSearch(BaseSearch):
+    """Unified Bing search interface."""
+
+    def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[TextResult]:
+        search = BingTextSearch()
+        return search.run(keywords, region, safesearch, max_results, unique=unique)
+
+    def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[ImagesResult]:
+        search = BingImagesSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[NewsResult]:
+        search = BingNewsSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def suggestions(self, query: str, region: str = "en-US") -> List[Dict[str, str]]:
+        search = BingSuggestionsSearch()
+        result = search.run(query, region)
+        return [{'suggestion': s} for s in result]
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        raise NotImplementedError("Answers not implemented for Bing")
+
+    def maps(self, *args, **kwargs) -> List[Dict[str, str]]:
+        raise NotImplementedError("Maps not implemented for Bing")
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        raise NotImplementedError("Translate not implemented for Bing")
+
+    def videos(self, *args, **kwargs) -> List[Dict[str, str]]:
+        raise NotImplementedError("Videos not implemented for Bing")

webscout/search/brave_main.py

@@ -0,0 +1,92 @@
+"""Brave unified search interface."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from .base import BaseSearch
+from .engines.brave.images import BraveImages
+from .engines.brave.news import BraveNews
+from .engines.brave.suggestions import BraveSuggestions
+from .engines.brave.text import BraveTextSearch
+from .engines.brave.videos import BraveVideos
+from .results import ImagesResult, NewsResult, TextResult, VideosResult
+
+
+class BraveSearch(BaseSearch):
+    """Unified Brave search interface."""
+
+    def text(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        max_results: Optional[int] = None,
+    ) -> List[TextResult]:
+        search = BraveTextSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def images(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        max_results: Optional[int] = None,
+    ) -> List[ImagesResult]:
+        """Search Brave Images."""
+        search = BraveImages()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def news(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        max_results: Optional[int] = None,
+    ) -> List[NewsResult]:
+        """Search Brave News."""
+        search = BraveNews()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def suggestions(
+        self,
+        query: str,
+        rich: bool = True,
+        country: Optional[str] = None,
+        max_results: Optional[int] = None,
+    ) -> List[Dict[str, str]]:
+        """Search Brave suggestions/autocomplete."""
+        search = BraveSuggestions()
+        results = search.run(query, rich=rich, country=country, max_results=max_results)
+        return [
+            {
+                "query": s.query,
+                "is_entity": str(s.is_entity),
+                "name": s.name,
+                "desc": s.desc,
+            }
+            for s in results
+        ]
+
+    def answers(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Instant answers not yet implemented for Brave."""
+        raise NotImplementedError("Brave instant answers not yet implemented")
+
+    def maps(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Maps search not yet implemented for Brave."""
+        raise NotImplementedError("Brave maps search not yet implemented")
+
+    def translate(self, *args, **kwargs) -> List[Dict[str, str]]:
+        """Translation not yet implemented for Brave."""
+        raise NotImplementedError("Brave translation not yet implemented")
+
+    def videos(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        max_results: Optional[int] = None,
+    ) -> List[VideosResult]:
+        """Search Brave Videos."""
+        search = BraveVideos()
+        return search.run(keywords, region, safesearch, max_results)

webscout/search/duckduckgo_main.py

@@ -0,0 +1,57 @@
+"""DuckDuckGo unified search interface."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Union
+
+from .base import BaseSearch
+from .engines.duckduckgo.answers import DuckDuckGoAnswers
+from .engines.duckduckgo.images import DuckDuckGoImages
+from .engines.duckduckgo.maps import DuckDuckGoMaps
+from .engines.duckduckgo.news import DuckDuckGoNews
+from .engines.duckduckgo.suggestions import DuckDuckGoSuggestions
+from .engines.duckduckgo.text import DuckDuckGoTextSearch
+from .engines.duckduckgo.translate import DuckDuckGoTranslate
+from .engines.duckduckgo.videos import DuckDuckGoVideos
+from .engines.duckduckgo.weather import DuckDuckGoWeather, WeatherData
+from .results import ImagesResult, NewsResult, TextResult, VideosResult
+
+
+class DuckDuckGoSearch(BaseSearch):
+    """Unified DuckDuckGo search interface."""
+
+    def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[TextResult]:
+        search = DuckDuckGoTextSearch()
+        return search.run(keywords, region, safesearch, timelimit, backend, max_results)
+
+    def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[ImagesResult]:
+        search = DuckDuckGoImages()
+        return search.run(keywords, region, safesearch, timelimit, size, color, type_image, layout, license_image, max_results)
+
+    def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[VideosResult]:
+        search = DuckDuckGoVideos()
+        return search.run(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results)
+
+    def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[NewsResult]:
+        search = DuckDuckGoNews()
+        return search.run(keywords, region, safesearch, timelimit, max_results)
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        search = DuckDuckGoAnswers()
+        return search.run(keywords)
+
+    def suggestions(self, keywords: str, region: str = "wt-wt") -> List[Dict[str, str]]:
+        search = DuckDuckGoSuggestions()
+        return search.run(keywords, region)
+
+    def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        search = DuckDuckGoMaps()
+        return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        search = DuckDuckGoTranslate()
+        return search.run(keywords, from_lang, to_lang)
+
+    def weather(self, keywords: str) -> WeatherData:
+        search = DuckDuckGoWeather()
+        return search.run(keywords)

webscout/search/engines/__init__.py

@@ -0,0 +1,127 @@
+"""Static imports for all search engine modules."""
+
+from __future__ import annotations
+
+from ..base import BaseSearchEngine
+from .bing import BingBase, BingImagesSearch, BingNewsSearch, BingSuggestionsSearch, BingTextSearch
+from .brave import (
+    BraveBase,
+    BraveImages,
+    BraveNews,
+    BraveSuggestions,
+    BraveTextSearch,
+    BraveVideos,
+)
+from .duckduckgo import (
+    DuckDuckGoAnswers,
+    DuckDuckGoBase,
+    DuckDuckGoImages,
+    DuckDuckGoMaps,
+    DuckDuckGoNews,
+    DuckDuckGoSuggestions,
+    DuckDuckGoTextSearch,
+    DuckDuckGoTranslate,
+    DuckDuckGoVideos,
+    DuckDuckGoWeather,
+)
+from .mojeek import Mojeek
+from .wikipedia import Wikipedia
+from .yahoo import (
+    YahooImages,
+    YahooNews,
+    YahooSearchEngine,
+    YahooSuggestions,
+    YahooText,
+    YahooVideos,
+)
+from .yandex import Yandex
+from .yep import YepBase, YepImages, YepSuggestions, YepTextSearch
+
+# Engine categories mapping
+ENGINES = {
+    "text": {
+        "brave": BraveTextSearch,
+        "mojeek": Mojeek,
+        "yandex": Yandex,
+        "bing": BingTextSearch,
+        "duckduckgo": DuckDuckGoTextSearch,
+        "yep": YepTextSearch,
+        "yahoo": YahooText,
+    },
+    "images": {
+        "bing": BingImagesSearch,
+        "brave": BraveImages,
+        "duckduckgo": DuckDuckGoImages,
+        "yep": YepImages,
+        "yahoo": YahooImages,
+    },
+    "videos": {
+        "brave": BraveVideos,
+        "duckduckgo": DuckDuckGoVideos,
+        "yahoo": YahooVideos,
+    },
+    "news": {
+        "brave": BraveNews,
+        "bing": BingNewsSearch,
+        "duckduckgo": DuckDuckGoNews,
+        "yahoo": YahooNews,
+    },
+    "suggestions": {
+        "brave": BraveSuggestions,
+        "bing": BingSuggestionsSearch,
+        "duckduckgo": DuckDuckGoSuggestions,
+        "yep": YepSuggestions,
+        "yahoo": YahooSuggestions,
+    },
+    "answers": {
+        "duckduckgo": DuckDuckGoAnswers,
+    },
+    "maps": {
+        "duckduckgo": DuckDuckGoMaps,
+    },
+    "translate": {
+        "duckduckgo": DuckDuckGoTranslate,
+    },
+    "weather": {
+        "duckduckgo": DuckDuckGoWeather,
+    },
+}
+
+__all__ = [
+    "BraveBase",
+    "BraveTextSearch",
+    "BraveImages",
+    "BraveVideos",
+    "BraveNews",
+    "BraveSuggestions",
+    "Mojeek",
+    "Wikipedia",
+    "Yandex",
+    "BingBase",
+    "BingTextSearch",
+    "BingImagesSearch",
+    "BingNewsSearch",
+    "BingSuggestionsSearch",
+    "DuckDuckGoBase",
+    "DuckDuckGoTextSearch",
+    "DuckDuckGoImages",
+    "DuckDuckGoVideos",
+    "DuckDuckGoNews",
+    "DuckDuckGoAnswers",
+    "DuckDuckGoSuggestions",
+    "DuckDuckGoMaps",
+    "DuckDuckGoTranslate",
+    "DuckDuckGoWeather",
+    "YepBase",
+    "YepTextSearch",
+    "YepImages",
+    "YepSuggestions",
+    "YahooSearchEngine",
+    "YahooText",
+    "YahooImages",
+    "YahooVideos",
+    "YahooNews",
+    "YahooSuggestions",
+    "BaseSearchEngine",
+    "ENGINES",
+]

webscout/search/engines/bing/__init__.py

@@ -0,0 +1,15 @@
+"""Bing search engines."""
+
+from .base import BingBase
+from .images import BingImagesSearch
+from .news import BingNewsSearch
+from .suggestions import BingSuggestionsSearch
+from .text import BingTextSearch
+
+__all__ = [
+    "BingBase",
+    "BingTextSearch",
+    "BingImagesSearch",
+    "BingNewsSearch",
+    "BingSuggestionsSearch",
+]

webscout/search/engines/bing/base.py

@@ -0,0 +1,35 @@
+"""Base class for Bing search implementations."""
+
+from __future__ import annotations
+
+from curl_cffi.requests import Session
+
+from ....litagent import LitAgent
+
+
+class BingBase:
+    """Base class for Bing search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 10,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        lang: str = "en-US",
+        sleep_interval: float = 0.0,
+        impersonate: str = "chrome110",
+    ):
+        self.timeout = timeout
+        self.proxies = proxies
+        self.verify = verify
+        self.lang = lang
+        self.sleep_interval = sleep_interval
+        self.base_url = "https://www.bing.com"
+        from typing import Any, Optional, cast
+        self.session = Session(
+            proxies=cast(Any, proxies),
+            verify=verify,
+            timeout=timeout,
+            impersonate=cast(Any, impersonate),
+        )
+        self.session.headers.update(LitAgent().generate_fingerprint())

webscout/search/engines/bing/images.py

@@ -0,0 +1,114 @@
+"""Bing images search."""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from webscout.scout import Scout
+from webscout.search.results import ImagesResult
+
+from .base import BingBase
+
+
+class BingImagesSearch(BingBase):
+    name = "bing"
+    category = "images"
+    def run(self, *args, **kwargs) -> List[ImagesResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe_map.get(safesearch.lower(), "Moderate")
+
+        # Bing images URL
+        url = f"{self.base_url}/images/async"
+        params = {
+            'q': keywords,
+            'first': '1',
+            'count': '35',  # Fetch more to get max_results
+            'cw': '1177',
+            'ch': '759',
+            'tsc': 'ImageHoverTitle',
+            'layout': 'RowBased_Landscape',
+            't': '0',
+            'IG': '',
+            'SFX': '0',
+            'iid': 'images.1'
+        }
+
+        results = []
+        first = 1
+        sfx = 0
+
+        while len(results) < max_results:
+            params['first'] = str(first)
+            params['SFX'] = str(sfx)
+            full_url = f"{url}?{urlencode(params)}"
+
+            try:
+                response = self.session.get(full_url, timeout=self.timeout)
+                response.raise_for_status()
+                html = response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch images: {str(e)}")
+
+            soup = Scout(html)
+            img_tags = soup.select('a.iusc img')
+
+            for img in img_tags:
+                if len(results) >= max_results:
+                    break
+
+                title = img.get('alt', '')
+                src = img.get('src', '')
+                m_attr = img.parent.get('m', '') if img.parent else ''
+
+                # Parse m attribute for full image URL
+                image_url = src
+                thumbnail = src
+                if m_attr:
+                    try:
+                        import json
+                        m_data = json.loads(m_attr)
+                        image_url = m_data.get('murl', src)
+                        thumbnail = m_data.get('turl', src)
+                    except Exception:
+                        pass
+
+                source = ''
+                if img.parent and img.parent.parent:
+                    source_tag = img.parent.parent.select_one('.iusc .lnk')
+                    if source_tag:
+                        source = source_tag.get_text(strip=True)
+
+                results.append(ImagesResult(
+                    title=title,
+                    image=image_url,
+                    thumbnail=thumbnail,
+                    url=image_url,
+                    height=0,
+                    width=0,
+                    source=source
+                ))
+
+            first += 35
+            sfx += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return results[:max_results]