webscout 8.2.2__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/duckduckgo/text.py

@@ -0,0 +1,221 @@
+"""DuckDuckGo text search."""
+
+from __future__ import annotations
+
+import warnings
+from random import shuffle
+from typing import Optional
+
+from ....exceptions import WebscoutE
+from ....search.results import TextResult
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTextSearch(DuckDuckGoBase):
+    """DuckDuckGo text/web search."""
+
+    name = "duckduckgo"
+    category = "text"
+
+    def run(self, *args, **kwargs) -> list[TextResult]:
+        """Perform text search on DuckDuckGo.
+
+        Args:
+            keywords: Search query.
+            region: Region code (e.g., wt-wt, us-en).
+            safesearch: on, moderate, or off.
+            timelimit: d, w, m, or y.
+            backend: html, lite, or auto.
+            max_results: Maximum number of results.
+
+        Returns:
+            List of TextResult objects.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        backend = args[4] if len(args) > 4 else kwargs.get("backend", "auto")
+        max_results = args[5] if len(args) > 5 else kwargs.get("max_results")
+
+        if keywords is None:
+            raise ValueError("keywords cannot be None")
+
+        if backend in ("api", "ecosia"):
+            warnings.warn(f"{backend=} is deprecated, using backend='auto'", stacklevel=2)
+            backend = "auto"
+        backends = ["html", "lite"] if backend == "auto" else [backend]
+        shuffle(backends)
+
+        results, err = [], None
+        for b in backends:
+            try:
+                if b == "html":
+                    results = self._text_html(keywords, region, timelimit, max_results)
+                elif b == "lite":
+                    results = self._text_lite(keywords, region, timelimit, max_results)
+                return results
+            except Exception as ex:
+                err = ex
+
+        raise WebscoutE(err)
+
+    def _text_html(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[TextResult]:
+        """Text search using HTML backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+        if max_results and max_results > 20:
+            vqd = self._get_vqd(keywords)
+            payload["vqd"] = vqd
+
+        cache = set()
+        results: list[TextResult] = []
+
+        def _text_html_page(s: int) -> list[TextResult]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://html.duckduckgo.com/html", data=payload).content
+            if b"No  results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//div[h2]")
+            if not isinstance(elements, list):
+                return []
+            for e in elements:
+                if isinstance(e, self.parser.etree.Element):
+                    hrefxpath = e.xpath("./a/@href")
+                    href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                    if (
+                        href
+                        and href not in cache
+                        and not href.startswith(
+                            ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                        )
+                    ):
+                        cache.add(href)
+                        titlexpath = e.xpath("./h2/a/text()")
+                        title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                        bodyxpath = e.xpath("./a//text()")
+                        body = "".join(str(x) for x in bodyxpath) if bodyxpath and isinstance(bodyxpath, list) else ""
+                        result = TextResult(
+                            title=self._normalize(title),
+                            href=self._normalize_url(href),
+                            body=self._normalize(body),
+                        )
+                        page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_html_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+
+    def _text_lite(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[TextResult]:
+        """Text search using lite backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+
+        cache = set()
+        results: list[TextResult] = []
+
+        def _text_lite_page(s: int) -> list[TextResult]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://lite.duckduckgo.com/lite/", data=payload).content
+            if b"No more results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//table[last()]//tr")
+            if not isinstance(elements, list):
+                return []
+
+            href: Optional[str] = None
+            title: str = ""
+            data = zip(self.cycle(range(1, 5)), elements)
+            for i, e in data:
+                if isinstance(e, self.parser.etree.Element):
+                    if i == 1:
+                        hrefxpath = e.xpath(".//a//@href")
+                        href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                        if (
+                            href is None
+                            or href in cache
+                            or href.startswith(
+                                ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                            )
+                        ):
+                            [next(data, None) for _ in range(3)]  # skip block(i=1,2,3,4)
+                        else:
+                            cache.add(href)
+                            titlexpath = e.xpath(".//a//text()")
+                            title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                    elif i == 2:
+                        bodyxpath = e.xpath(".//td[@class='result-snippet']//text()")
+                        body = (
+                            "".join(str(x) for x in bodyxpath).strip()
+                            if bodyxpath and isinstance(bodyxpath, list)
+                            else ""
+                        )
+                        if href:
+                            result = TextResult(
+                                title=self._normalize(title),
+                                href=self._normalize_url(href),
+                                body=self._normalize(body),
+                            )
+                            page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_lite_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))

webscout/search/engines/duckduckgo/translate.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTranslate(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "translate"
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        from_ = args[1] if len(args) > 1 else kwargs.get("from_")
+        to = args[2] if len(args) > 2 else kwargs.get("to", "en")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd("translate")
+
+        payload = {
+            "vqd": vqd,
+            "query": "translate",
+            "to": to,
+        }
+        if from_:
+            payload["from"] = from_
+
+        def _translate_keyword(keyword: str) -> dict[str, str]:
+            resp_content = self._get_url(
+                "POST",
+                "https://duckduckgo.com/translation.js",
+                params=payload,
+                content=keyword.encode(),
+            ).content
+            page_data: dict[str, str] = self.json_loads(resp_content)
+            page_data["original"] = keyword
+            return page_data
+
+        if isinstance(keywords, str):
+            keywords = [keywords]
+
+        results = []
+        try:
+            for r in self._executor.map(_translate_keyword, keywords):
+                results.append(r)
+        except Exception as e:
+            raise e
+
+        return results
+

webscout/search/engines/duckduckgo/videos.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from ....search.results import VideosResult
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoVideos(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "videos"
+    def run(self, *args, **kwargs) -> list[VideosResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        resolution = args[4] if len(args) > 4 else kwargs.get("resolution")
+        duration = args[5] if len(args) > 5 else kwargs.get("duration")
+        license_videos = args[6] if len(args) > 6 else kwargs.get("license_videos")
+        max_results = args[7] if len(args) > 7 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "-1", "off": "-2"}
+        timelimit = f"publishedAfter:{timelimit}" if timelimit else ""
+        resolution = f"videoDefinition:{resolution}" if resolution else ""
+        duration = f"videoDuration:{duration}" if duration else ""
+        license_videos = f"videoLicense:{license_videos}" if license_videos else ""
+        payload = {
+            "l": region,
+            "o": "json",
+            "q": keywords,
+            "vqd": vqd,
+            "f": f"{timelimit},{resolution},{duration},{license_videos}",
+            "p": safesearch_base[safesearch.lower()],
+        }
+
+        cache = set()
+        results: list[VideosResult] = []
+
+        def _videos_page(s: int) -> list[VideosResult]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/v.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                if row["content"] not in cache:
+                    cache.add(row["content"])
+                    result = VideosResult(
+                        content=row.get("content", ""),
+                        description=row.get("description", ""),
+                        duration=row.get("duration", ""),
+                        embed_html=row.get("embed_html", ""),
+                        embed_url=row.get("embed_url", ""),
+                        image_token=row.get("image_token", ""),
+                        images=row.get("images", {}),
+                        provider=row.get("provider", ""),
+                        published=row.get("published", ""),
+                        publisher=row.get("publisher", ""),
+                        statistics=row.get("statistics", {}),
+                        title=row.get("title", ""),
+                        uploader=row.get("uploader", ""),
+                    )
+                    page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 400)
+            slist.extend(range(60, max_results, 60))
+        try:
+            for r in self._executor.map(_videos_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/weather.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Any, Dict, List, Optional, TypedDict
+from urllib.parse import quote
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class WeatherData(TypedDict):
+    location: str
+    current: Dict[str, Any]
+    daily_forecast: List[Dict[str, Any]]
+    hourly_forecast: List[Dict[str, Any]]
+
+
+class DuckDuckGoWeather(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "weather"
+    def run(self, *args, **kwargs) -> WeatherData:
+        location = args[0] if args else kwargs.get("location")
+        language = args[1] if len(args) > 1 else kwargs.get("language", "en")
+
+        assert location, "location is mandatory"
+        lang = language.split('-')[0]
+        url = f"https://duckduckgo.com/js/spice/forecast/{quote(location)}/{lang}"
+
+        resp = self._get_url("GET", url).content
+        resp_text = resp.decode('utf-8')
+
+        if "ddg_spice_forecast(" not in resp_text:
+            raise WebscoutE(f"No weather data found for {location}")
+
+        json_text = resp_text[resp_text.find('(') + 1:resp_text.rfind(')')]
+        try:
+            result = json.loads(json_text)
+        except Exception as e:
+            raise WebscoutE(f"Error parsing weather JSON: {e}")
+
+        if not result or 'currentWeather' not in result or 'forecastDaily' not in result:
+            raise WebscoutE(f"Invalid weather data format for {location}")
+
+        formatted_data: WeatherData = {
+            "location": result["currentWeather"]["metadata"].get("ddg-location", "Unknown"),
+            "current": {
+                "condition": result["currentWeather"].get("conditionCode"),
+                "temperature_c": result["currentWeather"].get("temperature"),
+                "feels_like_c": result["currentWeather"].get("temperatureApparent"),
+                "humidity": result["currentWeather"].get("humidity"),
+                "wind_speed_ms": result["currentWeather"].get("windSpeed"),
+                "wind_direction": result["currentWeather"].get("windDirection"),
+                "visibility_m": result["currentWeather"].get("visibility"),
+            },
+            "daily_forecast": [],
+            "hourly_forecast": []
+        }
+
+        for day in result["forecastDaily"]["days"]:
+            formatted_data["daily_forecast"].append({
+                "date": datetime.fromisoformat(day["forecastStart"].replace("Z", "+00:00")).strftime("%Y-%m-%d"),
+                "condition": day["daytimeForecast"].get("conditionCode"),
+                "max_temp_c": day["temperatureMax"],
+                "min_temp_c": day["temperatureMin"],
+                "sunrise": datetime.fromisoformat(day["sunrise"].replace("Z", "+00:00")).strftime("%H:%M"),
+                "sunset": datetime.fromisoformat(day["sunset"].replace("Z", "+00:00")).strftime("%H:%M"),
+            })
+
+        if 'forecastHourly' in result and 'hours' in result['forecastHourly']:
+            for hour in result['forecastHourly']['hours']:
+                formatted_data["hourly_forecast"].append({
+                    "time": datetime.fromisoformat(hour["forecastStart"].replace("Z", "+00:00")).strftime("%H:%M"),
+                    "condition": hour.get("conditionCode"),
+                    "temperature_c": hour.get("temperature"),
+                    "feels_like_c": hour.get("temperatureApparent"),
+                    "humidity": hour.get("humidity"),
+                    "wind_speed_ms": hour.get("windSpeed"),
+                    "wind_direction": hour.get("windDirection"),
+                    "visibility_m": hour.get("visibility"),
+                })
+
+        return formatted_data
+

webscout/search/engines/mojeek.py

@@ -0,0 +1,61 @@
+"""Mojeek search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, Optional
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Mojeek(BaseSearchEngine[TextResult]):
+    """Mojeek search engine."""
+
+    name = "mojeek"
+    category = "text"
+    provider = "mojeek"
+
+    search_url = "https://www.mojeek.com/search"
+    search_method = "GET"
+
+    items_xpath = "//ul[contains(@class, 'results')]/li"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//p[@class='s']//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {"q": query, "safe": safesearch_base[safesearch.lower()]}
+        if page > 1:
+            payload["s"] = f"{(page - 1) * 10 + 1}"
+        return payload
+
+    def run(self, *args, **kwargs) -> list[TextResult]:
+        """Run text search on Mojeek.
+
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            max_results: Maximum number of results (ignored for now).
+
+        Returns:
+            List of TextResult objects.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        if keywords is None:
+            keywords = ""
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        results = self.search(query=keywords, region=region, safesearch=safesearch)
+        if results and max_results:
+            results = results[:max_results]
+        return results or []

webscout/search/engines/wikipedia.py

@@ -0,0 +1,77 @@
+"""Wikipedia text search engine."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+from urllib.parse import quote
+
+from ...utils import json_loads
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Wikipedia(BaseSearchEngine[TextResult]):
+    """Wikipedia text search engine."""
+
+    name = "wikipedia"
+    category = "text"
+    provider = "wikipedia"
+    priority = 2
+
+    search_url = "https://{lang}.wikipedia.org/w/api.php?action=opensearch&search={query}"
+    search_method = "GET"
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        _country, lang = region.lower().split("-")
+        encoded_query = quote(query)
+        self.search_url = (
+            f"https://{lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search={encoded_query}"
+        )
+        payload: dict[str, Any] = {}
+        self.lang = lang  # used in extract_results
+        return payload
+
+    def extract_results(self, html_text: str) -> list[TextResult]:
+        """Extract search results from html text."""
+        json_data = json_loads(html_text)
+        if not json_data or len(json_data) < 4:
+            return []
+
+        results = []
+        titles, descriptions, urls = json_data[1], json_data[2], json_data[3]
+
+        for title, description, url in zip(titles, descriptions, urls):
+            result = TextResult()
+            result.title = title
+            result.body = description
+            result.href = url
+            results.append(result)
+
+        return results
+
+    def run(self, *args, **kwargs) -> list[TextResult]:
+        """Run text search on Wikipedia.
+
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level (ignored).
+            max_results: Maximum number of results.
+
+        Returns:
+            List of TextResult objects.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        if keywords is None:
+            keywords = ""
+        region = args[1] if len(args) > 1 else kwargs.get("region", "en-us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        results = self.search(query=keywords, region=region, safesearch=safesearch)
+        if results and max_results:
+            results = results[:max_results]
+        return results or []

webscout/search/engines/yahoo/__init__.py

@@ -0,0 +1,41 @@
+"""Yahoo search engines package.
+
+This package provides comprehensive Yahoo search functionality including:
+- Text search with multi-page pagination
+- Image search with advanced filters
+- Video search with quality and length filters
+- News search with time filtering
+- Search suggestions/autocomplete
+
+All engines support:
+- Human-like browsing through multiple pages
+- Rich metadata extraction
+- Filter support
+- Clean result formatting
+
+Example:
+    >>> from webscout.search.engines.yahoo import YahooText
+    >>>
+    >>> # Search with automatic pagination
+    >>> searcher = YahooText()
+    >>> results = searcher.search("python programming", max_results=50)
+    >>>
+    >>> for result in results:
+    ...     print(f"{result.title}: {result.url}")
+"""
+
+from .base import YahooSearchEngine
+from .images import YahooImages
+from .news import YahooNews
+from .suggestions import YahooSuggestions
+from .text import YahooText
+from .videos import YahooVideos
+
+__all__ = [
+    "YahooSearchEngine",
+    "YahooText",
+    "YahooImages",
+    "YahooVideos",
+    "YahooNews",
+    "YahooSuggestions",
+]

webscout/search/engines/yahoo/answers.py

@@ -0,0 +1,19 @@
+"""Yahoo answers search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooAnswers(YahooSearchEngine):
+    """Yahoo instant answers."""
+
+    def build_payload(self, *args, **kwargs) -> dict:
+        return {}
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get instant answers from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support instant answers")

webscout/search/engines/yahoo/base.py

@@ -0,0 +1,34 @@
+"""Base class for Yahoo search engines."""
+
+from __future__ import annotations
+
+from secrets import token_urlsafe
+from typing import Generic, Optional, TypeVar
+
+from ...base import BaseSearchEngine
+
+T = TypeVar("T")
+
+class YahooSearchEngine(BaseSearchEngine[T], Generic[T]):
+    """Base class for Yahoo search engines.
+
+    Yahoo search is powered by Bing but has its own interface.
+    All Yahoo searches use dynamic URLs with tokens for tracking.
+    """
+
+    provider = "yahoo"
+    _base_url = "https://search.yahoo.com"
+
+    def generate_ylt_token(self) -> str:
+        """Generate Yahoo _ylt tracking token."""
+        return token_urlsafe(24 * 3 // 4)
+
+    def generate_ylu_token(self) -> str:
+        """Generate Yahoo _ylu tracking token."""
+        return token_urlsafe(47 * 3 // 4)
+
+    def build_search_url(self, base_path: str) -> str:
+        """Build search URL with tracking tokens."""
+        ylt = self.generate_ylt_token()
+        ylu = self.generate_ylu_token()
+        return f"{self._base_url}/{base_path};_ylt={ylt};_ylu={ylu}"