webscout 2025.10.11__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/search/engines/duckduckgo/text.py

@@ -0,0 +1,211 @@
+"""DuckDuckGo text search."""
+
+from __future__ import annotations
+
+import warnings
+from random import shuffle
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTextSearch(DuckDuckGoBase):
+    """DuckDuckGo text/web search."""
+    
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Perform text search on DuckDuckGo.
+        
+        Args:
+            keywords: Search query.
+            region: Region code (e.g., wt-wt, us-en).
+            safesearch: on, moderate, or off.
+            timelimit: d, w, m, or y.
+            backend: html, lite, or auto.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of search result dictionaries.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        backend = args[4] if len(args) > 4 else kwargs.get("backend", "auto")
+        max_results = args[5] if len(args) > 5 else kwargs.get("max_results")
+
+        if backend in ("api", "ecosia"):
+            warnings.warn(f"{backend=} is deprecated, using backend='auto'", stacklevel=2)
+            backend = "auto"
+        backends = ["html", "lite"] if backend == "auto" else [backend]
+        shuffle(backends)
+
+        results, err = [], None
+        for b in backends:
+            try:
+                if b == "html":
+                    results = self._text_html(keywords, region, timelimit, max_results)
+                elif b == "lite":
+                    results = self._text_lite(keywords, region, timelimit, max_results)
+                return results
+            except Exception as ex:
+                err = ex
+
+        raise WebscoutE(err)
+
+    def _text_html(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Text search using HTML backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+        if max_results and max_results > 20:
+            vqd = self._get_vqd(keywords)
+            payload["vqd"] = vqd
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _text_html_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://html.duckduckgo.com/html", data=payload).content
+            if b"No  results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//div[h2]")
+            if not isinstance(elements, list):
+                return []
+            for e in elements:
+                if isinstance(e, self.parser.etree.Element):
+                    hrefxpath = e.xpath("./a/@href")
+                    href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                    if (
+                        href
+                        and href not in cache
+                        and not href.startswith(
+                            ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                        )
+                    ):
+                        cache.add(href)
+                        titlexpath = e.xpath("./h2/a/text()")
+                        title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                        bodyxpath = e.xpath("./a//text()")
+                        body = "".join(str(x) for x in bodyxpath) if bodyxpath and isinstance(bodyxpath, list) else ""
+                        result = {
+                            "title": self._normalize(title),
+                            "href": self._normalize_url(href),
+                            "body": self._normalize(body),
+                        }
+                        page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_html_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+
+    def _text_lite(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Text search using lite backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _text_lite_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://lite.duckduckgo.com/lite/", data=payload).content
+            if b"No more results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//table[last()]//tr")
+            if not isinstance(elements, list):
+                return []
+
+            data = zip(self.cycle(range(1, 5)), elements)
+            for i, e in data:
+                if isinstance(e, self.parser.etree.Element):
+                    if i == 1:
+                        hrefxpath = e.xpath(".//a//@href")
+                        href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                        if (
+                            href is None
+                            or href in cache
+                            or href.startswith(
+                                ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                            )
+                        ):
+                            [next(data, None) for _ in range(3)]  # skip block(i=1,2,3,4)
+                        else:
+                            cache.add(href)
+                            titlexpath = e.xpath(".//a//text()")
+                            title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                    elif i == 2:
+                        bodyxpath = e.xpath(".//td[@class='result-snippet']//text()")
+                        body = (
+                            "".join(str(x) for x in bodyxpath).strip()
+                            if bodyxpath and isinstance(bodyxpath, list)
+                            else ""
+                        )
+                        if href:
+                            result = {
+                                "title": self._normalize(title),
+                                "href": self._normalize_url(href),
+                                "body": self._normalize(body),
+                            }
+                            page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_lite_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))

webscout/search/engines/duckduckgo/translate.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTranslate(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        from_ = args[1] if len(args) > 1 else kwargs.get("from_")
+        to = args[2] if len(args) > 2 else kwargs.get("to", "en")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd("translate")
+
+        payload = {
+            "vqd": vqd,
+            "query": "translate",
+            "to": to,
+        }
+        if from_:
+            payload["from"] = from_
+
+        def _translate_keyword(keyword: str) -> dict[str, str]:
+            resp_content = self._get_url(
+                "POST",
+                "https://duckduckgo.com/translation.js",
+                params=payload,
+                content=keyword.encode(),
+            ).content
+            page_data: dict[str, str] = self.json_loads(resp_content)
+            page_data["original"] = keyword
+            return page_data
+
+        if isinstance(keywords, str):
+            keywords = [keywords]
+
+        results = []
+        try:
+            for r in self._executor.map(_translate_keyword, keywords):
+                results.append(r)
+        except Exception as e:
+            raise e
+
+        return results
+

webscout/search/engines/duckduckgo/videos.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoVideos(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        resolution = args[4] if len(args) > 4 else kwargs.get("resolution")
+        duration = args[5] if len(args) > 5 else kwargs.get("duration")
+        license_videos = args[6] if len(args) > 6 else kwargs.get("license_videos")
+        max_results = args[7] if len(args) > 7 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "-1", "off": "-2"}
+        timelimit = f"publishedAfter:{timelimit}" if timelimit else ""
+        resolution = f"videoDefinition:{resolution}" if resolution else ""
+        duration = f"videoDuration:{duration}" if duration else ""
+        license_videos = f"videoLicense:{license_videos}" if license_videos else ""
+        payload = {
+            "l": region,
+            "o": "json",
+            "q": keywords,
+            "vqd": vqd,
+            "f": f"{timelimit},{resolution},{duration},{license_videos}",
+            "p": safesearch_base[safesearch.lower()],
+        }
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _videos_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/v.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                if row["content"] not in cache:
+                    cache.add(row["content"])
+                    page_results.append(row)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 400)
+            slist.extend(range(60, max_results, 60))
+        try:
+            for r in self._executor.map(_videos_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/weather.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from urllib.parse import quote
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoWeather(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> dict[str, any]:
+        location = args[0] if args else kwargs.get("location")
+        language = args[1] if len(args) > 1 else kwargs.get("language", "en")
+
+        assert location, "location is mandatory"
+        lang = language.split('-')[0]
+        url = f"https://duckduckgo.com/js/spice/forecast/{quote(location)}/{lang}"
+
+        resp = self._get_url("GET", url).content
+        resp_text = resp.decode('utf-8')
+
+        if "ddg_spice_forecast(" not in resp_text:
+            raise WebscoutE(f"No weather data found for {location}")
+
+        json_text = resp_text[resp_text.find('(') + 1:resp_text.rfind(')')]
+        try:
+            result = json.loads(json_text)
+        except Exception as e:
+            raise WebscoutE(f"Error parsing weather JSON: {e}")
+
+        if not result or 'currentWeather' not in result or 'forecastDaily' not in result:
+            raise WebscoutE(f"Invalid weather data format for {location}")
+
+        formatted_data = {
+            "location": result["currentWeather"]["metadata"].get("ddg-location", "Unknown"),
+            "current": {
+                "condition": result["currentWeather"].get("conditionCode"),
+                "temperature_c": result["currentWeather"].get("temperature"),
+                "feels_like_c": result["currentWeather"].get("temperatureApparent"),
+                "humidity": result["currentWeather"].get("humidity"),
+                "wind_speed_ms": result["currentWeather"].get("windSpeed"),
+                "wind_direction": result["currentWeather"].get("windDirection"),
+                "visibility_m": result["currentWeather"].get("visibility"),
+            },
+            "daily_forecast": [],
+            "hourly_forecast": []
+        }
+
+        for day in result["forecastDaily"]["days"]:
+            formatted_data["daily_forecast"].append({
+                "date": datetime.fromisoformat(day["forecastStart"].replace("Z", "+00:00")).strftime("%Y-%m-%d"),
+                "condition": day["daytimeForecast"].get("conditionCode"),
+                "max_temp_c": day["temperatureMax"],
+                "min_temp_c": day["temperatureMin"],
+                "sunrise": datetime.fromisoformat(day["sunrise"].replace("Z", "+00:00")).strftime("%H:%M"),
+                "sunset": datetime.fromisoformat(day["sunset"].replace("Z", "+00:00")).strftime("%H:%M"),
+            })
+
+        if 'forecastHourly' in result and 'hours' in result['forecastHourly']:
+            for hour in result['forecastHourly']['hours']:
+                formatted_data["hourly_forecast"].append({
+                    "time": datetime.fromisoformat(hour["forecastStart"].replace("Z", "+00:00")).strftime("%H:%M"),
+                    "condition": hour.get("conditionCode"),
+                    "temperature_c": hour.get("temperature"),
+                    "feels_like_c": hour.get("temperatureApparent"),
+                    "humidity": hour.get("humidity"),
+                    "wind_speed_ms": hour.get("windSpeed"),
+                    "wind_direction": hour.get("windDirection"),
+                    "visibility_m": hour.get("visibility"),
+                })
+
+        return formatted_data
+

webscout/search/engines/mojeek.py

@@ -0,0 +1,37 @@
+"""Mojeek search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Mojeek(BaseSearchEngine[TextResult]):
+    """Mojeek search engine."""
+
+    name = "mojeek"
+    category = "text"
+    provider = "mojeek"
+
+    search_url = "https://www.mojeek.com/search"
+    search_method = "GET"
+
+    items_xpath = "//ul[contains(@class, 'results')]/li"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//p[@class='s']//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {"q": query, "safe": safesearch_base[safesearch.lower()]}
+        if page > 1:
+            payload["s"] = f"{(page - 1) * 10 + 1}"
+        return payload

webscout/search/engines/wikipedia.py

@@ -0,0 +1,56 @@
+"""Wikipedia text search engine."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+from urllib.parse import quote
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+from ...utils import json_loads
+
+logger = logging.getLogger(__name__)
+
+
+class Wikipedia(BaseSearchEngine[TextResult]):
+    """Wikipedia text search engine."""
+
+    name = "wikipedia"
+    category = "text"
+    provider = "wikipedia"
+    priority = 2
+
+    search_url = "https://{lang}.wikipedia.org/w/api.php?action=opensearch&search={query}"
+    search_method = "GET"
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        _country, lang = region.lower().split("-")
+        encoded_query = quote(query)
+        self.search_url = (
+            f"https://{lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search={encoded_query}"
+        )
+        payload: dict[str, Any] = {}
+        self.lang = lang  # used in extract_results
+        return payload
+
+    def extract_results(self, html_text: str) -> list[TextResult]:
+        """Extract search results from html text."""
+        json_data = json_loads(html_text)
+        if not json_data or len(json_data) < 4:
+            return []
+        
+        results = []
+        titles, descriptions, urls = json_data[1], json_data[2], json_data[3]
+        
+        for title, description, url in zip(titles, descriptions, urls):
+            result = TextResult()
+            result.title = title
+            result.body = description
+            result.href = url
+            results.append(result)
+        
+        return results

webscout/search/engines/yahoo.py

@@ -0,0 +1,65 @@
+"""Yahoo search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+from urllib.parse import unquote_plus
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+def extract_url(u: str) -> str:
+    """Sanitize url."""
+    if "/RU=" in u:
+        start = u.find("/RU=") + 4
+        end = u.find("/RK=", start)
+        if end == -1:
+            end = len(u)
+        return unquote_plus(u[start:end])
+    return u
+
+
+class Yahoo(BaseSearchEngine[TextResult]):
+    """Yahoo search engine."""
+
+    name = "yahoo"
+    category = "text"
+    provider = "bing"
+
+    search_url = "https://search.yahoo.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'relsrch')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//div[contains(@class, 'Title')]//h3//text()",
+        "href": ".//div[contains(@class, 'Title')]//a/@href",
+        "body": ".//div[contains(@class, 'Text')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        self.search_url = (
+            f"https://search.yahoo.com/search;_ylt={token_urlsafe(24 * 3 // 4)};_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        payload = {"p": query}
+        if page > 1:
+            payload["b"] = f"{(page - 1) * 7 + 1}"
+        if timelimit:
+            payload["btf"] = timelimit
+        return payload
+
+    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
+        """Post-process search results."""
+        post_results = []
+        for result in results:
+            if result.href.startswith("https://www.bing.com/aclick?"):
+                continue
+            if "/RU=" in result.href:
+                result.href = extract_url(result.href)
+            post_results.append(result)
+        return post_results

webscout/search/engines/yahoo_news.py

@@ -0,0 +1,64 @@
+"""Yahoo news search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import NewsResult
+
+
+def extract_image(u: str) -> str:
+    """Sanitize image url."""
+    if u and u.startswith("data:image"):
+        return ""
+    return u
+
+
+def extract_source(s: str) -> str:
+    """Remove ' via Yahoo' from string."""
+    return s.replace(" via Yahoo", "") if s else s
+
+
+class YahooNews(BaseSearchEngine[NewsResult]):
+    """Yahoo news search engine."""
+
+    name = "yahoo"
+    category = "news"
+    provider = "bing"
+
+    search_url = "https://news.search.yahoo.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'NewsArticle')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[contains(@class, 'fc-2nd')]//text()",
+        "title": ".//h4//a//text()",
+        "url": ".//h4//a/@href",
+        "body": ".//p//text()",
+        "image": ".//img/@src",
+        "source": ".//span[contains(@class, 's-source')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        self.search_url = (
+            f"https://news.search.yahoo.com/search;_ylt={token_urlsafe(24 * 3 // 4)};_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        payload = {"p": query}
+        if page > 1:
+            payload["b"] = f"{(page - 1) * 10 + 1}"
+        if timelimit:
+            payload["btf"] = timelimit
+        return payload
+
+    def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
+        """Post-process search results."""
+        for result in results:
+            result.image = extract_image(result.image)
+            result.source = extract_source(result.source)
+        return results

webscout/search/engines/yandex.py

@@ -0,0 +1,43 @@
+"""Yandex search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from random import SystemRandom
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+random = SystemRandom()
+
+
+class Yandex(BaseSearchEngine[TextResult]):
+    """Yandex search engine."""
+
+    name = "yandex"
+    category = "text"
+    provider = "yandex"
+
+    search_url = "https://yandex.com/search/"
+    search_method = "GET"
+
+    items_xpath = "//li[contains(@class, 'serp-item')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//div[contains(@class, 'text-container')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {
+            "text": query,
+            "family": safesearch_base[safesearch.lower()],
+        }
+        if page > 1:
+            payload["p"] = str(page - 1)
+        return payload

webscout/search/engines/yep/__init__.py

@@ -0,0 +1,13 @@
+"""Yep search engines package."""
+
+from .base import YepBase
+from .images import YepImages
+from .suggestions import YepSuggestions
+from .text import YepSearch as YepTextSearch
+
+__all__ = [
+    "YepBase",
+    "YepTextSearch",
+    "YepImages",
+    "YepSuggestions",
+]