webscout 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/search/duckduckgo_main.py

@@ -0,0 +1,54 @@
+"""DuckDuckGo unified search interface."""
+
+from __future__ import annotations
+from typing import Dict, List, Optional
+from .base import BaseSearch
+from .engines.duckduckgo.text import DuckDuckGoTextSearch
+from .engines.duckduckgo.images import DuckDuckGoImages
+from .engines.duckduckgo.videos import DuckDuckGoVideos
+from .engines.duckduckgo.news import DuckDuckGoNews
+from .engines.duckduckgo.answers import DuckDuckGoAnswers
+from .engines.duckduckgo.suggestions import DuckDuckGoSuggestions
+from .engines.duckduckgo.maps import DuckDuckGoMaps
+from .engines.duckduckgo.translate import DuckDuckGoTranslate
+from .engines.duckduckgo.weather import DuckDuckGoWeather
+
+
+class DuckDuckGoSearch(BaseSearch):
+    """Unified DuckDuckGo search interface."""
+
+    def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoTextSearch()
+        return search.run(keywords, region, safesearch, timelimit, backend, max_results)
+
+    def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoImages()
+        return search.run(keywords, region, safesearch, timelimit, size, color, type_image, layout, license_image, max_results)
+
+    def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoVideos()
+        return search.run(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results)
+
+    def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoNews()
+        return search.run(keywords, region, safesearch, timelimit, max_results)
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        search = DuckDuckGoAnswers()
+        return search.run(keywords)
+
+    def suggestions(self, keywords: str, region: str = "wt-wt") -> List[str]:
+        search = DuckDuckGoSuggestions()
+        return search.run(keywords, region)
+
+    def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoMaps()
+        return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        search = DuckDuckGoTranslate()
+        return search.run(keywords, from_lang, to_lang)
+
+    def weather(self, keywords: str) -> List[Dict[str, str]]:
+        search = DuckDuckGoWeather()
+        return search.run(keywords)

webscout/search/engines/__init__.py

@@ -0,0 +1,48 @@
+"""__init__.py for engines package - auto-discovers and registers search engines."""
+
+from __future__ import annotations
+
+import importlib
+import inspect
+import logging
+import pkgutil
+from collections import defaultdict
+from typing import Any
+
+from ..base import BaseSearchEngine
+
+logger = logging.getLogger(__name__)
+
+# ENGINES[category][name] = class
+ENGINES: dict[str, dict[str, type[BaseSearchEngine[Any]]]] = defaultdict(dict)
+
+package_name = __name__
+package = importlib.import_module(package_name)
+
+# Auto-discover all search engine classes
+for finder, modname, _ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
+    try:
+        module = importlib.import_module(modname)
+        for _, cls in inspect.getmembers(module, inspect.isclass):
+            # Must subclass BaseSearchEngine (but not the base itself)
+            if not issubclass(cls, BaseSearchEngine) or cls is BaseSearchEngine:
+                continue
+
+            # Skip any class whose name starts with "Base"
+            if cls.__name__.startswith("Base"):
+                continue
+
+            # Skip disabled engines
+            if getattr(cls, "disabled", False):
+                logger.info("Skipping disabled engine: %s", cls.name)
+                continue
+
+            # Register the engine
+            if hasattr(cls, "name") and hasattr(cls, "category"):
+                ENGINES[cls.category][cls.name] = cls
+                logger.debug("Registered engine: %s (%s)", cls.name, cls.category)
+    except Exception as ex:
+        logger.warning("Failed to import module %s: %r", modname, ex)
+
+
+__all__ = ["ENGINES"]

webscout/search/engines/bing.py

@@ -0,0 +1,84 @@
+"""Bing search engine implementation."""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping
+from time import time
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+def unwrap_bing_url(raw_url: str) -> str | None:
+    """Decode the Bing-wrapped raw_url to extract the original url."""
+    try:
+        parsed = urlparse(raw_url)
+        if parsed.netloc == "www.bing.com" and parsed.path == "/ck/a":
+            query_params = parse_qs(parsed.query)
+            if "u" in query_params:
+                encoded_url = query_params["u"][0]
+                # Decode the base64-like encoding
+                if encoded_url.startswith("a1"):
+                    encoded_url = encoded_url[2:]
+                # Add padding if needed
+                padding = len(encoded_url) % 4
+                if padding:
+                    encoded_url += "=" * (4 - padding)
+                try:
+                    decoded = base64.urlsafe_b64decode(encoded_url).decode()
+                    return decoded
+                except Exception:
+                    pass
+        return raw_url
+    except Exception:
+        return raw_url
+
+
+class Bing(BaseSearchEngine[TextResult]):
+    """Bing search engine."""
+
+    name = "bing"
+    category = "text"
+    provider = "bing"
+
+    search_url = "https://www.bing.com/search"
+    search_method = "GET"
+
+    items_xpath = "//li[contains(@class, 'b_algo')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2/a//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//p//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the Bing search request."""
+        country, lang = region.lower().split("-")
+        payload = {"q": query, "pq": query, "cc": lang}
+        cookies = {
+            "_EDGE_CD": f"m={lang}-{country}&u={lang}-{country}",
+            "_EDGE_S": f"mkt={lang}-{country}&ui={lang}-{country}",
+        }
+        self.http_client.set_cookies("https://www.bing.com", cookies)
+        if timelimit:
+            d = int(time() // 86400)
+            payload["filters"] = {
+                "d": f"ex1:\"ez1_{d - 1}_{d}\"",
+                "w": f"ex1:\"ez1_{d - 7}_{d}\"",
+                "m": f"ex1:\"ez1_{d - 30}_{d}\"",
+                "y": f"ex1:\"ez1_{d - 365}_{d}\"",
+            }[timelimit]
+        if page > 1:
+            payload["first"] = f"{(page - 1) * 10 + 1}"
+        return payload
+
+    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
+        """Post-process search results."""
+        for result in results:
+            result.href = unwrap_bing_url(result.href)
+        return results

webscout/search/engines/bing_news.py

@@ -0,0 +1,52 @@
+"""Bing news search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import NewsResult
+
+
+class BingNews(BaseSearchEngine[NewsResult]):
+    """Bing news engine."""
+
+    name = "bing"
+    category = "news"
+    provider = "bing"
+
+    search_url = "https://www.bing.com/news/infinitescrollajax"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'newsitem')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[@aria-label]//@aria-label",
+        "title": "@data-title",
+        "body": ".//div[@class='snippet']//text()",
+        "url": "@url",
+        "image": ".//a[contains(@class, 'image')]//@src",
+        "source": "@data-author",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the Bing search request."""
+        country, lang = region.lower().split("-")
+        payload = {
+            "q": query,
+            "InfiniteScroll": "1",
+            "first": f"{page * 10 + 1}",
+            "SFX": f"{page}",
+            "cc": country,
+            "setlang": lang,
+        }
+        if timelimit:
+            payload["qft"] = {
+                "d": 'interval="4"',  # doesn't exist so it's the same as one hour
+                "w": 'interval="7"',
+                "m": 'interval="9"',
+                "y": 'interval="9"',  # doesn't exist so it's the same as month
+            }[timelimit]
+        return payload

webscout/search/engines/brave.py

@@ -0,0 +1,43 @@
+"""Brave search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Brave(BaseSearchEngine[TextResult]):
+    """Brave search engine."""
+
+    name = "brave"
+    category = "text"
+    provider = "brave"
+
+    search_url = "https://search.brave.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[@data-type='web']"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//div[(contains(@class,'title') or contains(@class,'sitename-container')) and position()=last()]//text()",
+        "href": "./a/@href",
+        "body": ".//div[contains(@class, 'description')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "strict", "moderate": "moderate", "off": "off"}
+        payload = {
+            "q": query,
+            "source": "web",
+            "safesearch": safesearch_base[safesearch.lower()],
+        }
+        if timelimit:
+            payload["tf"] = timelimit
+        if page > 1:
+            payload["offset"] = f"{(page - 1) * 10}"
+        return payload

webscout/search/engines/duckduckgo/__init__.py

@@ -0,0 +1,25 @@
+"""DuckDuckGo search engines package."""
+
+from .answers import DuckDuckGoAnswers
+from .base import DuckDuckGoBase
+from .images import DuckDuckGoImages
+from .maps import DuckDuckGoMaps
+from .news import DuckDuckGoNews
+from .suggestions import DuckDuckGoSuggestions
+from .text import DuckDuckGoTextSearch
+from .translate import DuckDuckGoTranslate
+from .videos import DuckDuckGoVideos
+from .weather import DuckDuckGoWeather
+
+__all__ = [
+    "DuckDuckGoBase",
+    "DuckDuckGoTextSearch",
+    "DuckDuckGoImages",
+    "DuckDuckGoVideos",
+    "DuckDuckGoNews",
+    "DuckDuckGoAnswers",
+    "DuckDuckGoSuggestions",
+    "DuckDuckGoMaps",
+    "DuckDuckGoTranslate",
+    "DuckDuckGoWeather",
+]

webscout/search/engines/duckduckgo/answers.py

@@ -0,0 +1,78 @@
+"""DuckDuckGo answers search."""
+
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoAnswers(DuckDuckGoBase):
+    """DuckDuckGo instant answers."""
+    
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get instant answers from DuckDuckGo.
+        
+        Args:
+            keywords: Search query.
+            
+        Returns:
+            List of answer dictionaries.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": f"what is {keywords}",
+            "format": "json",
+        }
+        resp_content = self._get_url("GET", "https://api.duckduckgo.com/", params=payload).content
+        page_data = self.json_loads(resp_content)
+
+        results = []
+        answer = page_data.get("AbstractText")
+        url = page_data.get("AbstractURL")
+        if answer:
+            results.append(
+                {
+                    "icon": None,
+                    "text": answer,
+                    "topic": None,
+                    "url": url,
+                }
+            )
+
+        # related
+        payload = {
+            "q": f"{keywords}",
+            "format": "json",
+        }
+        resp_content = self._get_url("GET", "https://api.duckduckgo.com/", params=payload).content
+        resp_json = self.json_loads(resp_content)
+        page_data = resp_json.get("RelatedTopics", [])
+
+        for row in page_data:
+            topic = row.get("Name")
+            if not topic:
+                icon = row["Icon"].get("URL")
+                results.append(
+                    {
+                        "icon": f"https://duckduckgo.com{icon}" if icon else "",
+                        "text": row["Text"],
+                        "topic": None,
+                        "url": row["FirstURL"],
+                    }
+                )
+            else:
+                for subrow in row["Topics"]:
+                    icon = subrow["Icon"].get("URL")
+                    results.append(
+                        {
+                            "icon": f"https://duckduckgo.com{icon}" if icon else "",
+                            "text": subrow["Text"],
+                            "topic": topic,
+                            "url": subrow["FirstURL"],
+                        }
+                    )
+
+        return results

webscout/search/engines/duckduckgo/base.py

@@ -0,0 +1,187 @@
+"""Base class for DuckDuckGo search implementations."""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+from functools import cached_property
+from itertools import cycle, islice
+from random import choice
+from time import sleep, time
+from typing import Any
+
+try:
+    import trio
+except ImportError:
+    pass
+
+import curl_cffi.requests
+
+try:
+    from lxml.html import HTMLParser as LHTMLParser
+    from lxml.html import document_fromstring
+    LXML_AVAILABLE = True
+except ImportError:
+    LXML_AVAILABLE = False
+
+from ....exceptions import RatelimitE, TimeoutE, WebscoutE
+from ....utils import (
+    _extract_vqd,
+    _normalize,
+    _normalize_url,
+    json_loads,
+)
+from ....litagent import LitAgent
+
+
+class DuckDuckGoBase:
+    """Base class for DuckDuckGo search operations."""
+
+    _executor: ThreadPoolExecutor = ThreadPoolExecutor()
+    _impersonates = (
+        "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
+        "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133a",
+        "chrome99_android", "chrome131_android",
+        "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios", "safari18_0", "safari18_0_ios",
+        "edge99", "edge101",
+        "firefox133", "firefox135",
+    )
+
+    def __init__(
+        self,
+        headers: dict[str, str] | None = None,
+        proxy: str | None = None,
+        proxies: dict[str, str] | str | None = None,
+        timeout: int | None = 10,
+        verify: bool = True,
+    ) -> None:
+        """Initialize DuckDuckGo base client.
+
+        Args:
+            headers: Dictionary of headers for the HTTP client.
+            proxy: Proxy for the HTTP client (http/https/socks5).
+            proxies: Deprecated, use proxy instead.
+            timeout: Timeout value for the HTTP client.
+            verify: SSL verification when making requests.
+        """
+        ddgs_proxy: str | None = os.environ.get("DDGS_PROXY")
+        self.proxy: str | None = ddgs_proxy if ddgs_proxy else proxy
+        
+        if not proxy and proxies:
+            self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies
+
+        default_headers = {
+            **LitAgent().generate_fingerprint(),
+            "Origin": "https://duckduckgo.com",
+            "Referer": "https://duckduckgo.com/",
+        }
+
+        self.headers = headers if headers else {}
+        self.headers.update(default_headers)
+
+        impersonate_browser = choice(self._impersonates)
+        self.client = curl_cffi.requests.Session(
+            headers=self.headers,
+            proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None,
+            timeout=timeout,
+            impersonate=impersonate_browser,
+            verify=verify,
+        )
+        self.timeout = timeout
+        self.sleep_timestamp = 0.0
+        
+        # Utility methods
+        self.cycle = cycle
+        self.islice = islice
+
+    @cached_property
+    def parser(self) -> Any:
+        """Get HTML parser."""
+        if not LXML_AVAILABLE:
+            raise ImportError("lxml is required for HTML parsing")
+        
+        class Parser:
+            def __init__(self):
+                self.lhtml_parser = LHTMLParser(
+                    remove_blank_text=True,
+                    remove_comments=True,
+                    remove_pis=True,
+                    collect_ids=False
+                )
+                self.etree = __import__('lxml.etree', fromlist=['Element'])
+            
+            def fromstring(self, html: bytes | str) -> Any:
+                return document_fromstring(html, parser=self.lhtml_parser)
+        
+        return Parser()
+
+    def _sleep(self, sleeptime: float = 0.75) -> None:
+        """Sleep between API requests."""
+        delay = 0.0 if not self.sleep_timestamp else 0.0 if time() - self.sleep_timestamp >= 20 else sleeptime
+        self.sleep_timestamp = time()
+        sleep(delay)
+
+    def _get_url(
+        self,
+        method: str,
+        url: str,
+        params: dict[str, str] | None = None,
+        content: bytes | None = None,
+        data: dict[str, str] | None = None,
+        headers: dict[str, str] | None = None,
+        cookies: dict[str, str] | None = None,
+        json: Any = None,
+        timeout: float | None = None,
+    ) -> Any:
+        """Make HTTP request."""
+        self._sleep()
+        try:
+            request_kwargs = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+            }
+
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+
+            if method == "GET":
+                if content:
+                    request_kwargs["data"] = content
+                resp = self.client.get(url, **request_kwargs)
+            elif method == "POST":
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = self.client.post(url, **request_kwargs)
+            else:
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = self.client.request(method, url, **request_kwargs)
+        except Exception as ex:
+            if "time" in str(ex).lower():
+                raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+            raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+        
+        if resp.status_code == 200:
+            return resp
+        elif resp.status_code in (202, 301, 403, 400, 429, 418):
+            raise RatelimitE(f"{resp.url} {resp.status_code} Ratelimit")
+        raise WebscoutE(f"{resp.url} return None. {params=} {content=} {data=}")
+
+    def _get_vqd(self, keywords: str) -> str:
+        """Get vqd value for a search query."""
+        resp_content = self._get_url("GET", "https://duckduckgo.com", params={"q": keywords}).content
+        return _extract_vqd(resp_content, keywords)
+
+    def json_loads(self, obj: str | bytes) -> Any:
+        """Load JSON from string or bytes."""
+        return json_loads(obj)
+
+    def _normalize(self, text: str) -> str:
+        """Normalize text."""
+        return _normalize(text)
+
+    def _normalize_url(self, url: str) -> str:
+        """Normalize URL."""
+        return _normalize_url(url)

webscout/search/engines/duckduckgo/images.py

@@ -0,0 +1,97 @@
+"""DuckDuckGo image search."""
+
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoImages(DuckDuckGoBase):
+    """DuckDuckGo image search."""
+    
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Perform image search on DuckDuckGo.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: on, moderate, or off.
+            timelimit: d, w, m, or y.
+            size: Small, Medium, Large, Wallpaper.
+            color: color name or Monochrome.
+            type_image: photo, clipart, gif, transparent, line.
+            layout: Square, Tall, Wide.
+            license_image: any, Public, Share, etc.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of image result dictionaries.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        size = args[4] if len(args) > 4 else kwargs.get("size")
+        color = args[5] if len(args) > 5 else kwargs.get("color")
+        type_image = args[6] if len(args) > 6 else kwargs.get("type_image")
+        layout = args[7] if len(args) > 7 else kwargs.get("layout")
+        license_image = args[8] if len(args) > 8 else kwargs.get("license_image")
+        max_results = args[9] if len(args) > 9 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "1", "off": "-1"}
+        timelimit = f"time:{timelimit}" if timelimit else ""
+        size = f"size:{size}" if size else ""
+        color = f"color:{color}" if color else ""
+        type_image = f"type:{type_image}" if type_image else ""
+        layout = f"layout:{layout}" if layout else ""
+        license_image = f"license:{license_image}" if license_image else ""
+        payload = {
+            "l": region,
+            "o": "json",
+            "q": keywords,
+            "vqd": vqd,
+            "f": f"{timelimit},{size},{color},{type_image},{layout},{license_image}",
+            "p": safesearch_base[safesearch.lower()],
+        }
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _images_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/i.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                image_url = row.get("image")
+                if image_url and image_url not in cache:
+                    cache.add(image_url)
+                    result = {
+                        "title": row["title"],
+                        "image": self._normalize_url(image_url),
+                        "thumbnail": self._normalize_url(row["thumbnail"]),
+                        "url": self._normalize_url(row["url"]),
+                        "height": row["height"],
+                        "width": row["width"],
+                        "source": row["source"],
+                    }
+                    page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 500)
+            slist.extend(range(100, max_results, 100))
+        try:
+            for r in self._executor.map(_images_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))