webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/duckduckgo/base.py

@@ -0,0 +1,189 @@
+"""Base class for DuckDuckGo search implementations."""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+from functools import cached_property
+from itertools import cycle, islice
+from random import choice
+from time import sleep, time
+from typing import Any, Optional, cast
+
+try:
+    import trio  # type: ignore
+except ImportError:
+    pass
+
+import curl_cffi.requests
+
+try:
+    from lxml.html import HTMLParser as LHTMLParser
+    from lxml.html import document_fromstring
+    LXML_AVAILABLE = True
+except ImportError:
+    LXML_AVAILABLE = False
+    LHTMLParser = None # type: ignore
+    document_fromstring = None # type: ignore
+
+from ....exceptions import RatelimitE, TimeoutE, WebscoutE
+from ....litagent import LitAgent
+from ....utils import (
+    _extract_vqd,
+    _normalize,
+    _normalize_url,
+    json_loads,
+)
+
+
+class DuckDuckGoBase:
+    """Base class for DuckDuckGo search operations."""
+
+    _executor: ThreadPoolExecutor = ThreadPoolExecutor()
+    _impersonates = (
+        "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110",
+        "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133a",
+        "chrome99_android", "chrome131_android",
+        "safari15_3", "safari15_5", "safari17_0", "safari17_2_ios", "safari18_0", "safari18_0_ios",
+        "edge99", "edge101",
+        "firefox133", "firefox135",
+    )
+
+    def __init__(
+        self,
+        headers: dict[str, str] | None = None,
+        proxy: str | None = None,
+        proxies: dict[str, str] | str | None = None,
+        timeout: int | None = 10,
+        verify: bool = True,
+    ) -> None:
+        """Initialize DuckDuckGo base client.
+
+        Args:
+            headers: Dictionary of headers for the HTTP client.
+            proxy: Proxy for the HTTP client (http/https/socks5).
+            proxies: Deprecated, use proxy instead.
+            timeout: Timeout value for the HTTP client.
+            verify: SSL verification when making requests.
+        """
+        ddgs_proxy: str | None = os.environ.get("DDGS_PROXY")
+        self.proxy: str | None = ddgs_proxy if ddgs_proxy else proxy
+
+        if not proxy and proxies:
+            self.proxy = proxies.get("http") or proxies.get("https") if isinstance(proxies, dict) else proxies
+
+        default_headers = {
+            **LitAgent().generate_fingerprint(),
+            "Origin": "https://duckduckgo.com",
+            "Referer": "https://duckduckgo.com/",
+        }
+
+        self.headers = headers if headers else {}
+        self.headers.update(default_headers)
+
+        impersonate_browser = choice(self._impersonates)
+        self.client = curl_cffi.requests.Session(
+            headers=self.headers,
+            proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None,
+            timeout=timeout,
+            impersonate=impersonate_browser,
+            verify=verify,
+        )
+        self.timeout = timeout
+        self.sleep_timestamp = 0.0
+
+        # Utility methods
+        self.cycle = cycle
+        self.islice = islice
+
+    @cached_property
+    def parser(self) -> Any:
+        """Get HTML parser."""
+        if not LXML_AVAILABLE:
+            raise ImportError("lxml is required for HTML parsing")
+
+        class Parser:
+            def __init__(self):
+                self.lhtml_parser = LHTMLParser(
+                    remove_blank_text=True,
+                    remove_comments=True,
+                    remove_pis=True,
+                    collect_ids=False
+                ) if LHTMLParser else None
+                self.etree = __import__('lxml.etree', fromlist=['Element'])
+
+            def fromstring(self, html: bytes | str) -> Any:
+                return document_fromstring(html, parser=self.lhtml_parser) if document_fromstring and self.lhtml_parser else None
+
+        return Parser()
+
+    def _sleep(self, sleeptime: float = 0.75) -> None:
+        """Sleep between API requests."""
+        delay = 0.0 if not self.sleep_timestamp else 0.0 if time() - self.sleep_timestamp >= 20 else sleeptime
+        self.sleep_timestamp = time()
+        sleep(delay)
+
+    def _get_url(
+        self,
+        method: str,
+        url: str,
+        params: dict[str, str] | None = None,
+        content: bytes | None = None,
+        data: dict[str, str] | None = None,
+        headers: dict[str, str] | None = None,
+        cookies: dict[str, str] | None = None,
+        json: Any = None,
+        timeout: float | None = None,
+    ) -> Any:
+        """Make HTTP request."""
+        self._sleep()
+        try:
+            request_kwargs: dict[str, Any] = {
+                "params": params,
+                "headers": headers,
+                "json": json,
+                "timeout": timeout or self.timeout,
+            }
+
+            if isinstance(cookies, dict):
+                request_kwargs["cookies"] = cookies
+
+            if method == "GET":
+                if content:
+                    request_kwargs["data"] = content
+                resp = self.client.get(url, **request_kwargs)
+            elif method == "POST":
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = self.client.post(url, **request_kwargs)
+            else:
+                if data or content:
+                    request_kwargs["data"] = data or content
+                resp = self.client.request(cast(Any, method), url, **request_kwargs)
+        except Exception as ex:
+            if "time" in str(ex).lower():
+                raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+            raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex
+
+        if resp.status_code == 200:
+            return resp
+        elif resp.status_code in (202, 301, 403, 400, 429, 418):
+            raise RatelimitE(f"{resp.url} {resp.status_code} Ratelimit")
+        raise WebscoutE(f"{resp.url} return None. {params=} {content=} {data=}")
+
+    def _get_vqd(self, keywords: str) -> str:
+        """Get vqd value for a search query."""
+        resp_content = self._get_url("GET", "https://duckduckgo.com", params={"q": keywords}).content
+        return _extract_vqd(resp_content, keywords)
+
+    def json_loads(self, obj: str | bytes) -> Any:
+        """Load JSON from string or bytes."""
+        return json_loads(obj)
+
+    def _normalize(self, text: str) -> str:
+        """Normalize text."""
+        return _normalize(text)
+
+    def _normalize_url(self, url: str) -> str:
+        """Normalize URL."""
+        return _normalize_url(url)

webscout/search/engines/duckduckgo/images.py

@@ -0,0 +1,100 @@
+"""DuckDuckGo image search."""
+
+from __future__ import annotations
+
+from ....search.results import ImagesResult
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoImages(DuckDuckGoBase):
+    """DuckDuckGo image search."""
+
+    name = "duckduckgo"
+    category = "images"
+
+    def run(self, *args, **kwargs) -> list[ImagesResult]:
+        """Perform image search on DuckDuckGo.
+
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: on, moderate, or off.
+            timelimit: d, w, m, or y.
+            size: Small, Medium, Large, Wallpaper.
+            color: color name or Monochrome.
+            type_image: photo, clipart, gif, transparent, line.
+            layout: Square, Tall, Wide.
+            license_image: any, Public, Share, etc.
+            max_results: Maximum number of results.
+
+        Returns:
+            List of ImagesResult objects.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        size = args[4] if len(args) > 4 else kwargs.get("size")
+        color = args[5] if len(args) > 5 else kwargs.get("color")
+        type_image = args[6] if len(args) > 6 else kwargs.get("type_image")
+        layout = args[7] if len(args) > 7 else kwargs.get("layout")
+        license_image = args[8] if len(args) > 8 else kwargs.get("license_image")
+        max_results = args[9] if len(args) > 9 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "1", "off": "-1"}
+        timelimit = f"time:{timelimit}" if timelimit else ""
+        size = f"size:{size}" if size else ""
+        color = f"color:{color}" if color else ""
+        type_image = f"type:{type_image}" if type_image else ""
+        layout = f"layout:{layout}" if layout else ""
+        license_image = f"license:{license_image}" if license_image else ""
+        payload = {
+            "l": region,
+            "o": "json",
+            "q": keywords,
+            "vqd": vqd,
+            "f": f"{timelimit},{size},{color},{type_image},{layout},{license_image}",
+            "p": safesearch_base[safesearch.lower()],
+        }
+
+        cache = set()
+        results: list[ImagesResult] = []
+
+        def _images_page(s: int) -> list[ImagesResult]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/i.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                image_url = row.get("image")
+                if image_url and image_url not in cache:
+                    cache.add(image_url)
+                    result = ImagesResult(
+                        title=row["title"],
+                        image=self._normalize_url(image_url),
+                        thumbnail=self._normalize_url(row["thumbnail"]),
+                        url=self._normalize_url(row["url"]),
+                        height=row["height"],
+                        width=row["width"],
+                        source=row["source"],
+                    )
+                    page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 500)
+            slist.extend(range(100, max_results, 100))
+        try:
+            for r in self._executor.map(_images_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))

webscout/search/engines/duckduckgo/maps.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from math import sqrt
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoMaps(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "maps"
+    def _calculate_distance(self, lat_t: Decimal, lon_l: Decimal, lat_b: Decimal, lon_r: Decimal) -> float:
+        """Calculate the Euclidean distance between top-left and bottom-right corners of bounding box."""
+        # Convert to float for math operations
+        lat_t_f = float(lat_t)
+        lon_l_f = float(lon_l)
+        lat_b_f = float(lat_b)
+        lon_r_f = float(lon_r)
+
+        # Calculate Euclidean distance
+        distance = sqrt((lat_t_f - lat_b_f) ** 2 + (lon_r_f - lon_l_f) ** 2)
+        return distance
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        place = args[1] if len(args) > 1 else kwargs.get("place")
+        street = args[2] if len(args) > 2 else kwargs.get("street")
+        city = args[3] if len(args) > 3 else kwargs.get("city")
+        county = args[4] if len(args) > 4 else kwargs.get("county")
+        state = args[5] if len(args) > 5 else kwargs.get("state")
+        country = args[6] if len(args) > 6 else kwargs.get("country")
+        postalcode = args[7] if len(args) > 7 else kwargs.get("postalcode")
+        latitude = args[8] if len(args) > 8 else kwargs.get("latitude")
+        longitude = args[9] if len(args) > 9 else kwargs.get("longitude")
+        radius = args[10] if len(args) > 10 else kwargs.get("radius", 0)
+        max_results = args[11] if len(args) > 11 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        # if longitude and latitude are specified, skip the request about bbox to the nominatim api
+        if latitude and longitude:
+            lat_t = Decimal(latitude.replace(",", "."))
+            lat_b = Decimal(latitude.replace(",", "."))
+            lon_l = Decimal(longitude.replace(",", "."))
+            lon_r = Decimal(longitude.replace(",", "."))
+            if radius == 0:
+                radius = 1
+        # otherwise request about bbox to nominatim api
+        else:
+            if place:
+                params = {
+                    "q": place,
+                    "polygon_geojson": "0",
+                    "format": "jsonv2",
+                }
+            else:
+                params = {
+                    "polygon_geojson": "0",
+                    "format": "jsonv2",
+                }
+                if street:
+                    params["street"] = street
+                if city:
+                    params["city"] = city
+                if county:
+                    params["county"] = county
+                if state:
+                    params["state"] = state
+                if country:
+                    params["country"] = country
+                if postalcode:
+                    params["postalcode"] = postalcode
+            # request nominatim api to get coordinates box
+            resp_content = self._get_url(
+                "GET",
+                "https://nominatim.openstreetmap.org/search.php",
+                params=params,
+            ).content
+            if resp_content == b"[]":
+                raise WebscoutE("maps() Coordinates are not found, check function parameters.")
+            resp_json = self.json_loads(resp_content)
+            coordinates = resp_json[0]["boundingbox"]
+            lat_t, lon_l = Decimal(coordinates[1]), Decimal(coordinates[2])
+            lat_b, lon_r = Decimal(coordinates[0]), Decimal(coordinates[3])
+
+        # if a radius is specified, expand the search square
+        lat_t += Decimal(radius) * Decimal(0.008983)
+        lat_b -= Decimal(radius) * Decimal(0.008983)
+        lon_l -= Decimal(radius) * Decimal(0.008983)
+        lon_r += Decimal(radius) * Decimal(0.008983)
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _maps_page(
+            bbox: tuple[Decimal, Decimal, Decimal, Decimal],
+        ) -> list[dict[str, str]] | None:
+            if max_results and len(results) >= max_results:
+                return None
+            lat_t, lon_l, lat_b, lon_r = bbox
+            params = {
+                "q": keywords,
+                "vqd": vqd,
+                "tg": "maps_places",
+                "rt": "D",
+                "mkexp": "b",
+                "wiki_info": "1",
+                "is_requery": "1",
+                "bbox_tl": f"{lat_t},{lon_l}",
+                "bbox_br": f"{lat_b},{lon_r}",
+                "strict_bbox": "1",
+            }
+            resp_content = self._get_url("GET", "https://duckduckgo.com/local.js", params=params).content
+            resp_json = self.json_loads(resp_content)
+            page_data = resp_json.get("results", [])
+
+            page_results = []
+            for res in page_data:
+                r_name = f'{res["name"]} {res["address"]}'
+                if r_name in cache:
+                    continue
+                else:
+                    cache.add(r_name)
+                    result = {
+                        "title": res["name"],
+                        "address": res["address"],
+                        "country_code": res["country_code"],
+                        "url": self._normalize_url(res["website"]),
+                        "phone": res["phone"] or "",
+                        "latitude": res["coordinates"]["latitude"],
+                        "longitude": res["coordinates"]["longitude"],
+                        "source": self._normalize_url(res["url"]),
+                        "image": x.get("image", "") if (x := res["embed"]) else "",
+                        "desc": x.get("description", "") if (x := res["embed"]) else "",
+                        "hours": res["hours"] or "",
+                        "category": res["ddg_category"] or "",
+                        "facebook": f"www.facebook.com/profile.php?id={x}" if (x := res["facebook_id"]) else "",
+                        "instagram": f"https://www.instagram.com/{x}" if (x := res["instagram_id"]) else "",
+                        "twitter": f"https://twitter.com/{x}" if (x := res["twitter_id"]) else "",
+                    }
+                    page_results.append(result)
+            return page_results
+
+        start_bbox = (lat_t, lon_l, lat_b, lon_r)
+        work_bboxes = [start_bbox]
+        while work_bboxes:
+            queue_bboxes = []
+            tasks = []
+            for bbox in work_bboxes:
+                tasks.append(bbox)
+                if self._calculate_distance(lat_t, lon_l, lat_b, lon_r) > 1:
+                    lat_t, lon_l, lat_b, lon_r = bbox
+                    lat_middle = (lat_t + lat_b) / 2
+                    lon_middle = (lon_l + lon_r) / 2
+                    bbox1 = (lat_t, lon_l, lat_middle, lon_middle)
+                    bbox2 = (lat_t, lon_middle, lat_middle, lon_r)
+                    bbox3 = (lat_middle, lon_l, lat_b, lon_middle)
+                    bbox4 = (lat_middle, lon_middle, lat_b, lon_r)
+                    queue_bboxes.extend([bbox1, bbox2, bbox3, bbox4])
+
+            work_bboxes_results = []
+            try:
+                for r in self._executor.map(_maps_page, tasks):
+                    if r:
+                        work_bboxes_results.extend(r)
+            except Exception as e:
+                raise e
+
+            for x in work_bboxes_results:
+                if isinstance(x, list):
+                    results.extend(x)
+                elif isinstance(x, dict):
+                    results.append(x)
+
+            work_bboxes = queue_bboxes
+            if not max_results or len(results) >= max_results or len(work_bboxes_results) == 0:
+                break
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/news.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from ....search.results import NewsResult
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoNews(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "news"
+    def run(self, *args, **kwargs) -> list[NewsResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        max_results = args[4] if len(args) > 4 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "-1", "off": "-2"}
+        payload = {
+            "l": region,
+            "o": "json",
+            "noamp": "1",
+            "q": keywords,
+            "vqd": vqd,
+            "p": safesearch_base[safesearch.lower()],
+        }
+        if timelimit:
+            payload["df"] = timelimit
+
+        cache = set()
+        results: list[NewsResult] = []
+
+        def _news_page(s: int) -> list[NewsResult]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/news.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                if row["url"] not in cache:
+                    cache.add(row["url"])
+                    image_url = row.get("image", None)
+                    result = NewsResult(
+                        date=datetime.fromtimestamp(row["date"], timezone.utc).isoformat(),
+                        title=row["title"],
+                        body=self._normalize(row["excerpt"]),
+                        url=self._normalize_url(row["url"]),
+                        image=self._normalize_url(image_url),
+                        source=row["source"],
+                    )
+                    page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 120)
+            slist.extend(range(30, max_results, 30))
+        try:
+            for r in self._executor.map(_news_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/suggestions.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoSuggestions(DuckDuckGoBase):
+    name = "duckduckgo"
+    category = "suggestions"
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "kl": region,
+        }
+        resp_content = self._get_url("GET", "https://duckduckgo.com/ac/", params=payload).content
+        page_data = self.json_loads(resp_content)
+        return [r for r in page_data]
+