webscout 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/search/engines/duckduckgo/maps.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from decimal import Decimal
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoMaps(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        place = args[1] if len(args) > 1 else kwargs.get("place")
+        street = args[2] if len(args) > 2 else kwargs.get("street")
+        city = args[3] if len(args) > 3 else kwargs.get("city")
+        county = args[4] if len(args) > 4 else kwargs.get("county")
+        state = args[5] if len(args) > 5 else kwargs.get("state")
+        country = args[6] if len(args) > 6 else kwargs.get("country")
+        postalcode = args[7] if len(args) > 7 else kwargs.get("postalcode")
+        latitude = args[8] if len(args) > 8 else kwargs.get("latitude")
+        longitude = args[9] if len(args) > 9 else kwargs.get("longitude")
+        radius = args[10] if len(args) > 10 else kwargs.get("radius", 0)
+        max_results = args[11] if len(args) > 11 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        # if longitude and latitude are specified, skip the request about bbox to the nominatim api
+        if latitude and longitude:
+            lat_t = Decimal(latitude.replace(",", "."))
+            lat_b = Decimal(latitude.replace(",", "."))
+            lon_l = Decimal(longitude.replace(",", "."))
+            lon_r = Decimal(longitude.replace(",", "."))
+            if radius == 0:
+                radius = 1
+        # otherwise request about bbox to nominatim api
+        else:
+            if place:
+                params = {
+                    "q": place,
+                    "polygon_geojson": "0",
+                    "format": "jsonv2",
+                }
+            else:
+                params = {
+                    "polygon_geojson": "0",
+                    "format": "jsonv2",
+                }
+                if street:
+                    params["street"] = street
+                if city:
+                    params["city"] = city
+                if county:
+                    params["county"] = county
+                if state:
+                    params["state"] = state
+                if country:
+                    params["country"] = country
+                if postalcode:
+                    params["postalcode"] = postalcode
+            # request nominatim api to get coordinates box
+            resp_content = self._get_url(
+                "GET",
+                "https://nominatim.openstreetmap.org/search.php",
+                params=params,
+            ).content
+            if resp_content == b"[]":
+                raise WebscoutE("maps() Coordinates are not found, check function parameters.")
+            resp_json = self.json_loads(resp_content)
+            coordinates = resp_json[0]["boundingbox"]
+            lat_t, lon_l = Decimal(coordinates[1]), Decimal(coordinates[2])
+            lat_b, lon_r = Decimal(coordinates[0]), Decimal(coordinates[3])
+
+        # if a radius is specified, expand the search square
+        lat_t += Decimal(radius) * Decimal(0.008983)
+        lat_b -= Decimal(radius) * Decimal(0.008983)
+        lon_l -= Decimal(radius) * Decimal(0.008983)
+        lon_r += Decimal(radius) * Decimal(0.008983)
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _maps_page(
+            bbox: tuple[Decimal, Decimal, Decimal, Decimal],
+        ) -> list[dict[str, str]] | None:
+            if max_results and len(results) >= max_results:
+                return None
+            lat_t, lon_l, lat_b, lon_r = bbox
+            params = {
+                "q": keywords,
+                "vqd": vqd,
+                "tg": "maps_places",
+                "rt": "D",
+                "mkexp": "b",
+                "wiki_info": "1",
+                "is_requery": "1",
+                "bbox_tl": f"{lat_t},{lon_l}",
+                "bbox_br": f"{lat_b},{lon_r}",
+                "strict_bbox": "1",
+            }
+            resp_content = self._get_url("GET", "https://duckduckgo.com/local.js", params=params).content
+            resp_json = self.json_loads(resp_content)
+            page_data = resp_json.get("results", [])
+
+            page_results = []
+            for res in page_data:
+                r_name = f'{res["name"]} {res["address"]}'
+                if r_name in cache:
+                    continue
+                else:
+                    cache.add(r_name)
+                    result = {
+                        "title": res["name"],
+                        "address": res["address"],
+                        "country_code": res["country_code"],
+                        "url": self._normalize_url(res["website"]),
+                        "phone": res["phone"] or "",
+                        "latitude": res["coordinates"]["latitude"],
+                        "longitude": res["coordinates"]["longitude"],
+                        "source": self._normalize_url(res["url"]),
+                        "image": x.get("image", "") if (x := res["embed"]) else "",
+                        "desc": x.get("description", "") if (x := res["embed"]) else "",
+                        "hours": res["hours"] or "",
+                        "category": res["ddg_category"] or "",
+                        "facebook": f"www.facebook.com/profile.php?id={x}" if (x := res["facebook_id"]) else "",
+                        "instagram": f"https://www.instagram.com/{x}" if (x := res["instagram_id"]) else "",
+                        "twitter": f"https://twitter.com/{x}" if (x := res["twitter_id"]) else "",
+                    }
+                    page_results.append(result)
+            return page_results
+
+        start_bbox = (lat_t, lon_l, lat_b, lon_r)
+        work_bboxes = [start_bbox]
+        while work_bboxes:
+            queue_bboxes = []
+            tasks = []
+            for bbox in work_bboxes:
+                tasks.append(bbox)
+                if self._calculate_distance(lat_t, lon_l, lat_b, lon_r) > 1:
+                    lat_t, lon_l, lat_b, lon_r = bbox
+                    lat_middle = (lat_t + lat_b) / 2
+                    lon_middle = (lon_l + lon_r) / 2
+                    bbox1 = (lat_t, lon_l, lat_middle, lon_middle)
+                    bbox2 = (lat_t, lon_middle, lat_middle, lon_r)
+                    bbox3 = (lat_middle, lon_l, lat_b, lon_middle)
+                    bbox4 = (lat_middle, lon_middle, lat_b, lon_r)
+                    queue_bboxes.extend([bbox1, bbox2, bbox3, bbox4])
+
+            work_bboxes_results = []
+            try:
+                for r in self._executor.map(_maps_page, tasks):
+                    if r:
+                        work_bboxes_results.extend(r)
+            except Exception as e:
+                raise e
+
+            for x in work_bboxes_results:
+                if isinstance(x, list):
+                    results.extend(x)
+                elif isinstance(x, dict):
+                    results.append(x)
+
+            work_bboxes = queue_bboxes
+            if not max_results or len(results) >= max_results or len(work_bboxes_results) == 0:
+                break
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/news.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoNews(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        max_results = args[4] if len(args) > 4 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "-1", "off": "-2"}
+        payload = {
+            "l": region,
+            "o": "json",
+            "noamp": "1",
+            "q": keywords,
+            "vqd": vqd,
+            "p": safesearch_base[safesearch.lower()],
+        }
+        if timelimit:
+            payload["df"] = timelimit
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _news_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/news.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                if row["url"] not in cache:
+                    cache.add(row["url"])
+                    image_url = row.get("image", None)
+                    result = {
+                        "date": datetime.fromtimestamp(row["date"], timezone.utc).isoformat(),
+                        "title": row["title"],
+                        "body": self._normalize(row["excerpt"]),
+                        "url": self._normalize_url(row["url"]),
+                        "image": self._normalize_url(image_url),
+                        "source": row["source"],
+                    }
+                    page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 120)
+            slist.extend(range(30, max_results, 30))
+        try:
+            for r in self._executor.map(_news_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/suggestions.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoSuggestions(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "kl": region,
+        }
+        resp_content = self._get_url("GET", "https://duckduckgo.com/ac/", params=payload).content
+        page_data = self.json_loads(resp_content)
+        return [r for r in page_data]
+

webscout/search/engines/duckduckgo/text.py

@@ -0,0 +1,211 @@
+"""DuckDuckGo text search."""
+
+from __future__ import annotations
+
+import warnings
+from random import shuffle
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTextSearch(DuckDuckGoBase):
+    """DuckDuckGo text/web search."""
+    
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Perform text search on DuckDuckGo.
+        
+        Args:
+            keywords: Search query.
+            region: Region code (e.g., wt-wt, us-en).
+            safesearch: on, moderate, or off.
+            timelimit: d, w, m, or y.
+            backend: html, lite, or auto.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of search result dictionaries.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        backend = args[4] if len(args) > 4 else kwargs.get("backend", "auto")
+        max_results = args[5] if len(args) > 5 else kwargs.get("max_results")
+
+        if backend in ("api", "ecosia"):
+            warnings.warn(f"{backend=} is deprecated, using backend='auto'", stacklevel=2)
+            backend = "auto"
+        backends = ["html", "lite"] if backend == "auto" else [backend]
+        shuffle(backends)
+
+        results, err = [], None
+        for b in backends:
+            try:
+                if b == "html":
+                    results = self._text_html(keywords, region, timelimit, max_results)
+                elif b == "lite":
+                    results = self._text_lite(keywords, region, timelimit, max_results)
+                return results
+            except Exception as ex:
+                err = ex
+
+        raise WebscoutE(err)
+
+    def _text_html(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Text search using HTML backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+        if max_results and max_results > 20:
+            vqd = self._get_vqd(keywords)
+            payload["vqd"] = vqd
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _text_html_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://html.duckduckgo.com/html", data=payload).content
+            if b"No  results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//div[h2]")
+            if not isinstance(elements, list):
+                return []
+            for e in elements:
+                if isinstance(e, self.parser.etree.Element):
+                    hrefxpath = e.xpath("./a/@href")
+                    href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                    if (
+                        href
+                        and href not in cache
+                        and not href.startswith(
+                            ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                        )
+                    ):
+                        cache.add(href)
+                        titlexpath = e.xpath("./h2/a/text()")
+                        title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                        bodyxpath = e.xpath("./a//text()")
+                        body = "".join(str(x) for x in bodyxpath) if bodyxpath and isinstance(bodyxpath, list) else ""
+                        result = {
+                            "title": self._normalize(title),
+                            "href": self._normalize_url(href),
+                            "body": self._normalize(body),
+                        }
+                        page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_html_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+
+    def _text_lite(
+        self,
+        keywords: str,
+        region: str = "wt-wt",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Text search using lite backend."""
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": keywords,
+            "s": "0",
+            "o": "json",
+            "api": "d.js",
+            "vqd": "",
+            "kl": region,
+            "bing_market": region,
+        }
+        if timelimit:
+            payload["df"] = timelimit
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _text_lite_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("POST", "https://lite.duckduckgo.com/lite/", data=payload).content
+            if b"No more results." in resp_content:
+                return []
+
+            page_results = []
+            tree = self.parser.fromstring(resp_content)
+            elements = tree.xpath("//table[last()]//tr")
+            if not isinstance(elements, list):
+                return []
+
+            data = zip(self.cycle(range(1, 5)), elements)
+            for i, e in data:
+                if isinstance(e, self.parser.etree.Element):
+                    if i == 1:
+                        hrefxpath = e.xpath(".//a//@href")
+                        href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None
+                        if (
+                            href is None
+                            or href in cache
+                            or href.startswith(
+                                ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain")
+                            )
+                        ):
+                            [next(data, None) for _ in range(3)]  # skip block(i=1,2,3,4)
+                        else:
+                            cache.add(href)
+                            titlexpath = e.xpath(".//a//text()")
+                            title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else ""
+                    elif i == 2:
+                        bodyxpath = e.xpath(".//td[@class='result-snippet']//text()")
+                        body = (
+                            "".join(str(x) for x in bodyxpath).strip()
+                            if bodyxpath and isinstance(bodyxpath, list)
+                            else ""
+                        )
+                        if href:
+                            result = {
+                                "title": self._normalize(title),
+                                "href": self._normalize_url(href),
+                                "body": self._normalize(body),
+                            }
+                            page_results.append(result)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 2023)
+            slist.extend(range(23, max_results, 50))
+        try:
+            for r in self._executor.map(_text_lite_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))

webscout/search/engines/duckduckgo/translate.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoTranslate(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        from_ = args[1] if len(args) > 1 else kwargs.get("from_")
+        to = args[2] if len(args) > 2 else kwargs.get("to", "en")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd("translate")
+
+        payload = {
+            "vqd": vqd,
+            "query": "translate",
+            "to": to,
+        }
+        if from_:
+            payload["from"] = from_
+
+        def _translate_keyword(keyword: str) -> dict[str, str]:
+            resp_content = self._get_url(
+                "POST",
+                "https://duckduckgo.com/translation.js",
+                params=payload,
+                content=keyword.encode(),
+            ).content
+            page_data: dict[str, str] = self.json_loads(resp_content)
+            page_data["original"] = keyword
+            return page_data
+
+        if isinstance(keywords, str):
+            keywords = [keywords]
+
+        results = []
+        try:
+            for r in self._executor.map(_translate_keyword, keywords):
+                results.append(r)
+        except Exception as e:
+            raise e
+
+        return results
+

webscout/search/engines/duckduckgo/videos.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoVideos(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "wt-wt")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        timelimit = args[3] if len(args) > 3 else kwargs.get("timelimit")
+        resolution = args[4] if len(args) > 4 else kwargs.get("resolution")
+        duration = args[5] if len(args) > 5 else kwargs.get("duration")
+        license_videos = args[6] if len(args) > 6 else kwargs.get("license_videos")
+        max_results = args[7] if len(args) > 7 else kwargs.get("max_results")
+
+        assert keywords, "keywords is mandatory"
+
+        vqd = self._get_vqd(keywords)
+
+        safesearch_base = {"on": "1", "moderate": "-1", "off": "-2"}
+        timelimit = f"publishedAfter:{timelimit}" if timelimit else ""
+        resolution = f"videoDefinition:{resolution}" if resolution else ""
+        duration = f"videoDuration:{duration}" if duration else ""
+        license_videos = f"videoLicense:{license_videos}" if license_videos else ""
+        payload = {
+            "l": region,
+            "o": "json",
+            "q": keywords,
+            "vqd": vqd,
+            "f": f"{timelimit},{resolution},{duration},{license_videos}",
+            "p": safesearch_base[safesearch.lower()],
+        }
+
+        cache = set()
+        results: list[dict[str, str]] = []
+
+        def _videos_page(s: int) -> list[dict[str, str]]:
+            payload["s"] = f"{s}"
+            resp_content = self._get_url("GET", "https://duckduckgo.com/v.js", params=payload).content
+            resp_json = self.json_loads(resp_content)
+
+            page_data = resp_json.get("results", [])
+            page_results = []
+            for row in page_data:
+                if row["content"] not in cache:
+                    cache.add(row["content"])
+                    page_results.append(row)
+            return page_results
+
+        slist = [0]
+        if max_results:
+            max_results = min(max_results, 400)
+            slist.extend(range(60, max_results, 60))
+        try:
+            for r in self._executor.map(_videos_page, slist):
+                results.extend(r)
+        except Exception as e:
+            raise e
+
+        return list(self.islice(results, max_results))
+

webscout/search/engines/duckduckgo/weather.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from urllib.parse import quote
+
+from ....exceptions import WebscoutE
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoWeather(DuckDuckGoBase):
+    def run(self, *args, **kwargs) -> dict[str, any]:
+        location = args[0] if args else kwargs.get("location")
+        language = args[1] if len(args) > 1 else kwargs.get("language", "en")
+
+        assert location, "location is mandatory"
+        lang = language.split('-')[0]
+        url = f"https://duckduckgo.com/js/spice/forecast/{quote(location)}/{lang}"
+
+        resp = self._get_url("GET", url).content
+        resp_text = resp.decode('utf-8')
+
+        if "ddg_spice_forecast(" not in resp_text:
+            raise WebscoutE(f"No weather data found for {location}")
+
+        json_text = resp_text[resp_text.find('(') + 1:resp_text.rfind(')')]
+        try:
+            result = json.loads(json_text)
+        except Exception as e:
+            raise WebscoutE(f"Error parsing weather JSON: {e}")
+
+        if not result or 'currentWeather' not in result or 'forecastDaily' not in result:
+            raise WebscoutE(f"Invalid weather data format for {location}")
+
+        formatted_data = {
+            "location": result["currentWeather"]["metadata"].get("ddg-location", "Unknown"),
+            "current": {
+                "condition": result["currentWeather"].get("conditionCode"),
+                "temperature_c": result["currentWeather"].get("temperature"),
+                "feels_like_c": result["currentWeather"].get("temperatureApparent"),
+                "humidity": result["currentWeather"].get("humidity"),
+                "wind_speed_ms": result["currentWeather"].get("windSpeed"),
+                "wind_direction": result["currentWeather"].get("windDirection"),
+                "visibility_m": result["currentWeather"].get("visibility"),
+            },
+            "daily_forecast": [],
+            "hourly_forecast": []
+        }
+
+        for day in result["forecastDaily"]["days"]:
+            formatted_data["daily_forecast"].append({
+                "date": datetime.fromisoformat(day["forecastStart"].replace("Z", "+00:00")).strftime("%Y-%m-%d"),
+                "condition": day["daytimeForecast"].get("conditionCode"),
+                "max_temp_c": day["temperatureMax"],
+                "min_temp_c": day["temperatureMin"],
+                "sunrise": datetime.fromisoformat(day["sunrise"].replace("Z", "+00:00")).strftime("%H:%M"),
+                "sunset": datetime.fromisoformat(day["sunset"].replace("Z", "+00:00")).strftime("%H:%M"),
+            })
+
+        if 'forecastHourly' in result and 'hours' in result['forecastHourly']:
+            for hour in result['forecastHourly']['hours']:
+                formatted_data["hourly_forecast"].append({
+                    "time": datetime.fromisoformat(hour["forecastStart"].replace("Z", "+00:00")).strftime("%H:%M"),
+                    "condition": hour.get("conditionCode"),
+                    "temperature_c": hour.get("temperature"),
+                    "feels_like_c": hour.get("temperatureApparent"),
+                    "humidity": hour.get("humidity"),
+                    "wind_speed_ms": hour.get("windSpeed"),
+                    "wind_direction": hour.get("windDirection"),
+                    "visibility_m": hour.get("visibility"),
+                })
+
+        return formatted_data
+