webscout 1.0.0__py3-none-any.whl

webscout/__init__.py

@@ -0,0 +1,18 @@
+"""webscout.
+
+Search for words, documents, images, videos, news, maps and text translation 
+using the DuckDuckGo.com search engine.
+"""
+
+import logging
+
+# votext: noqa: F401
+from .webscout_search import DDGS
+from .webscout_search_async import AsyncDDGS
+from .version import __version__
+
+__all__ = ["DDGS", "AsyncDDGS", "__version__", "cli"]
+
+# A do-nothing logging handler
+# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
+logging.getLogger("duckduckgo_search").addHandler(logging.NullHandler())

webscout/__main__.py

@@ -0,0 +1,5 @@
+"""For using as 'python3 -m webscout'."""
+from .cli import cli
+
+if __name__ == "__main__":
+    cli(prog_name="duckduckgo_search")

webscout/cli.py

@@ -0,0 +1,432 @@
+import csv
+import json
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from urllib.parse import unquote
+
+import click
+from curl_cffi import requests
+
+from .webscout_search import DDGS
+from .version import __version__
+
+logger = logging.getLogger(__name__)
+
+COLORS = {
+    0: "black",
+    1: "red",
+    2: "green",
+    3: "yellow",
+    4: "blue",
+    5: "magenta",
+    6: "cyan",
+    7: "bright_black",
+    8: "bright_red",
+    9: "bright_green",
+    10: "bright_yellow",
+    11: "bright_blue",
+    12: "bright_magenta",
+    13: "bright_cyan",
+    14: "white",
+    15: "bright_white",
+}
+
+
+def _save_json(jsonfile, data):
+    with open(jsonfile, "w", encoding="utf-8") as file:
+        json.dump(data, file, ensure_ascii=False, indent=4)
+
+
+def _save_csv(csvfile, data):
+    with open(csvfile, "w", newline="", encoding="utf-8") as file:
+        if data:
+            headers = data[0].keys()
+            writer = csv.DictWriter(file, fieldnames=headers, quoting=csv.QUOTE_MINIMAL)
+            writer.writeheader()
+            writer.writerows(data)
+
+
+def _print_data(data):
+    if data:
+        for i, e in enumerate(data, start=1):
+            click.secho(f"{i}.\t    {'=' * 78}", bg="black", fg="white")
+            for j, (k, v) in enumerate(e.items(), start=1):
+                if v:
+                    width = 300 if k in ("content", "href", "image", "source", "thumbnail", "url") else 78
+                    k = "language" if k == "detected_language" else k
+                    text = click.wrap_text(
+                        f"{v}", width=width, initial_indent="", subsequent_indent=" " * 12, preserve_paragraphs=True
+                    )
+                else:
+                    text = v
+                click.secho(f"{k:<12}{text}", bg="black", fg=COLORS[j], overline=True)
+            input()
+
+
+def _sanitize_keywords(keywords):
+    keywords = (
+        keywords.replace("filetype", "")
+        .replace(":", "")
+        .replace('"', "'")
+        .replace("site", "")
+        .replace(" ", "_")
+        .replace("/", "_")
+        .replace("\\", "_")
+        .replace(" ", "")
+    )
+    return keywords
+
+
+def _download_file(url, dir_path, filename, proxy):
+    try:
+        resp = requests.get(url, proxies=proxy, impersonate="chrome", timeout=10)
+        resp.raise_for_status()
+        with open(os.path.join(dir_path, filename[:200]), "wb") as file:
+            file.write(resp.content)
+    except Exception as ex:
+        logger.debug(f"download_file url={url} {type(ex).__name__} {ex}")
+
+
+def _download_results(keywords, results, images=False, proxy=None, threads=None):
+    path_type = "images" if images else "text"
+    path = f"{path_type}_{keywords}_{datetime.now():%Y%m%d_%H%M%S}"
+    os.makedirs(path, exist_ok=True)
+    proxy = {"http": proxy, "https": proxy}
+
+    threads = 10 if threads is None else threads
+    with ThreadPoolExecutor(max_workers=threads) as executor:
+        futures = []
+        for i, res in enumerate(results, start=1):
+            url = res["image"] if images else res["href"]
+            filename = unquote(url.split("/")[-1].split("?")[0])
+            f = executor.submit(_download_file, url, path, f"{i}_{filename}", proxy)
+            futures.append(f)
+
+        with click.progressbar(
+            length=len(futures), label="Downloading", show_percent=True, show_pos=True, width=50
+        ) as bar:
+            for future in as_completed(futures):
+                future.result()
+                bar.update(1)
+
+
+@click.group(chain=True)
+def cli():
+    """A decorator that creates a command group.
+
+    This decorator is used to create a group of commands.
+    The `chain=True` parameter allows the commands in the group to be chained together,
+    meaning the output of one command can be used as the input for the next command.
+    """
+    pass
+
+
+@cli.command()
+def version():
+    """A command-line interface command that prints and returns the current version of the program."""
+    print(__version__)
+    return __version__
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="text search, keywords for query")
+@click.option("-r", "--region", default="wt-wt", help="wt-wt, us-en, ru-ru, etc. -region https://duckduckgo.com/params")
+@click.option("-s", "--safesearch", default="moderate", type=click.Choice(["on", "moderate", "off"]))
+@click.option("-t", "--timelimit", default=None, type=click.Choice(["d", "w", "m", "y"]), help="day, week, month, year")
+@click.option("-m", "--max_results", default=20, help="maximum number of results, default=20")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-d", "--download", is_flag=True, default=False, help="download results to 'keywords' folder")
+@click.option("-b", "--backend", default="api", type=click.Choice(["api", "html", "lite"]), help="which backend to use")
+@click.option("-th", "--threads", default=10, help="download threads, default=10")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def text(keywords, region, safesearch, timelimit, backend, output, download, threads, max_results, proxy):
+    """CLI function to perform a text search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).text(
+        keywords=keywords,
+        region=region,
+        safesearch=safesearch,
+        timelimit=timelimit,
+        backend=backend,
+        max_results=max_results,
+    ):
+        data.append(r)
+    keywords = _sanitize_keywords(keywords)
+    filename = f"text_{keywords}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print" and not download:
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+    if download:
+        _download_results(keywords, data, proxy=proxy, threads=threads)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="answers search, keywords for query")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def answers(keywords, output, proxy):
+    """CLI function to perform a answers search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).answers(keywords=keywords):
+        data.append(r)
+    filename = f"answers_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="keywords for query")
+@click.option("-r", "--region", default="wt-wt", help="wt-wt, us-en, ru-ru, etc. -region https://duckduckgo.com/params")
+@click.option("-s", "--safesearch", default="moderate", type=click.Choice(["on", "moderate", "off"]))
+@click.option("-t", "--timelimit", default=None, type=click.Choice(["Day", "Week", "Month", "Year"]))
+@click.option("-size", "--size", default=None, type=click.Choice(["Small", "Medium", "Large", "Wallpaper"]))
+@click.option(
+    "-c",
+    "--color",
+    default=None,
+    type=click.Choice(
+        [
+            "color",
+            "Monochrome",
+            "Red",
+            "Orange",
+            "Yellow",
+            "Green",
+            "Blue",
+            "Purple",
+            "Pink",
+            "Brown",
+            "Black",
+            "Gray",
+            "Teal",
+            "White",
+        ]
+    ),
+)
+@click.option(
+    "-type", "--type_image", default=None, type=click.Choice(["photo", "clipart", "gif", "transparent", "line"])
+)
+@click.option("-l", "--layout", default=None, type=click.Choice(["Square", "Tall", "Wide"]))
+@click.option(
+    "-lic",
+    "--license_image",
+    default=None,
+    type=click.Choice(["any", "Public", "Share", "Modify", "ModifyCommercially"]),
+)
+@click.option("-m", "--max_results", default=90, help="maximum number of results, default=90")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-d", "--download", is_flag=True, default=False, help="download and save images to 'keywords' folder")
+@click.option("-th", "--threads", default=10, help="download threads, default=10")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def images(
+    keywords,
+    region,
+    safesearch,
+    timelimit,
+    size,
+    color,
+    type_image,
+    layout,
+    license_image,
+    download,
+    threads,
+    max_results,
+    output,
+    proxy,
+):
+    """CLI function to perform a images search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).images(
+        keywords=keywords,
+        region=region,
+        safesearch=safesearch,
+        timelimit=timelimit,
+        size=size,
+        color=color,
+        type_image=type_image,
+        layout=layout,
+        license_image=license_image,
+        max_results=max_results,
+    ):
+        data.append(r)
+    keywords = _sanitize_keywords(keywords)
+    filename = f"images_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print" and not download:
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+    if download:
+        _download_results(keywords, data, images=True, proxy=proxy, threads=threads)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="keywords for query")
+@click.option("-r", "--region", default="wt-wt", help="wt-wt, us-en, ru-ru, etc. -region https://duckduckgo.com/params")
+@click.option("-s", "--safesearch", default="moderate", type=click.Choice(["on", "moderate", "off"]))
+@click.option("-t", "--timelimit", default=None, type=click.Choice(["d", "w", "m"]), help="day, week, month")
+@click.option("-res", "--resolution", default=None, type=click.Choice(["high", "standart"]))
+@click.option("-d", "--duration", default=None, type=click.Choice(["short", "medium", "long"]))
+@click.option("-lic", "--license_videos", default=None, type=click.Choice(["creativeCommon", "youtube"]))
+@click.option("-m", "--max_results", default=50, help="maximum number of results, default=50")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def videos(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results, output, proxy):
+    """CLI function to perform a videos search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).videos(
+        keywords=keywords,
+        region=region,
+        safesearch=safesearch,
+        timelimit=timelimit,
+        resolution=resolution,
+        duration=duration,
+        license_videos=license_videos,
+        max_results=max_results,
+    ):
+        data.append(r)
+    filename = f"videos_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="keywords for query")
+@click.option("-r", "--region", default="wt-wt", help="wt-wt, us-en, ru-ru, etc. -region https://duckduckgo.com/params")
+@click.option("-s", "--safesearch", default="moderate", type=click.Choice(["on", "moderate", "off"]))
+@click.option("-t", "--timelimit", default=None, type=click.Choice(["d", "w", "m", "y"]), help="day, week, month, year")
+@click.option("-m", "--max_results", default=25, help="maximum number of results, default=25")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def news(keywords, region, safesearch, timelimit, max_results, output, proxy):
+    """CLI function to perform a news search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).news(
+        keywords=keywords, region=region, safesearch=safesearch, timelimit=timelimit, max_results=max_results
+    ):
+        data.append(r)
+    filename = f"news_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="keywords for query")
+@click.option("-p", "--place", default=None, help="simplified search - if set, the other parameters are not used")
+@click.option("-s", "--street", default=None, help="house number/street")
+@click.option("-c", "--city", default=None, help="city of search")
+@click.option("-county", "--county", default=None, help="county of search")
+@click.option("-state", "--state", default=None, help="state of search")
+@click.option("-country", "--country", default=None, help="country of search")
+@click.option("-post", "--postalcode", default=None, help="postalcode of search")
+@click.option("-lat", "--latitude", default=None, help="""if lat and long are set, the other params are not used""")
+@click.option("-lon", "--longitude", default=None, help="""if lat and long are set, the other params are not used""")
+@click.option("-r", "--radius", default=0, help="expand the search square by the distance in kilometers")
+@click.option("-m", "--max_results", default=50, help="number of results, default=50")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-proxy", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def maps(
+    keywords,
+    place,
+    street,
+    city,
+    county,
+    state,
+    country,
+    postalcode,
+    latitude,
+    longitude,
+    radius,
+    max_results,
+    output,
+    proxy,
+):
+    """CLI function to perform a maps search using DuckDuckGo API."""
+    data = []
+    for i, r in enumerate(
+        DDGS(proxies=proxy).maps(
+            keywords=keywords,
+            place=place,
+            street=street,
+            city=city,
+            county=county,
+            state=state,
+            country=country,
+            postalcode=postalcode,
+            latitude=latitude,
+            longitude=longitude,
+            radius=radius,
+            max_results=max_results,
+        ),
+        start=1,
+    ):
+        data.append(r)
+        if i % 100 == 0:
+            print(i)
+    filename = f"maps_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="text for translation")
+@click.option("-f", "--from_", help="What language to translate from (defaults automatically)")
+@click.option("-t", "--to", default="en", help="de, ru, fr, etc. What language to translate, defaults='en'")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def translate(keywords, from_, to, output, proxy):
+    """CLI function to perform translate using DuckDuckGo API."""
+    data = DDGS(proxies=proxy).translate(keywords=keywords, from_=from_, to=to)
+    data = [data]
+    filename = f"translate_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+@cli.command()
+@click.option("-k", "--keywords", required=True, help="keywords for query")
+@click.option("-r", "--region", default="wt-wt", help="wt-wt, us-en, ru-ru, etc. -region https://duckduckgo.com/params")
+@click.option("-o", "--output", default="print", help="csv, json (save the results to a csv or json file)")
+@click.option("-p", "--proxy", default=None, help="the proxy to send requests, example: socks5://localhost:9150")
+def suggestions(keywords, region, output, proxy):
+    """CLI function to perform a suggestions search using DuckDuckGo API."""
+    data = []
+    for r in DDGS(proxies=proxy).suggestions(keywords=keywords, region=region):
+        data.append(r)
+    filename = f"suggestions_{_sanitize_keywords(keywords)}_{datetime.now():%Y%m%d_%H%M%S}"
+    if output == "print":
+        _print_data(data)
+    elif output == "csv":
+        _save_csv(f"{filename}.csv", data)
+    elif output == "json":
+        _save_json(f"{filename}.json", data)
+
+
+if __name__ == "__main__":
+    cli(prog_name="ddgs")

webscout/exceptions.py

@@ -0,0 +1,2 @@
+class DuckDuckGoSearchException(Exception):
+    """Base exception class for webscout."""

webscout/models.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+
+@dataclass
+class MapsResult:
+    """Represents a result from the maps search."""
+
+    title: Optional[str] = None
+    address: Optional[str] = None
+    country_code: Optional[str] = None
+    latitude: Optional[str] = None
+    longitude: Optional[str] = None
+    url: Optional[str] = None
+    desc: Optional[str] = None
+    phone: Optional[str] = None
+    image: Optional[str] = None
+    source: Optional[str] = None
+    hours: Optional[Dict[str, str]] = None
+    category: Optional[str] = None
+    facebook: Optional[str] = None
+    instagram: Optional[str] = None
+    twitter: Optional[str] = None

webscout/utils.py

@@ -0,0 +1,48 @@
+import json
+import re
+from html import unescape
+from typing import Optional
+from urllib.parse import unquote
+
+from .exceptions import DuckDuckGoSearchException
+
+REGEX_500_IN_URL = re.compile(r"(?:\d{3}-\d{2}\.js)")
+REGEX_STRIP_TAGS = re.compile("<.*?>")
+REGEX_VQD = re.compile(rb"""vqd=['"]?([^&"']+)""")
+
+
+def _extract_vqd(html_bytes: bytes, keywords: str) -> Optional[str]:
+    """Extract vqd from html using a regular expression."""
+    try:
+        match = REGEX_VQD.search(html_bytes)
+        if match:
+            return match.group(1).decode()
+    except Exception:
+        pass
+    raise DuckDuckGoSearchException(f"_extract_vqd() {keywords=} Could not extract vqd.")
+
+
+def _text_extract_json(html_bytes: bytes, keywords: str) -> Optional[str]:
+    """text(backend="api") -> extract json from html."""
+    try:
+        start = html_bytes.index(b"DDG.pageLayout.load('d',") + 24
+        end = html_bytes.index(b");DDG.duckbar.load(", start)
+        data = html_bytes[start:end]
+        return json.loads(data)
+    except Exception as ex:
+        raise DuckDuckGoSearchException(f"_text_extract_json() {keywords=} {type(ex).__name__}: {ex}") from ex
+
+
+def _is_500_in_url(url: str) -> bool:
+    """Something like '506-00.js' inside the url."""
+    return bool(REGEX_500_IN_URL.search(url))
+
+
+def _normalize(raw_html: str) -> str:
+    """Strip HTML tags from the raw_html string."""
+    return unescape(REGEX_STRIP_TAGS.sub("", raw_html)) if raw_html else ""
+
+
+def _normalize_url(url: str) -> str:
+    """Unquote URL and replace spaces with '+'."""
+    return unquote(url.replace(" ", "+")) if url else ""

webscout/version.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"  # Replace with your actual version number

webscout/webscout_search.py

@@ -0,0 +1,65 @@
+import asyncio
+import logging
+import warnings
+from typing import Dict, Generator, Optional
+
+import nest_asyncio
+
+from .webscout_search_async import AsyncDDGS
+
+logger = logging.getLogger("duckduckgo_search.DDGS")
+nest_asyncio.apply()
+
+
+class DDGS(AsyncDDGS):
+    def __init__(self, headers=None, proxies=None, timeout=10):
+        if asyncio.get_event_loop().is_running():
+            warnings.warn("DDGS running in an async loop. This may cause errors. Use AsyncDDGS instead.", stacklevel=2)
+        super().__init__(headers, proxies, timeout)
+        self._loop = asyncio.get_event_loop()
+
+    def __enter__(self) -> "DDGS":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self._loop.create_task(self.__aexit__(exc_type, exc_val, exc_tb))
+
+    def _iter_over_async(self, async_gen):
+        """Iterate over an async generator."""
+        while True:
+            try:
+                yield self._loop.run_until_complete(async_gen.__anext__())
+            except StopAsyncIteration:
+                break
+
+    def text(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().text(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def images(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().images(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def videos(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().videos(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def news(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().news(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def answers(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().answers(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def suggestions(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().suggestions(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def maps(self, *args, **kwargs) -> Generator[Dict[str, Optional[str]], None, None]:
+        async_gen = super().maps(*args, **kwargs)
+        return self._iter_over_async(async_gen)
+
+    def translate(self, *args, **kwargs) -> Optional[Dict[str, Optional[str]]]:
+        async_coro = super().translate(*args, **kwargs)
+        return self._loop.run_until_complete(async_coro)