domaintools-api 2.5.3__tar.gz → 2.6.0__tar.gz

{domaintools_api-2.5.3/domaintools_api.egg-info → domaintools_api-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domaintools_api
-Version: 2.5.3
+Version: 2.6.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -354,6 +354,10 @@ To add more e2e tests, put these in the `../tests/e2e` folder.
         export TEST_USER=<user-key>
         export TEST_KEY=<api-key>
     ```
+- Run unit tests.
+    ```bash
+        tox -e
+    ```
 
 ## Run the end-to-end test script
 - Before running the test, be sure that docker is running.

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/README.md

@@ -304,6 +304,10 @@ To add more e2e tests, put these in the `../tests/e2e` folder.
         export TEST_USER=<user-key>
         export TEST_KEY=<api-key>
     ```
+- Run unit tests.
+    ```bash
+        tox -e
+    ```
 
 ## Run the end-to-end test script
 - Before running the test, be sure that docker is running.

domaintools_api-2.6.0/VERSION

@@ -0,0 +1 @@
+2.6.0

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/domaintools/_version.py

@@ -20,4 +20,4 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 """
 
-current = "2.5.3"
+current = "2.6.0"

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/domaintools/api.py

@@ -6,7 +6,13 @@ from typing import Union
 import re
 import ssl
 
-from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, RTTF_PRODUCTS_LIST, OutputFormat
+from domaintools.constants import (
+    Endpoint,
+    OutputFormat,
+    ENDPOINT_TO_SOURCE_MAP,
+    RTTF_PRODUCTS_LIST,
+    RTTF_PRODUCTS_CMD_MAPPING,
+)
 from domaintools._version import current as version
 from domaintools.results import (
     GroupedIterable,
@@ -92,7 +98,9 @@ class API(object):
         self._build_api_url(api_url, api_port)
 
         if not https:
-            raise Exception("The DomainTools API endpoints no longer support http traffic. Please make sure https=True.")
+            raise Exception(
+                "The DomainTools API endpoints no longer support http traffic. Please make sure https=True."
+            )
         if proxy_url and not isinstance(proxy_url, str):
             raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
 
@@ -110,8 +118,12 @@ class API(object):
 
         self._rest_api_url = rest_api_url
 
-    def _rate_limit(self):
+    def _rate_limit(self, product):
         """Pulls in and enforces the latest rate limits for the specified user"""
+        if product in RTTF_PRODUCTS_LIST:
+            self.limits_set = False
+            return
+
         self.limits_set = True
         for product in self.account_information():
             limit_minutes = product["per_minute_limit"] or None
@@ -128,8 +140,9 @@ class API(object):
         if product != "account-information" and self.rate_limit and not self.limits_set and not self.limits:
             always_sign_api_key_previous_value = self.always_sign_api_key
             header_authentication_previous_value = self.header_authentication
-            self._rate_limit()
-            # Reset always_sign_api_key and header_authentication to its original User-set values as these might be affected when self.account_information() was executed
+            self._rate_limit(product)
+            # Reset always_sign_api_key and header_authentication to its original
+            # User-set values as these might be affected when self.account_information() was executed
             self.always_sign_api_key = always_sign_api_key_previous_value
             self.header_authentication = header_authentication_previous_value
 
@@ -139,7 +152,13 @@ class API(object):
         is_rttf_product = product in RTTF_PRODUCTS_LIST
         self._handle_api_key_parameters(is_rttf_product)
         self.handle_api_key(is_rttf_product, path, parameters)
-        parameters.update({key: str(value).lower() if value in (True, False) else value for key, value in kwargs.items() if value is not None})
+        parameters.update(
+            {
+                key: str(value).lower() if value in (True, False) else value
+                for key, value in kwargs.items()
+                if value is not None
+            }
+        )
 
         return cls(self, product, uri, **parameters)
 
@@ -189,8 +208,30 @@ class API(object):
                 string[1:],
             )
 
-        api_calls = tuple((api_call for api_call in dir(API) if not api_call.startswith("_") and callable(getattr(API, api_call, None))))
-        return sorted([snakecase(p["id"]) for p in self.account_information()["products"] if snakecase(p["id"]) in api_calls])
+        api_calls = tuple(
+            (
+                api_call
+                for api_call in dir(API)
+                if not api_call.startswith("_") and callable(getattr(API, api_call, None))
+            )
+        )
+
+        account_information = self.account_information()
+
+        available_calls = set()
+        for product in self.account_information():
+            product_id = product["id"]
+            # for RTUF endpoints as we use different func name in our wrapper
+            if product_id in RTTF_PRODUCTS_LIST:
+                if rttf_api_command := RTTF_PRODUCTS_CMD_MAPPING.get(product_id):
+                    available_calls.add(rttf_api_command)
+
+            # for IRIS endpoints
+            snakecase_pid = snakecase(product_id)
+            if snakecase_pid in api_calls:
+                available_calls.add(snakecase_pid)
+
+        return sorted(available_calls)
 
     def brand_monitor(self, query, exclude=None, domain_status=None, days_back=None, **kwargs):
         """Pass in one or more terms as a list or separated by the pipe character ( | )"""
@@ -445,7 +486,16 @@ class API(object):
         """Performs a search for the provided search terms ANDed together,
         returning the pivot engine row data for the resulting domains.
         """
-        if not domain and not ip and not email and not nameserver and not registrar and not registrant and not registrant_org and not kwargs:
+        if (
+            not domain
+            and not ip
+            and not email
+            and not nameserver
+            and not registrar
+            and not registrant
+            and not registrant_org
+            and not kwargs
+        ):
             raise ValueError("At least one search term must be specified")
 
         return self._results(
@@ -1069,7 +1119,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1101,7 +1154,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1162,7 +1218,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1194,7 +1253,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1225,12 +1287,15 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
         return self._results(
-            f"domain-risk-feed-({source})",
+            f"real-time-domain-risk-({source})",
             f"v1/{endpoint}/domainrisk/",
             response_path=(),
             cls=FeedsResults,
@@ -1256,12 +1321,15 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
         return self._results(
-            f"domain-hotlist-feed-({source})",
+            f"real-time-domain-hotlist-({source})",
             f"v1/{endpoint}/domainhotlist/",
             response_path=(),
             cls=FeedsResults,

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/domaintools/base_results.py

@@ -9,7 +9,11 @@ from copy import deepcopy
 from datetime import datetime
 from httpx import Client
 
-from domaintools.constants import RTTF_PRODUCTS_LIST, OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
+from domaintools.constants import (
+    RTTF_PRODUCTS_LIST,
+    OutputFormat,
+    HEADER_ACCEPT_KEY_CSV_FORMAT,
+)
 from domaintools.exceptions import (
     BadRequestException,
     InternalServerErrorException,
@@ -53,6 +57,7 @@ class Results(MutableMapping, MutableSequence):
         self._response = None
         self._items_list = None
         self._data = None
+        self._status = None
 
     def _wait_time(self):
         if not self.api.rate_limit or not self.product in self.api.limits:
@@ -92,10 +97,10 @@ class Results(MutableMapping, MutableSequence):
             header_key_for_api_key = "X-Api-Key" if is_rttf_product else "X-API-Key"
             headers[header_key_for_api_key] = self.api.key
 
-        return {"parameters": parameters, "headers": headers}
+        session_param_and_headers = {"parameters": parameters, "headers": headers}
+        return session_param_and_headers
 
     def _make_request(self):
-
         with Client(verify=self.api.verify_ssl, proxy=self.api.proxy_url, timeout=None) as session:
             session_params_and_headers = self._get_session_params_and_headers()
             headers = session_params_and_headers.get("headers")
@@ -113,7 +118,12 @@ class Results(MutableMapping, MutableSequence):
                 return session.patch(url=self.url, json=patch_data, headers=headers)
             else:
                 parameters = session_params_and_headers.get("parameters")
-                return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params)
+                return session.get(
+                    url=self.url,
+                    params=parameters,
+                    headers=headers,
+                    **self.api.extra_request_params,
+                )
 
     def _get_results(self):
         wait_for = self._wait_time()
@@ -152,7 +162,9 @@ class Results(MutableMapping, MutableSequence):
     def check_limit_exceeded(self):
         limit_exceeded, reason = False, ""
         if isinstance(self._data, dict) and (
-            "response" in self._data and "limit_exceeded" in self._data["response"] and self._data["response"]["limit_exceeded"] is True
+            "response" in self._data
+            and "limit_exceeded" in self._data["response"]
+            and self._data["response"]["limit_exceeded"] is True
         ):
             limit_exceeded, reason = True, self._data["response"]["message"]
         elif "response" in self._data and "limit_exceeded" in self._data:
@@ -163,12 +175,12 @@ class Results(MutableMapping, MutableSequence):
 
     @property
     def status(self):
-        if not getattr(self, "_status", None):
+        if not getattr(self, "_status", None) and not self.product in RTTF_PRODUCTS_LIST:
             self._status = self._get_results().status_code
 
         return self._status
 
-    def setStatus(self, code, response=None):
+    def setStatus(self, code, response=None, reason_text=None):
         self._status = code
         if code == 200 or (self.product in RTTF_PRODUCTS_LIST and code == 206):
             return
@@ -181,6 +193,9 @@ class Results(MutableMapping, MutableSequence):
                 reason = response.text
                 if callable(reason):
                     reason = reason()
+        else:  # optionally pass a customize reason of error for better traceback
+            if reason_text is not None:
+                reason = reason_text
 
         if code in (400, 422):
             raise BadRequestException(code, reason)
@@ -330,4 +345,8 @@ class Results(MutableMapping, MutableSequence):
         return "\n".join([json.dumps(item, indent=4, separators=(",", ": ")) for item in self._items()])
 
     def __str__(self):
-        return str(json.dumps(self.data(), indent=4, separators=(",", ": ")) if self.kwargs.get("format", "json") == "json" else self.data())
+        return str(
+            json.dumps(self.data(), indent=4, separators=(",", ": "))
+            if self.kwargs.get("format", "json") == "json"
+            else self.data()
+        )

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/domaintools/cli/api.py

@@ -58,7 +58,9 @@ class DTCLICommand:
             datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
             return value
         except:
-            raise typer.BadParameter(f"{value} is neither an integer or a valid ISO 8601 datetime string in UTC form")
+            raise typer.BadParameter(
+                f"{value} is neither an integer or a valid ISO 8601 datetime string in UTC form"
+            )
 
     @staticmethod
     def validate_source_file_extension(value: str):
@@ -78,7 +80,9 @@ class DTCLICommand:
         ext = get_file_extension(value)
 
         if ext.lower() not in VALID_EXTENSIONS:
-            raise typer.BadParameter(f"{value} is not in valid extensions. Valid file extensions: {VALID_EXTENSIONS}")
+            raise typer.BadParameter(
+                f"{value} is not in valid extensions. Valid file extensions: {VALID_EXTENSIONS}"
+            )
 
         return value
 
@@ -111,7 +115,7 @@ class DTCLICommand:
         if cmd_name in ("available_api_calls",):
             return "\n".join(response)
         if response.product in RTTF_PRODUCTS_LIST:
-            return "\n".join([data for data in response.response()])
+            pass  # do nothing
         return str(getattr(response, out_format) if out_format != "list" else response.as_list())
 
     @classmethod
@@ -203,7 +207,7 @@ class DTCLICommand:
                 transient=True,
             ) as progress:
 
-                progress.add_task(
+                task_id = progress.add_task(
                     description=f"Using api credentials with a username of: [cyan]{user}[/cyan]\nExecuting [green]{name}[/green] api call...",
                     total=None,
                 )
@@ -218,27 +222,36 @@ class DTCLICommand:
                     header_authentication=header_authentication,
                 )
                 dt_api_func = getattr(dt_api, name)
-
                 params = params | kwargs
 
                 response = dt_api_func(**params)
-                progress.add_task(
+                progress.update(
+                    task_id,
                     description=f"Preparing results with format of {response_format}...",
-                    total=None,
                 )
 
-                output = cls._get_formatted_output(cmd_name=name, response=response, out_format=response_format)
+                output = cls._get_formatted_output(
+                    cmd_name=name, response=response, out_format=response_format
+                )
 
                 if isinstance(out_file, _io.TextIOWrapper):
+                    progress.update(
+                        task_id,
+                        description=f"Printing the results with format of {response_format}...",
+                    )
                     # use rich `print` command to prettify the ouput in sys.stdout
-                    if response.product in RTTF_PRODUCTS_LIST:
-                        print(output)
+                    if name not in ("available_api_calls",) and response.product in RTTF_PRODUCTS_LIST:
+                        for feeds in response.response():
+                            print(feeds)
                     else:
                         print(response)
                 else:
+                    progress.update(
+                        task_id,
+                        description=f"Writing results to {out_file}",
+                    )
                     # if it's a file then write
                     out_file.write(output if output.endswith("\n") else output + "\n")
-                time.sleep(0.25)
         except Exception as e:
             if isinstance(e, ServiceException):
                 code = typer.style(getattr(e, "code", 400), fg=typer.colors.BRIGHT_RED)

domaintools_api-2.6.0/domaintools/constants.py

@@ -0,0 +1,58 @@
+from enum import Enum
+
+
+class Endpoint(Enum):
+    FEED = "feed"
+    DOWNLOAD = "download"
+
+
+class Source(Enum):
+    API = "api"
+    S3 = "s3"
+
+
+class OutputFormat(Enum):
+    JSONL = "jsonl"
+    CSV = "csv"
+
+
+HEADER_ACCEPT_KEY_CSV_FORMAT = "text/csv"
+
+ENDPOINT_TO_SOURCE_MAP = {
+    Endpoint.FEED.value: Source.API,
+    Endpoint.DOWNLOAD.value: Source.S3,
+}
+
+RTTF_PRODUCTS_LIST = [
+    "newly-active-domains-feed-(api)",
+    "newly-active-domains-feed-(s3)",
+    "newly-observed-domains-feed-(api)",
+    "newly-observed-domains-feed-(s3)",
+    "newly-observed-hosts-feed-(api)",
+    "newly-observed-hosts-feed-(s3)",
+    "real-time-domain-hotlist-(api)",
+    "real-time-domain-hotlist-(s3)",
+    "domain-registration-data-access-protocol-feed-(api)",
+    "domain-registration-data-access-protocol-feed-(s3)",
+    "real-time-domain-risk-(api)",
+    "real-time-domain-risk-(s3)",
+    "real-time-domain-discovery-feed-(api)",
+    "real-time-domain-discovery-feed-(s3)",
+]
+
+RTTF_PRODUCTS_CMD_MAPPING = {
+    "newly-active-domains-feed-(api)": "nad",
+    "newly-active-domains-feed-(s3)": "nad",
+    "newly-observed-domains-feed-(api)": "nod",
+    "newly-observed-domains-feed-(s3)": "nod",
+    "newly-observed-hosts-feed-(api)": "noh",
+    "newly-observed-hosts-feed-(s3)": "noh",
+    "real-time-domain-hotlist-(api)": "domainhotlist",
+    "real-time-domain-hotlist-(s3)": "domainhotlist",
+    "domain-registration-data-access-protocol-feed-(api)": "domainrdap",
+    "domain-registration-data-access-protocol-feed-(s3)": "domainrdap",
+    "real-time-domain-risk-(api)": "realtime_domain_risk",
+    "real-time-domain-risk-(s3)": "realtime_domain_risk",
+    "real-time-domain-discovery-feed-(api)": "domaindiscovery",
+    "real-time-domain-discovery-feed-(s3)": "domaindiscovery",
+}

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/domaintools/results.py

@@ -1,20 +1,24 @@
-"""Defines the used Result object based on the current versions and/or features available to Python runtime
-
+"""
+Defines the used Result object based on the current versions and/or features available to Python runtime
 Additionally, defines any custom result objects that may be used to enable more Pythonic interaction with endpoints.
 """
 
-from itertools import chain
+import logging
+from itertools import zip_longest, chain
+from typing import Generator
+
+import httpx
 
 try:  # pragma: no cover
     from collections import OrderedDict
 except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict
 
-from itertools import zip_longest
-from typing import Generator
 
 from domaintools_async import AsyncResults as Results
 
+log = logging.getLogger(__name__)
+
 
 class Reputation(Results):
     """Returns the reputation results in a format that can quickly be converted into floats / ints"""
@@ -32,7 +36,11 @@ class GroupedIterable(Results):
     def _items(self):
         if self._items_list is None:
             self._items_list = chain(
-                *[zip_longest([], value, fillvalue=key) for key, value in self.response().items() if type(value) in (list, tuple)]
+                *[
+                    zip_longest([], value, fillvalue=key)
+                    for key, value in self.response().items()
+                    if type(value) in (list, tuple)
+                ]
             )
 
         return self._items_list
@@ -45,13 +53,27 @@ class ParsedWhois(Results):
         """Returns a flattened version of the parsed whois data"""
         parsed = self["parsed_whois"]
         flat = OrderedDict()
-        for key in ("domain", "created_date", "updated_date", "expired_date", "statuses", "name_servers"):
+        for key in (
+            "domain",
+            "created_date",
+            "updated_date",
+            "expired_date",
+            "statuses",
+            "name_servers",
+        ):
             if key in parsed:
                 value = parsed[key]
                 flat[key] = " | ".join(value) if type(value) in (list, tuple) else value
 
         registrar = parsed.get("registrar", {})
-        for key in ("name", "abuse_contact_phone", "abuse_contact_email", "iana_id", "url", "whois_server"):
+        for key in (
+            "name",
+            "abuse_contact_phone",
+            "abuse_contact_email",
+            "iana_id",
+            "url",
+            "whois_server",
+        ):
             if key in registrar:
                 flat["registrar_{0}".format(key)] = registrar[key]
 
@@ -59,10 +81,24 @@ class ParsedWhois(Results):
             networks = parsed.get("networks")
             for network in networks:
                 id = network.get("id")
-                for key in ("range", "asn", "org", "parent", "customer", "country", "phone", "status", "source", "updated_date", "created_date"):
+                for key in (
+                    "range",
+                    "asn",
+                    "org",
+                    "parent",
+                    "customer",
+                    "country",
+                    "phone",
+                    "status",
+                    "source",
+                    "updated_date",
+                    "created_date",
+                ):
                     if key in network:
                         value = network[key]
-                        flat["network_{0}".format(id)] = " ".join(value) if type(value) in (list, tuple) else value
+                        flat["network_{0}".format(id)] = (
+                            " ".join(value) if type(value) in (list, tuple) else value
+                        )
 
         if "contacts" in parsed:
             contacts = parsed.get("contacts")
@@ -86,15 +122,30 @@ class ParsedWhois(Results):
                     ):
                         if key in contact:
                             value = contact[key]
-                            flat["{0}_{1}".format(contact_type, key)] = " ".join(value) if type(value) in (list, tuple) else value
+                            flat["{0}_{1}".format(contact_type, key)] = (
+                                " ".join(value) if type(value) in (list, tuple) else value
+                            )
 
             elif type(contacts) is dict:
                 for contact_type in ("registrant", "admin", "tech", "billing"):
                     contact = contacts.get(contact_type, {})
-                    for key in ("name", "email", "org", "street", "city", "state", "postal", "country", "phone", "fax"):
+                    for key in (
+                        "name",
+                        "email",
+                        "org",
+                        "street",
+                        "city",
+                        "state",
+                        "postal",
+                        "country",
+                        "phone",
+                        "fax",
+                    ):
                         if key in contact:
                             value = contact[key]
-                            flat["{0}_{1}".format(contact_type, key)] = " ".join(value) if type(value) in (list, tuple) else value
+                            flat["{0}_{1}".format(contact_type, key)] = (
+                                " ".join(value) if type(value) in (list, tuple) else value
+                            )
 
         return flat
 
@@ -130,7 +181,9 @@ class ParsedDomainRdap(Results):
                 for i, contact in enumerate(registrar_value, start=1):
                     for contact_key, contact_value in contact.items():
                         flat[f"registrar_contacts_{contact_key}"] = (
-                            " | ".join(contact_value) if type(contact_value) in (list, tuple) else contact_value
+                            " | ".join(contact_value)
+                            if type(contact_value) in (list, tuple)
+                            else contact_value
                         )
 
                 continue
@@ -140,31 +193,69 @@ class ParsedDomainRdap(Results):
         if contacts:
             for i, contact in enumerate(contacts, start=1):
                 for contact_key, contact_value in contact.items():
-                    flat[f"contact_{contact_key}_{i}"] = " | ".join(contact_value) if type(contact_value) in (list, tuple) else contact_value
+                    flat[f"contact_{contact_key}_{i}"] = (
+                        " | ".join(contact_value) if type(contact_value) in (list, tuple) else contact_value
+                    )
 
         return flat
 
 
 class FeedsResults(Results):
-    """Returns the generator for feeds results"""
+    """
+    Real Time Threat Feeds (RTTF) returns an application/ndjson stream.
+    With this we use httpx stream to process each JSON object efficiently.
+
+    Highlevel process:
+
+    httpx stream -> check status code -> yield back data to client -> repeat if 206
+
+    Returns the generator object for feeds results.
+    """
+
+    def _make_request(self) -> Generator:
+        """
+        Creates and manages the httpx stream request, yielding data line by line.
+        This is the core generator that communicates with the DT frontend API server.
+        """
+        session_info = self._get_session_params_and_headers()
+        headers = session_info.get("headers")
+        parameters = session_info.get("parameters")
+
+        with httpx.stream(
+            "GET",
+            self.url,
+            headers=headers,
+            params=parameters,
+            verify=self.api.verify_ssl,
+            proxy=self.api.proxy_url,
+            timeout=None,
+        ) as response:
+            # set the status already
+            error_text = ""
+            status_code = response.status_code
+            if status_code not in [200, 206]:
+                response.read()
+                error_text = response.text
+
+            self.setStatus(status_code, reason_text=error_text)
+
+            for line in response.iter_lines():
+                yield line
+
+    def data(self) -> Generator:
+        self._data = self._make_request()
+        return self._data
 
     def response(self) -> Generator:
-        status_code = None
-        while status_code != 200:
-            resp_data = self.data()
-            status_code = self.status
-            yield resp_data
+        while self.status != 200:
+            yield from self.data()
 
-            self._data = None  # clear the data here
             if not self.kwargs.get("sessionID"):
                 # we'll only do iterative request for queries that has sessionID.
-                # Otherwise, we will have an infinite request if sessionID was not provided but the required data asked is more than the maximum (1 hour of data)
+                # Otherwise, we will have an infinite request if sessionID was not provided
+                # but the required data asked is more than the maximum (1 hour of data)
                 break
+        self._status = None
 
-    def data(self):
-        results = self._get_results()
-        self.setStatus(results.status_code, results)
-        self._data = results.text
-        self.check_limit_exceeded()
-
-        return self._data
+    def __str__(self):
+        return f"{self.__class__.__name__} - {self.product}"

{domaintools_api-2.5.3 → domaintools_api-2.6.0/domaintools_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domaintools_api
-Version: 2.5.3
+Version: 2.6.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -354,6 +354,10 @@ To add more e2e tests, put these in the `../tests/e2e` folder.
         export TEST_USER=<user-key>
         export TEST_KEY=<api-key>
     ```
+- Run unit tests.
+    ```bash
+        tox -e
+    ```
 
 ## Run the end-to-end test script
 - Before running the test, be sure that docker is running.

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/pyproject.toml

@@ -49,3 +49,6 @@ packages = [
 
 [tool.setuptools.dynamic]
 version = { "file" = "VERSION" }
+
+[tool.black]
+line-length = 110

{domaintools_api-2.5.3 → domaintools_api-2.6.0}/tests/test_api.py

@@ -473,7 +473,9 @@ def test_iris_detect_watched_domains():
     detect_results = api.iris_detect_watched_domains()
     assert detect_results["count"] >= 0
 
-    detect_results = api.iris_detect_watched_domains(monitor_id="nAwmQg2pqg", sort=["risk_score"], order="desc")
+    detect_results = api.iris_detect_watched_domains(
+        monitor_id="nAwmQg2pqg", sort=["risk_score"], order="desc"
+    )
     assert len(detect_results["watchlist_domains"]) == 3
 
     detect_results = api.iris_detect_watched_domains(escalation_types="blocked")
@@ -482,17 +484,23 @@ def test_iris_detect_watched_domains():
 
 @vcr.use_cassette
 def test_iris_detect_manage_watchlist_domains():
-    detect_results = api.iris_detect_manage_watchlist_domains(watchlist_domain_ids=["gae08rdVWG"], state="watched")
+    detect_results = api.iris_detect_manage_watchlist_domains(
+        watchlist_domain_ids=["gae08rdVWG"], state="watched"
+    )
     assert detect_results["watchlist_domains"][0]["state"] == "watched"
 
 
 @vcr.use_cassette
 def test_iris_detect_escalate_domains():
     # If you rerun this test without VCR, it will fail because the domain is already escalated
-    detect_results = api.iris_detect_escalate_domains(watchlist_domain_ids=["OWxzqKqQEY"], escalation_type="blocked")
+    detect_results = api.iris_detect_escalate_domains(
+        watchlist_domain_ids=["OWxzqKqQEY"], escalation_type="blocked"
+    )
     assert detect_results["escalations"][0]["escalation_type"] == "blocked"
 
-    detect_results = api.iris_detect_escalate_domains(watchlist_domain_ids=["OWxzqKqQEY"], escalation_type="google_safe")
+    detect_results = api.iris_detect_escalate_domains(
+        watchlist_domain_ids=["OWxzqKqQEY"], escalation_type="google_safe"
+    )
     assert detect_results["escalations"][0]["escalation_type"] == "google_safe"
 
 
@@ -515,34 +523,40 @@ def test_limit_exceeded():
 @vcr.use_cassette
 def test_newly_observed_domains_feed():
     results = feeds_api.nod(after="-60", top=5)
+    total_count = 0
     for response in results.response():
         assert results.status == 200
 
         rows = response.strip().split("\n")
         assert response is not None
-        assert len(rows) >= 1
+        total_count += 1
 
         for row in rows:
             feed_result = json.loads(row)
             assert "timestamp" in feed_result.keys()
             assert "domain" in feed_result.keys()
 
+    assert total_count == 5
+
 
 @vcr.use_cassette
 def test_newly_observed_hosts_feed():
     results = feeds_api.noh(after="-60", top=5)
+    total_count = 0
     for response in results.response():
         assert results.status == 200
 
         rows = response.strip().split("\n")
         assert response is not None
-        assert len(rows) >= 1
+        total_count += 1
 
         for row in rows:
             feed_result = json.loads(row)
             assert "timestamp" in feed_result.keys()
             assert "domain" in feed_result.keys()
 
+    assert total_count == 5
+
 
 @vcr.use_cassette
 @pytest.mark.skip(reason="Failing test due to unknown reason. Still investigating the root-cause.")
@@ -553,16 +567,14 @@ def test_newly_observed_domains_feed_pagination():
     results = feeds_api.nod(sessionID="integrations-testing", after=after)
     page_count = 0
     for response in results.response():
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
         page_count += 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
 
     assert page_count >= 1
 
@@ -573,14 +585,12 @@ def test_newly_active_domains_feed():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
 
 
 @vcr.use_cassette
@@ -589,19 +599,17 @@ def test_domainrdap_feed():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
 
         assert response is not None
-        assert len(rows) == 2
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
-            assert "parsed_record" in feed_result.keys()
-            assert "domain" in feed_result["parsed_record"]["parsed_fields"]
-            assert "emails" in feed_result["parsed_record"]["parsed_fields"]
-            assert "contacts" in feed_result["parsed_record"]["parsed_fields"]
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
+        assert "parsed_record" in feed_result.keys()
+        assert "domain" in feed_result["parsed_record"]["parsed_fields"]
+        assert "emails" in feed_result["parsed_record"]["parsed_fields"]
+        assert "contacts" in feed_result["parsed_record"]["parsed_fields"]
 
 
 @vcr.use_cassette
@@ -610,14 +618,12 @@ def test_domain_discovery_feed():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
 
 
 @vcr.use_cassette
@@ -626,19 +632,17 @@ def test_domainrdap_feed_not_api_header_auth():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
 
         assert response is not None
-        assert len(rows) == 5
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
-            assert "parsed_record" in feed_result.keys()
-            assert "domain" in feed_result["parsed_record"]["parsed_fields"]
-            assert "emails" in feed_result["parsed_record"]["parsed_fields"]
-            assert "contacts" in feed_result["parsed_record"]["parsed_fields"]
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
+        assert "parsed_record" in feed_result.keys()
+        assert "domain" in feed_result["parsed_record"]["parsed_fields"]
+        assert "emails" in feed_result["parsed_record"]["parsed_fields"]
+        assert "contacts" in feed_result["parsed_record"]["parsed_fields"]
 
 
 @vcr.use_cassette
@@ -654,14 +658,12 @@ def test_feeds_endpoint_should_non_header_auth_be_the_default():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "download_name" in feed_result["response"].keys()
-            assert "files" in feed_result["response"].keys()
+        feed_result = json.loads(response)
+        assert "download_name" in feed_result["response"].keys()
+        assert "files" in feed_result["response"].keys()
 
 
 @vcr.use_cassette
@@ -686,18 +688,16 @@ def test_realtime_domain_risk():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
-            assert "phishing_risk" in feed_result.keys()
-            assert "malware_risk" in feed_result.keys()
-            assert "proximity_risk" in feed_result.keys()
-            assert "overall_risk" in feed_result.keys()
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
+        assert "phishing_risk" in feed_result.keys()
+        assert "malware_risk" in feed_result.keys()
+        assert "proximity_risk" in feed_result.keys()
+        assert "overall_risk" in feed_result.keys()
 
 
 @vcr.use_cassette
@@ -706,18 +706,16 @@ def test_domain_hotlist():
     for response in results.response():
         assert results.status == 200
 
-        rows = response.strip().split("\n")
+        response = response.strip()
         assert response is not None
-        assert len(rows) >= 1
 
-        for row in rows:
-            feed_result = json.loads(row)
-            assert "timestamp" in feed_result.keys()
-            assert "domain" in feed_result.keys()
-            assert "phishing_risk" in feed_result.keys()
-            assert "malware_risk" in feed_result.keys()
-            assert "proximity_risk" in feed_result.keys()
-            assert "overall_risk" in feed_result.keys()
+        feed_result = json.loads(response)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
+        assert "phishing_risk" in feed_result.keys()
+        assert "malware_risk" in feed_result.keys()
+        assert "proximity_risk" in feed_result.keys()
+        assert "overall_risk" in feed_result.keys()
 
 
 @vcr.use_cassette

domaintools_api-2.5.3/VERSION

@@ -1 +0,0 @@
-2.5.3

domaintools_api-2.5.3/domaintools/constants.py

@@ -1,41 +0,0 @@
-from enum import Enum
-
-
-class Endpoint(Enum):
-    FEED = "feed"
-    DOWNLOAD = "download"
-
-
-class Source(Enum):
-    API = "api"
-    S3 = "s3"
-
-
-class OutputFormat(Enum):
-    JSONL = "jsonl"
-    CSV = "csv"
-
-
-HEADER_ACCEPT_KEY_CSV_FORMAT = "text/csv"
-
-ENDPOINT_TO_SOURCE_MAP = {
-    Endpoint.FEED.value: Source.API,
-    Endpoint.DOWNLOAD.value: Source.S3,
-}
-
-RTTF_PRODUCTS_LIST = [
-    "newly-active-domains-feed-(api)",
-    "newly-active-domains-feed-(s3)",
-    "newly-observed-domains-feed-(api)",
-    "newly-observed-domains-feed-(s3)",
-    "newly-observed-hosts-feed-(api)",
-    "newly-observed-hosts-feed-(s3)",
-    "domain-hotlist-feed-(api)",
-    "domain-hotlist-feed-(s3)",
-    "domain-registration-data-access-protocol-feed-(api)",
-    "domain-registration-data-access-protocol-feed-(s3)",
-    "domain-risk-feed-(api)",
-    "domain-risk-feed-(s3)",
-    "real-time-domain-discovery-feed-(api)",
-    "real-time-domain-discovery-feed-(s3)",
-]