domaintools-api-test-upload 2.5.5__tar.gz → 2.6.1__tar.gz

{domaintools_api_test_upload-2.5.5/domaintools_api_test_upload.egg-info → domaintools_api_test_upload-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
-Name: domaintools_api_test_upload
-Version: 2.5.5
+Name: domaintools-api-test-upload
+Version: 2.6.1
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)

domaintools_api_test_upload-2.6.1/VERSION

@@ -0,0 +1 @@
+2.6.1

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/domaintools/_version.py

@@ -20,4 +20,4 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 """
 
-current = "2.5.5"
+current = "2.6.1"

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/domaintools/api.py

@@ -92,7 +92,9 @@ class API(object):
         self._build_api_url(api_url, api_port)
 
         if not https:
-            raise Exception("The DomainTools API endpoints no longer support http traffic. Please make sure https=True.")
+            raise Exception(
+                "The DomainTools API endpoints no longer support http traffic. Please make sure https=True."
+            )
         if proxy_url and not isinstance(proxy_url, str):
             raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
 
@@ -129,7 +131,8 @@ class API(object):
             always_sign_api_key_previous_value = self.always_sign_api_key
             header_authentication_previous_value = self.header_authentication
             self._rate_limit()
-            # Reset always_sign_api_key and header_authentication to its original User-set values as these might be affected when self.account_information() was executed
+            # Reset always_sign_api_key and header_authentication to its original
+            # User-set values as these might be affected when self.account_information() was executed
             self.always_sign_api_key = always_sign_api_key_previous_value
             self.header_authentication = header_authentication_previous_value
 
@@ -139,7 +142,13 @@ class API(object):
         is_rttf_product = product in RTTF_PRODUCTS_LIST
         self._handle_api_key_parameters(is_rttf_product)
         self.handle_api_key(is_rttf_product, path, parameters)
-        parameters.update({key: str(value).lower() if value in (True, False) else value for key, value in kwargs.items() if value is not None})
+        parameters.update(
+            {
+                key: str(value).lower() if value in (True, False) else value
+                for key, value in kwargs.items()
+                if value is not None
+            }
+        )
 
         return cls(self, product, uri, **parameters)
 
@@ -189,8 +198,20 @@ class API(object):
                 string[1:],
             )
 
-        api_calls = tuple((api_call for api_call in dir(API) if not api_call.startswith("_") and callable(getattr(API, api_call, None))))
-        return sorted([snakecase(p["id"]) for p in self.account_information()["products"] if snakecase(p["id"]) in api_calls])
+        api_calls = tuple(
+            (
+                api_call
+                for api_call in dir(API)
+                if not api_call.startswith("_") and callable(getattr(API, api_call, None))
+            )
+        )
+        return sorted(
+            [
+                snakecase(p["id"])
+                for p in self.account_information()["products"]
+                if snakecase(p["id"]) in api_calls
+            ]
+        )
 
     def brand_monitor(self, query, exclude=None, domain_status=None, days_back=None, **kwargs):
         """Pass in one or more terms as a list or separated by the pipe character ( | )"""
@@ -445,7 +466,16 @@ class API(object):
         """Performs a search for the provided search terms ANDed together,
         returning the pivot engine row data for the resulting domains.
         """
-        if not domain and not ip and not email and not nameserver and not registrar and not registrant and not registrant_org and not kwargs:
+        if (
+            not domain
+            and not ip
+            and not email
+            and not nameserver
+            and not registrar
+            and not registrant
+            and not registrant_org
+            and not kwargs
+        ):
             raise ValueError("At least one search term must be specified")
 
         return self._results(
@@ -1069,7 +1099,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1101,7 +1134,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1162,7 +1198,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1194,7 +1233,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1225,7 +1267,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 
@@ -1256,7 +1301,10 @@ class API(object):
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
-        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
             # headers param is allowed only in Feed API and CSV format
             kwargs.pop("headers", None)
 

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/domaintools/base_results.py

@@ -9,7 +9,11 @@ from copy import deepcopy
 from datetime import datetime
 from httpx import Client
 
-from domaintools.constants import RTTF_PRODUCTS_LIST, OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
+from domaintools.constants import (
+    RTTF_PRODUCTS_LIST,
+    OutputFormat,
+    HEADER_ACCEPT_KEY_CSV_FORMAT,
+)
 from domaintools.exceptions import (
     BadRequestException,
     InternalServerErrorException,
@@ -53,6 +57,7 @@ class Results(MutableMapping, MutableSequence):
         self._response = None
         self._items_list = None
         self._data = None
+        self._status = None
 
     def _wait_time(self):
         if not self.api.rate_limit or not self.product in self.api.limits:
@@ -92,10 +97,10 @@ class Results(MutableMapping, MutableSequence):
             header_key_for_api_key = "X-Api-Key" if is_rttf_product else "X-API-Key"
             headers[header_key_for_api_key] = self.api.key
 
-        return {"parameters": parameters, "headers": headers}
+        session_param_and_headers = {"parameters": parameters, "headers": headers}
+        return session_param_and_headers
 
     def _make_request(self):
-
         with Client(verify=self.api.verify_ssl, proxy=self.api.proxy_url, timeout=None) as session:
             session_params_and_headers = self._get_session_params_and_headers()
             headers = session_params_and_headers.get("headers")
@@ -113,7 +118,12 @@ class Results(MutableMapping, MutableSequence):
                 return session.patch(url=self.url, json=patch_data, headers=headers)
             else:
                 parameters = session_params_and_headers.get("parameters")
-                return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params)
+                return session.get(
+                    url=self.url,
+                    params=parameters,
+                    headers=headers,
+                    **self.api.extra_request_params,
+                )
 
     def _get_results(self):
         wait_for = self._wait_time()
@@ -152,7 +162,9 @@ class Results(MutableMapping, MutableSequence):
     def check_limit_exceeded(self):
         limit_exceeded, reason = False, ""
         if isinstance(self._data, dict) and (
-            "response" in self._data and "limit_exceeded" in self._data["response"] and self._data["response"]["limit_exceeded"] is True
+            "response" in self._data
+            and "limit_exceeded" in self._data["response"]
+            and self._data["response"]["limit_exceeded"] is True
         ):
             limit_exceeded, reason = True, self._data["response"]["message"]
         elif "response" in self._data and "limit_exceeded" in self._data:
@@ -163,12 +175,12 @@ class Results(MutableMapping, MutableSequence):
 
     @property
     def status(self):
-        if not getattr(self, "_status", None):
+        if not getattr(self, "_status", None) and not self.product in RTTF_PRODUCTS_LIST:
             self._status = self._get_results().status_code
 
         return self._status
 
-    def setStatus(self, code, response=None):
+    def setStatus(self, code, response=None, reason_text=None):
         self._status = code
         if code == 200 or (self.product in RTTF_PRODUCTS_LIST and code == 206):
             return
@@ -181,6 +193,9 @@ class Results(MutableMapping, MutableSequence):
                 reason = response.text
                 if callable(reason):
                     reason = reason()
+        else:  # optionally pass a customize reason of error for better traceback
+            if reason_text is not None:
+                reason = reason_text
 
         if code in (400, 422):
             raise BadRequestException(code, reason)
@@ -330,4 +345,8 @@ class Results(MutableMapping, MutableSequence):
         return "\n".join([json.dumps(item, indent=4, separators=(",", ": ")) for item in self._items()])
 
     def __str__(self):
-        return str(json.dumps(self.data(), indent=4, separators=(",", ": ")) if self.kwargs.get("format", "json") == "json" else self.data())
+        return str(
+            json.dumps(self.data(), indent=4, separators=(",", ": "))
+            if self.kwargs.get("format", "json") == "json"
+            else self.data()
+        )

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/domaintools/cli/api.py

@@ -58,7 +58,9 @@ class DTCLICommand:
             datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
             return value
         except:
-            raise typer.BadParameter(f"{value} is neither an integer or a valid ISO 8601 datetime string in UTC form")
+            raise typer.BadParameter(
+                f"{value} is neither an integer or a valid ISO 8601 datetime string in UTC form"
+            )
 
     @staticmethod
     def validate_source_file_extension(value: str):
@@ -78,7 +80,9 @@ class DTCLICommand:
         ext = get_file_extension(value)
 
         if ext.lower() not in VALID_EXTENSIONS:
-            raise typer.BadParameter(f"{value} is not in valid extensions. Valid file extensions: {VALID_EXTENSIONS}")
+            raise typer.BadParameter(
+                f"{value} is not in valid extensions. Valid file extensions: {VALID_EXTENSIONS}"
+            )
 
         return value
 
@@ -111,7 +115,7 @@ class DTCLICommand:
         if cmd_name in ("available_api_calls",):
             return "\n".join(response)
         if response.product in RTTF_PRODUCTS_LIST:
-            return "\n".join([data for data in response.response()])
+            pass  # do nothing
         return str(getattr(response, out_format) if out_format != "list" else response.as_list())
 
     @classmethod
@@ -203,7 +207,7 @@ class DTCLICommand:
                 transient=True,
             ) as progress:
 
-                progress.add_task(
+                task_id = progress.add_task(
                     description=f"Using api credentials with a username of: [cyan]{user}[/cyan]\nExecuting [green]{name}[/green] api call...",
                     total=None,
                 )
@@ -222,23 +226,33 @@ class DTCLICommand:
                 params = params | kwargs
 
                 response = dt_api_func(**params)
-                progress.add_task(
+                progress.update(
+                    task_id,
                     description=f"Preparing results with format of {response_format}...",
-                    total=None,
                 )
 
-                output = cls._get_formatted_output(cmd_name=name, response=response, out_format=response_format)
+                output = cls._get_formatted_output(
+                    cmd_name=name, response=response, out_format=response_format
+                )
 
                 if isinstance(out_file, _io.TextIOWrapper):
+                    progress.update(
+                        task_id,
+                        description=f"Printing the results with format of {response_format}...",
+                    )
                     # use rich `print` command to prettify the ouput in sys.stdout
                     if response.product in RTTF_PRODUCTS_LIST:
-                        print(output)
+                        for feeds in response.response():
+                            print(feeds)
                     else:
                         print(response)
                 else:
+                    progress.update(
+                        task_id,
+                        description=f"Writing results to {out_file}",
+                    )
                     # if it's a file then write
                     out_file.write(output if output.endswith("\n") else output + "\n")
-                time.sleep(0.25)
         except Exception as e:
             if isinstance(e, ServiceException):
                 code = typer.style(getattr(e, "code", 400), fg=typer.colors.BRIGHT_RED)

domaintools_api_test_upload-2.6.1/domaintools/results.py

@@ -0,0 +1,292 @@
+"""Defines the used Result object based on the current versions and/or features available to Python runtime
+
+Additionally, defines any custom result objects that may be used to enable more Pythonic interaction with endpoints.
+"""
+
+import json
+import logging
+import httpx
+import time
+
+try:  # pragma: no cover
+    from collections import OrderedDict
+except ImportError:  # pragma: no cover
+    from ordereddict import OrderedDict
+
+from itertools import zip_longest, chain
+from typing import Generator
+from httpx import Client
+
+from domaintools_async import AsyncResults as Results
+
+
+log = logging.getLogger(__name__)
+
+
+class Reputation(Results):
+    """Returns the reputation results in a format that can quickly be converted into floats / ints"""
+
+    def __float__(self):
+        return float(self["risk_score"])
+
+    def __int__(self):
+        return int(self["risk_score"])
+
+
+class GroupedIterable(Results):
+    """Returns a results item in a format that allows for grouped iteration of mulpitle result lists"""
+
+    def _items(self):
+        if self._items_list is None:
+            self._items_list = chain(
+                *[
+                    zip_longest([], value, fillvalue=key)
+                    for key, value in self.response().items()
+                    if type(value) in (list, tuple)
+                ]
+            )
+
+        return self._items_list
+
+
+class ParsedWhois(Results):
+    """Returns the parsed whois results in a format that can quickly be flattened"""
+
+    def flattened(self):
+        """Returns a flattened version of the parsed whois data"""
+        parsed = self["parsed_whois"]
+        flat = OrderedDict()
+        for key in (
+            "domain",
+            "created_date",
+            "updated_date",
+            "expired_date",
+            "statuses",
+            "name_servers",
+        ):
+            if key in parsed:
+                value = parsed[key]
+                flat[key] = " | ".join(value) if type(value) in (list, tuple) else value
+
+        registrar = parsed.get("registrar", {})
+        for key in (
+            "name",
+            "abuse_contact_phone",
+            "abuse_contact_email",
+            "iana_id",
+            "url",
+            "whois_server",
+        ):
+            if key in registrar:
+                flat["registrar_{0}".format(key)] = registrar[key]
+
+        if "networks" in parsed:
+            networks = parsed.get("networks")
+            for network in networks:
+                id = network.get("id")
+                for key in (
+                    "range",
+                    "asn",
+                    "org",
+                    "parent",
+                    "customer",
+                    "country",
+                    "phone",
+                    "status",
+                    "source",
+                    "updated_date",
+                    "created_date",
+                ):
+                    if key in network:
+                        value = network[key]
+                        flat["network_{0}".format(id)] = (
+                            " ".join(value) if type(value) in (list, tuple) else value
+                        )
+
+        if "contacts" in parsed:
+            contacts = parsed.get("contacts")
+            if type(contacts) is list:
+                # handle IP-style contacts, which show up as a list
+                for contact in contacts:
+                    contact_type = contact.get("type")
+                    for key in (
+                        "name",
+                        "email",
+                        "org",
+                        "abuse_mailbos",
+                        "address",
+                        "street",
+                        "city",
+                        "state",
+                        "postal",
+                        "country",
+                        "phone",
+                        "fax",
+                    ):
+                        if key in contact:
+                            value = contact[key]
+                            flat["{0}_{1}".format(contact_type, key)] = (
+                                " ".join(value) if type(value) in (list, tuple) else value
+                            )
+
+            elif type(contacts) is dict:
+                for contact_type in ("registrant", "admin", "tech", "billing"):
+                    contact = contacts.get(contact_type, {})
+                    for key in (
+                        "name",
+                        "email",
+                        "org",
+                        "street",
+                        "city",
+                        "state",
+                        "postal",
+                        "country",
+                        "phone",
+                        "fax",
+                    ):
+                        if key in contact:
+                            value = contact[key]
+                            flat["{0}_{1}".format(contact_type, key)] = (
+                                " ".join(value) if type(value) in (list, tuple) else value
+                            )
+
+        return flat
+
+
+class ParsedDomainRdap(Results):
+    """Returns the parsed domain rdap results in a format that can quickly be flattened"""
+
+    def flattened(self):
+        """Returns a flattened version of the parsed domain rdap data"""
+        parsed = self["parsed_domain_rdap"]
+        flat = OrderedDict()
+        for key in (
+            "domain",
+            "handle",
+            "domain_statuses",
+            "creation_date",
+            "last_changed_date",
+            "expiration_date",
+            "dnssec",
+            "nameservers",
+            "conformance",
+            "emails",
+            "email_domains",
+            "unclassified_emails",
+        ):
+            if key in parsed:
+                value = parsed[key]
+                flat[key] = " | ".join(value) if type(value) in (list, tuple) else value
+
+        registrar = parsed.get("registrar", {})
+        for registrar_key, registrar_value in registrar.items():
+            if registrar_key == "contacts":
+                for i, contact in enumerate(registrar_value, start=1):
+                    for contact_key, contact_value in contact.items():
+                        flat[f"registrar_contacts_{contact_key}"] = (
+                            " | ".join(contact_value)
+                            if type(contact_value) in (list, tuple)
+                            else contact_value
+                        )
+
+                continue
+            flat[f"registrar_{registrar_key}"] = registrar_value
+
+        contacts = parsed.get("contacts")
+        if contacts:
+            for i, contact in enumerate(contacts, start=1):
+                for contact_key, contact_value in contact.items():
+                    flat[f"contact_{contact_key}_{i}"] = (
+                        " | ".join(contact_value) if type(contact_value) in (list, tuple) else contact_value
+                    )
+
+        return flat
+
+
+class FeedsResults(Results):
+    """
+    Real Time Threat Feeds (RTTF) returns an application/ndjson stream.
+    With this we use httpx stream to process each JSON object efficiently.
+
+    Highlevel process:
+
+    httpx stream -> yield each json line -> check status code -> yield back data to client
+
+    Returns the generator object for feeds results.
+    """
+
+    latest_feeds_status_code = None
+
+    def _make_request(self) -> Generator:
+        """
+        Creates and manages the httpx stream request, yielding data line by line.
+        This is the core generator that communicates with the DT frontend API server.
+        """
+        session_info = self._get_session_params_and_headers()
+        headers = session_info.get("headers")
+        parameters = session_info.get("parameters")
+
+        try:
+            with httpx.stream(
+                "GET",
+                self.url,
+                headers=headers,
+                params=parameters,
+                verify=self.api.verify_ssl,
+                proxy=self.api.proxy_url,
+                timeout=None,
+            ) as response:
+                self.latest_feeds_status_code = response.status_code
+                # assigned the status code to `latest_feeds_status_code`
+                # started the process
+                yield {"status_ready": True}
+
+                for line in response.iter_lines():
+                    if line:
+                        yield line
+        except Exception as e:
+            self.latest_feeds_status_code = 500
+            raise e
+
+    def _get_results(self) -> Generator:
+        try:
+            wait_for = self._wait_time()
+            if wait_for:
+                log.info(f"Sleeping for {wait_for}s BEFORE REQUEST.")
+                time.sleep(wait_for)
+
+            feeds_generator = self._make_request()
+
+            next(feeds_generator)  # to start the generator process
+            self.setStatus(self.latest_feeds_status_code)  # set the status already
+
+            # yield the rest of the feeds
+            yield from feeds_generator
+        except Exception as e:
+            self.setStatus(500, reason_text=e)
+            return
+
+    def data(self) -> Generator:
+        self._data = self._get_results()
+        return self._data
+
+    def response(self) -> Generator:
+        status_code = None
+        while status_code != 200:
+            try:
+                feed_response_generator = self.data()
+
+                yield from feed_response_generator
+                status_code = self.status
+                self._data = None  # clear the data here
+
+                if not self.kwargs.get("sessionID"):
+                    # we'll only do iterative request for queries that has sessionID.
+                    # Otherwise, we will have an infinite request if sessionID was not provided but the required data asked is more than the maximum (1 hour of data)
+                    break
+            except Exception as e:
+                self.setStatus(500, reason_text=e)
+                break  # safely close the while loop if there's any error above
+
+    def __str__(self):
+        return f"{self.__class__.__name__} - {self.product}"

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1/domaintools_api_test_upload.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
-Name: domaintools_api_test_upload
-Version: 2.5.5
+Name: domaintools-api-test-upload
+Version: 2.6.1
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "domaintools_api_test_upload"
+name = "domaintools-api-test-upload"
 dependencies = [
     "httpx==0.28.1",
     "rich",
@@ -49,3 +49,6 @@ packages = [
 
 [tool.setuptools.dynamic]
 version = { "file" = "VERSION" }
+
+[tool.black]
+line-length = 110

{domaintools_api_test_upload-2.5.5 → domaintools_api_test_upload-2.6.1}/setup.py

@@ -60,7 +60,7 @@ class PyTest(TestCommand):
 cmdclass["test"] = PyTest
 
 setup(
-    name="domaintools_api",
+    name="domaintools-api-test-upload",
     cmdclass=cmdclass,
     ext_modules=ext_modules,
 )