domaintools-api 2.0.0__tar.gz → 2.1.0__tar.gz

{domaintools_api-2.0.0/domaintools_api.egg-info → domaintools_api-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: domaintools_api
-Version: 2.0.0
+Version: 2.1.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)

domaintools_api-2.1.0/VERSION

@@ -0,0 +1 @@
+2.1.0

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/_version.py

@@ -20,4 +20,4 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 """
 
-current = "2.0.0"
+current = "2.1.0"

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/api.py

@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta, timezone
 from hashlib import sha1, sha256, md5
 from hmac import new as hmac
-from types import MethodType
 import re
 
 from domaintools._version import current as version
@@ -51,7 +50,7 @@ class API(object):
         verify_ssl=True,
         rate_limit=True,
         proxy_url=None,
-        always_sign_api_key=False,
+        always_sign_api_key=True,
         key_sign_hash="sha256",
         app_name="python_wrapper",
         app_version=version,
@@ -1120,3 +1119,31 @@ class API(object):
             response_path=(),
             **kwargs,
         )
+
+    def nod(self, **kwargs):
+        """Returns back list of the newly observed domains feed"""
+        sessionID = kwargs.get("sessionID")
+        after = kwargs.get("after")
+        if not (sessionID or after):
+            raise ValueError("sessionID or after (can be both) must be defined")
+
+        return self._results(
+            "newly-observed-domains-feed-(api)",
+            "v1/feed/nod/",
+            response_path=(),
+            **kwargs,
+        )
+
+    def nad(self, **kwargs):
+        """Returns back list of the newly active domains feed"""
+        sessionID = kwargs.get("sessionID")
+        after = kwargs.get("after")
+        if not (sessionID or after):
+            raise ValueError("sessionID or after (can be both) must be defined")
+
+        return self._results(
+            "newly-active-domains-feed-(api)",
+            "v1/feed/nad/",
+            response_path=(),
+            **kwargs,
+        )

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/base_results.py

@@ -1,27 +1,46 @@
 """Defines the base result object - which specifies how DomainTools API endpoints will be interacted with"""
-import collections
+
 import json
 import re
 import time
 import logging
 from datetime import datetime
 
-from domaintools.exceptions import (BadRequestException, InternalServerErrorException, NotAuthorizedException,
-                                    NotFoundException, ServiceException, ServiceUnavailableException,
-                                    IncompleteResponseException, RequestUriTooLongException)
+from domaintools.exceptions import (
+    BadRequestException,
+    InternalServerErrorException,
+    NotAuthorizedException,
+    NotFoundException,
+    ServiceException,
+    ServiceUnavailableException,
+    IncompleteResponseException,
+    RequestUriTooLongException,
+)
+from domaintools.utils import get_feeds_products_list
+
 from httpx import Client
 
-try: # pragma: no cover
+try:  # pragma: no cover
     from collections.abc import MutableMapping, MutableSequence
-except ImportError: # pragma: no cover
+except ImportError:  # pragma: no cover
     from collections import MutableMapping, MutableSequence
 
 log = logging.getLogger(__name__)
 
+
 class Results(MutableMapping, MutableSequence):
     """The base (abstract) DomainTools result definition"""
 
-    def __init__(self, api, product, url, items_path=(), response_path=('response', ), proxy_url=None, **kwargs):
+    def __init__(
+        self,
+        api,
+        product,
+        url,
+        items_path=(),
+        response_path=("response",),
+        proxy_url=None,
+        **kwargs,
+    ):
         self.api = api
         self.product = product
         self.url = url
@@ -41,51 +60,67 @@ class Results(MutableMapping, MutableSequence):
 
         now = datetime.now()
         limit = self.api.limits[self.product]
-        if 'last_scheduled' not in limit:
-            limit['last_scheduled'] = now
+        if "last_scheduled" not in limit:
+            limit["last_scheduled"] = now
             return None
 
-        safe_after = limit['last_scheduled'] + limit['interval']
+        safe_after = limit["last_scheduled"] + limit["interval"]
         wait_for = 0
         if now < safe_after:
             wait_for = safe_after - now
-            wait_for = float(wait_for.seconds) + (float(wait_for.microseconds) / 1000000.0)
-            limit['last_scheduled'] = safe_after
+            wait_for = float(wait_for.seconds) + (
+                float(wait_for.microseconds) / 1000000.0
+            )
+            limit["last_scheduled"] = safe_after
         else:
-            limit['last_scheduled'] = now
+            limit["last_scheduled"] = now
 
         return wait_for
 
     def _make_request(self):
 
-        with Client(verify=self.api.verify_ssl, proxies=self.api.proxy_url, timeout=None) as session:
-            if self.product in ['iris-investigate', 'iris-enrich', 'iris-detect-escalate-domains']:
+        with Client(
+            verify=self.api.verify_ssl, proxies=self.api.proxy_url, timeout=None
+        ) as session:
+            if self.product in [
+                "iris-investigate",
+                "iris-enrich",
+                "iris-detect-escalate-domains",
+            ]:
                 post_data = self.kwargs.copy()
                 post_data.update(self.api.extra_request_params)
                 return session.post(url=self.url, data=post_data)
-            elif self.product in ['iris-detect-manage-watchlist-domains']:
+            elif self.product in ["iris-detect-manage-watchlist-domains"]:
                 patch_data = self.kwargs.copy()
                 patch_data.update(self.api.extra_request_params)
                 return session.patch(url=self.url, json=patch_data)
             else:
-                return session.get(url=self.url, params=self.kwargs, **self.api.extra_request_params)
+                return session.get(
+                    url=self.url, params=self.kwargs, **self.api.extra_request_params
+                )
 
     def _get_results(self):
         wait_for = self._wait_time()
-        if self.api.rate_limit and (wait_for is None or self.product == 'account-information'):
+        if self.api.rate_limit and (
+            wait_for is None or self.product == "account-information"
+        ):
             data = self._make_request()
-            if data.status_code == 503: # pragma: no cover
+            if data.status_code == 503:  # pragma: no cover
                 sleeptime = 60
-                log.info('503 encountered for [%s] - sleeping [%s] seconds before retrying request.',
-                         self.product, sleeptime)
+                log.info(
+                    "503 encountered for [%s] - sleeping [%s] seconds before retrying request.",
+                    self.product,
+                    sleeptime,
+                )
                 time.sleep(sleeptime)
                 self._wait_time()
                 data = self._make_request()
             return data
 
         if wait_for > 0:
-            log.info('Sleeping for [%s] prior to requesting [%s].',
-                     wait_for, self.product)
+            log.info(
+                "Sleeping for [%s] prior to requesting [%s].", wait_for, self.product
+            )
             time.sleep(wait_for)
         return self._make_request()
 
@@ -93,7 +128,11 @@ class Results(MutableMapping, MutableSequence):
         if self._data is None:
             results = self._get_results()
             self.setStatus(results.status_code, results)
-            if self.kwargs.get('format', 'json') == 'json':
+            if (
+                self.kwargs.get("format", "json") == "json"
+                and self.product
+                not in get_feeds_products_list()  # Special handling of feeds products' data to preserve the result in jsonline format
+            ):
                 self._data = results.json()
             else:
                 self._data = results.text
@@ -104,16 +143,20 @@ class Results(MutableMapping, MutableSequence):
                 self._limit_exceeded_message = message
 
         if self._limit_exceeded is True:
-            raise ServiceException(503, "Limit Exceeded{}".format(self._limit_exceeded_message))
+            raise ServiceException(
+                503, "Limit Exceeded{}".format(self._limit_exceeded_message)
+            )
         else:
             return self._data
 
     def check_limit_exceeded(self):
-        if self.kwargs.get('format', 'json') == 'json':
-            if ("response" in self._data and
-                "limit_exceeded" in self._data['response'] and
-                self._data['response']['limit_exceeded'] is True):
-                return True, self._data['response']['message']
+        if self.kwargs.get("format", "json") == "json":
+            if (
+                "response" in self._data
+                and "limit_exceeded" in self._data["response"]
+                and self._data["response"]["limit_exceeded"] is True
+            ):
+                return True, self._data["response"]["message"]
         # TODO: handle html, xml response errors better.
         elif "response" in self._data and "limit_exceeded" in self._data:
             return True, "limit exceeded"
@@ -121,7 +164,7 @@ class Results(MutableMapping, MutableSequence):
 
     @property
     def status(self):
-        if not getattr(self, '_status', None):
+        if not getattr(self, "_status", None):
             self._status = self._get_results().status_code
 
         return self._status
@@ -135,7 +178,7 @@ class Results(MutableMapping, MutableSequence):
         if response is not None:
             try:
                 reason = response.json()
-            except Exception: # pragma: no cover
+            except Exception:  # pragma: no cover
                 reason = response.text
                 if callable(reason):
                     reason = reason()
@@ -146,16 +189,16 @@ class Results(MutableMapping, MutableSequence):
             raise NotAuthorizedException(code, reason)
         elif code == 404:
             raise NotFoundException(code, reason)
-        elif code == 500: # pragma: no cover
+        elif code == 500:  # pragma: no cover
             raise InternalServerErrorException(code, reason)
-        elif code == 503: # pragma: no cover
+        elif code == 503:  # pragma: no cover
             raise ServiceUnavailableException(code, reason)
-        elif code == 206: # pragma: no cover
+        elif code == 206:  # pragma: no cover
             raise IncompleteResponseException(code, reason)
-        elif code == 414: # pragma: no cover
+        elif code == 414:  # pragma: no cover
             raise RequestUriTooLongException(code, reason)
-        else: # pragma: no cover
-            raise ServiceException(code, 'Unknown Exception')
+        else:  # pragma: no cover
+            raise ServiceException(code, "Unknown Exception")
 
     def response(self):
         if self._response is None:
@@ -171,7 +214,7 @@ class Results(MutableMapping, MutableSequence):
 
     def emails(self):
         """Find and returns all emails mentioned in the response"""
-        return set(re.findall(r'[\w\.-]+@[\w\.-]+', str(self.response())))
+        return set(re.findall(r"[\w\.-]+@[\w\.-]+", str(self.response())))
 
     def _items(self):
         if self._items_list is None:
@@ -221,25 +264,54 @@ class Results(MutableMapping, MutableSequence):
 
     @property
     def json(self):
-        self.kwargs.pop('format', None)
-        return self.__class__(format='json', product=self.product, url=self.url, items_path=self.items_path,
-                              response_path=self.response_path, api=self.api, **self.kwargs)
+        self.kwargs.pop("format", None)
+        return self.__class__(
+            format="json",
+            product=self.product,
+            url=self.url,
+            items_path=self.items_path,
+            response_path=self.response_path,
+            api=self.api,
+            **self.kwargs,
+        )
 
     @property
     def xml(self):
-        self.kwargs.pop('format', None)
-        return self.__class__(format='xml', product=self.product, url=self.url, items_path=self.items_path,
-                              response_path=self.response_path, api=self.api, **self.kwargs)
+        self.kwargs.pop("format", None)
+        return self.__class__(
+            format="xml",
+            product=self.product,
+            url=self.url,
+            items_path=self.items_path,
+            response_path=self.response_path,
+            api=self.api,
+            **self.kwargs,
+        )
 
     @property
     def html(self):
-        self.kwargs.pop('format', None)
-        return self.__class__(api=self.api, product=self.product, url=self.url, items_path=self.items_path,
-                              response_path=self.response_path, format='html', **self.kwargs)
+        self.kwargs.pop("format", None)
+        return self.__class__(
+            api=self.api,
+            product=self.product,
+            url=self.url,
+            items_path=self.items_path,
+            response_path=self.response_path,
+            format="html",
+            **self.kwargs,
+        )
 
     def as_list(self):
-        return '\n'.join([json.dumps(item, indent=4, separators=(',', ': ')) for item in self._items()])
+        return "\n".join(
+            [
+                json.dumps(item, indent=4, separators=(",", ": "))
+                for item in self._items()
+            ]
+        )
 
     def __str__(self):
-        return str(json.dumps(self.data(), indent=4,
-                              separators=(',', ': ')) if self.kwargs.get('format', 'json') == 'json' else self.data())
+        return str(
+            json.dumps(self.data(), indent=4, separators=(",", ": "))
+            if self.kwargs.get("format", "json") == "json"
+            else self.data()
+        )

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/cli/api.py

@@ -148,10 +148,11 @@ class DTCLICommand:
             kwargs (Optional[Dict], optional): The command available kwargs to pass in domaintools API
         """
         try:
-            rate_limit = params.pop("rate_limit") or False
-            response_format = params.pop("format") or "json"
-            out_file = params.pop("out_file") or sys.stdout
-            verify_ssl = params.pop("no_verify_ssl") or False
+            rate_limit = params.pop("rate_limit", False)
+            response_format = params.pop("format", "json")
+            out_file = params.pop("out_file", sys.stdout)
+            verify_ssl = params.pop("no_verify_ssl", False)
+            always_sign_api_key = params.pop("no_sign_api_key", False)
             source = None
 
             if "src_file" in params:
@@ -185,6 +186,7 @@ class DTCLICommand:
                     app_name=cls.APP_PARTNER_NAME,
                     verify_ssl=verify_ssl,
                     rate_limit=rate_limit,
+                    always_sign_api_key=always_sign_api_key,
                 )
                 dt_api_func = getattr(dt_api, name)
 

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/cli/commands/__init__.py

@@ -4,3 +4,4 @@ from .domains import *
 from .ips import *
 from .phisheye import *
 from .detects import *
+from .feeds import *

domaintools_api-2.1.0/domaintools/cli/commands/feeds.py

@@ -0,0 +1,138 @@
+import sys
+import typer
+
+
+from domaintools.cli.main import dt_cli
+from domaintools.cli.api import DTCLICommand
+from domaintools.cli.utils import get_cli_helptext_by_name
+from domaintools.cli import constants as c
+
+
+@dt_cli.command(
+    name=c.FEEDS_NAD,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_NAD),
+)
+def feeds_nad(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    rate_limit: bool = typer.Option(
+        False,
+        "-l",
+        "--rate-limit",
+        help="Rate limit API calls against the API based on per minute limits.",
+    ),
+    format: str = typer.Option(
+        "json",
+        "-f",
+        "--format",
+        help="Output format in {'list', 'json', 'xml', 'html'}",
+        callback=DTCLICommand.validate_format_input,
+    ),
+    out_file: typer.FileTextWrite = typer.Option(
+        sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_NAD, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_NOD,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_NOD),
+)
+def feeds_nod(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    rate_limit: bool = typer.Option(
+        False,
+        "-l",
+        "--rate-limit",
+        help="Rate limit API calls against the API based on per minute limits.",
+    ),
+    format: str = typer.Option(
+        "json",
+        "-f",
+        "--format",
+        help="Output format in {'list', 'json', 'xml', 'html'}",
+        callback=DTCLICommand.validate_format_input,
+    ),
+    out_file: typer.FileTextWrite = typer.Option(
+        sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_NOD, params=ctx.params)

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/cli/constants.py

@@ -41,3 +41,7 @@ IRIS_DETECT_IGNORED_DOMAINS = "iris_detect_ignored_domains"
 # phisheye
 PHISHEYE = "phisheye"
 PHISHEYE_TERM_LIST = "phisheye_term_list"
+
+# feeds
+FEEDS_NAD = "nad"
+FEEDS_NOD = "nod"

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/cli/utils.py

@@ -81,6 +81,8 @@ _command_help_mapper = {
     c.IRIS_DETECT_MANAGE_WATCHLIST_DOMAINS: "Changes the watch state of a list of domains by their Iris Detect domain ID.",
     c.IRIS_DETECT_ESCALATE_DOMAINS: "Changes the escalation type of a list of domains by their Iris Detect domain ID.",
     c.IRIS_DETECT_IGNORED_DOMAINS: "Returns back a list of ignored domains in Iris Detect based on the provided filters.",
+    c.FEEDS_NAD: "Returns back newly active domains feed.",
+    c.FEEDS_NOD: "Returns back newly observed domains feed.",
 }
 
 

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools/utils.py

@@ -179,3 +179,10 @@ def convert_str_to_dateobj(
     string_date: str, date_format: Optional[str] = "%Y-%m-%d"
 ) -> datetime:
     return datetime.strptime(string_date, date_format)
+
+
+def get_feeds_products_list():
+    return [
+        "newly-active-domains-feed-(api)",
+        "newly-observed-domains-feed-(api)",
+    ]

{domaintools_api-2.0.0 → domaintools_api-2.1.0/domaintools_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: domaintools_api
-Version: 2.0.0
+Version: 2.1.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/domaintools_api.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ domaintools/cli/commands/__init__.py
 domaintools/cli/commands/accounts.py
 domaintools/cli/commands/detects.py
 domaintools/cli/commands/domains.py
+domaintools/cli/commands/feeds.py
 domaintools/cli/commands/ips.py
 domaintools/cli/commands/iris.py
 domaintools/cli/commands/phisheye.py

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/setup.py

@@ -60,7 +60,7 @@ class PyTest(TestCommand):
 cmdclass["test"] = PyTest
 
 setup(
+    name="domaintools_api",
     cmdclass=cmdclass,
     ext_modules=ext_modules,
-    **PyTest.extra_kwargs,
 )

{domaintools_api-2.0.0 → domaintools_api-2.1.0}/tests/test_api.py

@@ -1,11 +1,11 @@
 """Tests the Python interface for DomainTools APIs"""
 
 from os import environ
-from datetime import datetime
 
+import json
 import pytest
-from domaintools import API, exceptions
 
+from domaintools import API, exceptions
 from tests.settings import api, vcr
 
 
@@ -513,3 +513,35 @@ def test_limit_exceeded():
     with pytest.raises(exceptions.ServiceException):
         response = api.iris_investigate(ip="8.8.8.8")
         response.response()
+
+
+@vcr.use_cassette
+def test_newly_observed_domains_feed():
+    results = api.nod(after="-60")
+    response = results.response()
+    rows = response.strip().split("\n")
+
+    assert response is not None
+    assert results.status == 200
+    assert len(rows) >= 1
+
+    for row in rows:
+        feed_result = json.loads(row)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()
+
+
+@vcr.use_cassette
+def test_newly_active_domains_feed():
+    results = api.nad(after="-60")
+    response = results.response()
+    rows = response.strip().split("\n")
+
+    assert response is not None
+    assert results.status == 200
+    assert len(rows) >= 1
+
+    for row in rows:
+        feed_result = json.loads(row)
+        assert "timestamp" in feed_result.keys()
+        assert "domain" in feed_result.keys()

domaintools_api-2.0.0/VERSION

@@ -1 +0,0 @@
-2.0.0