domaintools-api 2.4.0__tar.gz → 2.5.0__tar.gz

{domaintools_api-2.4.0/domaintools_api.egg-info → domaintools_api-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domaintools_api
-Version: 2.4.0
+Version: 2.5.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -321,3 +321,43 @@ results = api.nod(sessionID="my-session-id", after=-7200)
 for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
     # do things to partial_result
 ```
+
+
+Running E2E Tests Locally
+===================
+For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
+To add more e2e tests, put these in the `../tests/e2e` folder.
+
+## Preparation
+- Create virtual environment.
+    ```bash
+        python3 -m venv venv
+    ```
+
+- Activate virtual environment
+    ```bash
+        source venv/bin/activate
+    ```
+
+- Install dependencies.
+    ```bash
+        pip install -r requirements/development.txt
+    ```
+
+- From the python_api project root directory, install the package.
+    ```bash
+        pip install -e .
+    ```
+
+- Export api credentials to use.
+    ```bash
+        export TEST_USER=<user-key>
+        export TEST_KEY=<api-key>
+    ```
+
+## Run the end-to-end test script
+- Before running the test, be sure that docker is running.
+- Execute the e2e test script .
+    ```bash
+        sh tests/e2e/scripts/test_e2e_runner.sh
+    ```

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/README.md

@@ -270,4 +270,44 @@ results = api.nod(sessionID="my-session-id", after=-7200)
 
 for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
     # do things to partial_result
-```
+```
+
+
+Running E2E Tests Locally
+===================
+For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
+To add more e2e tests, put these in the `../tests/e2e` folder.
+
+## Preparation
+- Create virtual environment.
+    ```bash
+        python3 -m venv venv
+    ```
+
+- Activate virtual environment
+    ```bash
+        source venv/bin/activate
+    ```
+
+- Install dependencies.
+    ```bash
+        pip install -r requirements/development.txt
+    ```
+
+- From the python_api project root directory, install the package.
+    ```bash
+        pip install -e .
+    ```
+
+- Export api credentials to use.
+    ```bash
+        export TEST_USER=<user-key>
+        export TEST_KEY=<api-key>
+    ```
+
+## Run the end-to-end test script
+- Before running the test, be sure that docker is running.
+- Execute the e2e test script .
+    ```bash
+        sh tests/e2e/scripts/test_e2e_runner.sh
+    ```

domaintools_api-2.5.0/VERSION

@@ -0,0 +1 @@
+2.5.0

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/_version.py

@@ -20,4 +20,4 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 """
 
-current = "2.4.0"
+current = "2.5.0"

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/api.py

@@ -1,8 +1,10 @@
 from datetime import datetime, timedelta, timezone
-from hashlib import sha1, sha256, md5
+from hashlib import sha1, sha256
 from hmac import new as hmac
+from typing import Union
 
 import re
+import ssl
 
 from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat
 from domaintools._version import current as version
@@ -24,7 +26,7 @@ from domaintools.filters import (
 from domaintools.utils import validate_feeds_parameters
 
 
-AVAILABLE_KEY_SIGN_HASHES = ["sha1", "sha256", "md5"]
+AVAILABLE_KEY_SIGN_HASHES = ["sha1", "sha256"]
 
 
 def delimited(items, character="|"):
@@ -76,7 +78,7 @@ class API(object):
         self.username = username
         self.key = key
         self.https = https
-        self.verify_ssl = verify_ssl
+        self.verify_ssl = self._get_ssl_default_context(verify_ssl)
         self.rate_limit = rate_limit
         self.proxy_url = proxy_url
         self.extra_request_params = {}
@@ -92,6 +94,9 @@ class API(object):
         if proxy_url and not isinstance(proxy_url, str):
             raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
 
+    def _get_ssl_default_context(self, verify_ssl: Union[str, bool]):
+        return ssl.create_default_context(cafile=verify_ssl) if isinstance(verify_ssl, str) else verify_ssl
+
     def _build_api_url(self, api_url=None, api_port=None):
         """Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`"""
         rest_api_url = "https://api.domaintools.com"
@@ -1187,3 +1192,65 @@ class API(object):
             cls=FeedsResults,
             **kwargs,
         )
+
+    def realtime_domain_risk(self, **kwargs) -> FeedsResults:
+        """Returns back list of the realtime domain risk feed.
+        Contains realtime domain risk information for apex-level domains, regardless of observed traffic.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"domain-risk-({source})",
+            f"v1/{endpoint}/domainrisk/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )
+
+    def domainhotlist(self, **kwargs) -> FeedsResults:
+        """Returns back list of domain hotlist feed.
+        Contains high-risk, apex-level domains that are observed by DomainTools' global sensor network to be active within 24 hours.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"domain-hotlist-feed-({source})",
+            f"v1/{endpoint}/domainhotlist/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/cli/commands/feeds.py

@@ -394,3 +394,163 @@ def feeds_noh(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_DOMAINHOTLIST,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_DOMAINHOTLIST),
+)
+def feeds_domainhotlist(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_DOMAINHOTLIST, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_REALTIME_DOMAIN_RISK,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_REALTIME_DOMAIN_RISK),
+)
+def feeds_realtime_domain_risk(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_REALTIME_DOMAIN_RISK, params=ctx.params)

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/cli/constants.py

@@ -43,5 +43,7 @@ IRIS_DETECT_IGNORED_DOMAINS = "iris_detect_ignored_domains"
 FEEDS_NAD = "nad"
 FEEDS_NOD = "nod"
 FEEDS_NOH = "noh"
+FEEDS_DOMAINHOTLIST = "domainhotlist"
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"
+FEEDS_REALTIME_DOMAIN_RISK = "realtime_domain_risk"

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/cli/utils.py

@@ -64,8 +64,10 @@ _command_help_mapper = {
     c.FEEDS_NAD: "Returns back newly active domains feed.",
     c.FEEDS_NOD: "Returns back newly observed domains feed.",
     c.FEEDS_NOH: "Returns back newly observed hosts feed.",
+    c.FEEDS_DOMAINHOTLIST: "Returns domaint hotlist feed.",
     c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
     c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
+    c.FEEDS_REALTIME_DOMAIN_RISK: "Returns realtime domain risk information for apex-level domains, regardless of observed traffic.",
 }
 
 

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/domaintools/constants.py

@@ -30,8 +30,12 @@ FEEDS_PRODUCTS_LIST = [
     "newly-observed-domains-feed-(s3)",
     "newly-observed-hosts-feed-(api)",
     "newly-observed-hosts-feed-(s3)",
+    "domain-hotlist-feed-(api)",
+    "domain-hotlist-feed-(s3)",
     "domain-registration-data-access-protocol-feed-(api)",
     "domain-registration-data-access-protocol-feed-(s3)",
+    "domain-risk-feed-(api)",
+    "domain-risk-feed-(s3)",
     "real-time-domain-discovery-feed-(api)",
     "real-time-domain-discovery-feed-(s3)",
 ]

{domaintools_api-2.4.0 → domaintools_api-2.5.0/domaintools_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: domaintools_api
-Version: 2.4.0
+Version: 2.5.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -321,3 +321,43 @@ results = api.nod(sessionID="my-session-id", after=-7200)
 for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
     # do things to partial_result
 ```
+
+
+Running E2E Tests Locally
+===================
+For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
+To add more e2e tests, put these in the `../tests/e2e` folder.
+
+## Preparation
+- Create virtual environment.
+    ```bash
+        python3 -m venv venv
+    ```
+
+- Activate virtual environment
+    ```bash
+        source venv/bin/activate
+    ```
+
+- Install dependencies.
+    ```bash
+        pip install -r requirements/development.txt
+    ```
+
+- From the python_api project root directory, install the package.
+    ```bash
+        pip install -e .
+    ```
+
+- Export api credentials to use.
+    ```bash
+        export TEST_USER=<user-key>
+        export TEST_KEY=<api-key>
+    ```
+
+## Run the end-to-end test script
+- Before running the test, be sure that docker is running.
+- Execute the e2e test script .
+    ```bash
+        sh tests/e2e/scripts/test_e2e_runner.sh
+    ```

{domaintools_api-2.4.0 → domaintools_api-2.5.0}/tests/test_api.py

@@ -342,11 +342,20 @@ def test_exception_handling():
         API("notauser", "notakey").domain_search("amazon").data()
     with pytest.raises(
         ValueError,
-        match=r"Invalid value 'notahash' for 'key_sign_hash'. Values available are sha1,sha256,md5",
+        match=r"Invalid value 'notahash' for 'key_sign_hash'. Values available are sha1,sha256",
     ):
         API("notauser", "notakey", always_sign_api_key=True, key_sign_hash="notahash").domain_search("amazon")
 
 
+@vcr.use_cassette
+def test_md5_is_not_supported():
+    with pytest.raises(
+        ValueError,
+        match=r"Invalid value 'md5' for 'key_sign_hash'. Values available are sha1,sha256",
+    ):
+        API("notauser", "notakey", always_sign_api_key=True, key_sign_hash="md5").domain_search("amazon")
+
+
 @vcr.use_cassette
 def test_rate_limiting():
     domain_searches = ["google"] * 31
@@ -377,6 +386,7 @@ def test_formats():
 
 
 @vcr.use_cassette
+@pytest.mark.skip(reason="Iris endpoint is already deprecated in light of enrich and investigate")
 def test_iris():
     with pytest.raises(ValueError):
         api.iris()
@@ -663,3 +673,43 @@ def test_feeds_endpoint_should_raise_error_if_asked_csv_format_for_download_api(
         feeds_api.domaindiscovery(after="-60", output_format="csv", endpoint="download")
 
     assert str(excinfo.value) == "csv format is not available in download API."
+
+
+@vcr.use_cassette
+def test_realtime_domain_risk():
+    results = feeds_api.realtime_domain_risk(after="-60", header_authentication=False)
+    for response in results.response():
+        assert results.status == 200
+
+        rows = response.strip().split("\n")
+        assert response is not None
+        assert len(rows) >= 1
+
+        for row in rows:
+            feed_result = json.loads(row)
+            assert "timestamp" in feed_result.keys()
+            assert "domain" in feed_result.keys()
+            assert "phishing_risk" in feed_result.keys()
+            assert "malware_risk" in feed_result.keys()
+            assert "proximity_risk" in feed_result.keys()
+            assert "overall_risk" in feed_result.keys()
+
+
+@vcr.use_cassette
+def test_domain_hotlist():
+    results = feeds_api.domainhotlist(after="-60", header_authentication=False)
+    for response in results.response():
+        assert results.status == 200
+
+        rows = response.strip().split("\n")
+        assert response is not None
+        assert len(rows) >= 1
+
+        for row in rows:
+            feed_result = json.loads(row)
+            assert "timestamp" in feed_result.keys()
+            assert "domain" in feed_result.keys()
+            assert "phishing_risk" in feed_result.keys()
+            assert "malware_risk" in feed_result.keys()
+            assert "proximity_risk" in feed_result.keys()
+            assert "overall_risk" in feed_result.keys()

domaintools_api-2.4.0/VERSION

@@ -1 +0,0 @@
-2.4.0