domaintools-api-test-upload 2.2.2__tar.gz → 2.4.0__tar.gz

{domaintools_api_test_upload-2.2.2/domaintools_api_test_upload.egg-info → domaintools_api_test_upload-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: domaintools_api_test_upload
-Version: 2.2.2
+Version: 2.4.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -46,6 +46,7 @@ Requires-Dist: typer
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: mock; extra == "test"
+Dynamic: license-file
 
 ![domaintools](https://github.com/DomainTools/python_api/raw/main/artwork/logo.png)
 ===================
@@ -275,7 +276,7 @@ Custom parameters aside from the common `GET` Request parameters:
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(endpoint="feed", **kwargs)
     ```
-- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so you need to set this to `False` when calling `download` API endpoints.
+- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so this will be defaulted to `False` even without explicitly setting it.
     ```python
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(header_authentication=False, **kwargs)

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/README.md

@@ -226,7 +226,7 @@ Custom parameters aside from the common `GET` Request parameters:
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(endpoint="feed", **kwargs)
     ```
-- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so you need to set this to `False` when calling `download` API endpoints.
+- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so this will be defaulted to `False` even without explicitly setting it.
     ```python
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(header_authentication=False, **kwargs)

domaintools_api_test_upload-2.4.0/VERSION

@@ -0,0 +1 @@
+2.4.0

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/_version.py

@@ -20,4 +20,4 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 """
 
-current = "2.2.2"
+current = "2.4.0"

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/api.py

@@ -89,11 +89,8 @@ class API(object):
 
         if not https:
             raise Exception("The DomainTools API endpoints no longer support http traffic. Please make sure https=True.")
-        if proxy_url:
-            if isinstance(proxy_url, str):
-                self.proxy_url = {"http://": proxy_url, "https://": proxy_url}
-            else:
-                raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
+        if proxy_url and not isinstance(proxy_url, str):
+            raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
 
     def _build_api_url(self, api_url=None, api_port=None):
         """Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`"""
@@ -414,38 +411,6 @@ class API(object):
             **kwargs,
         )
 
-    def phisheye(self, query, days_back=None, **kwargs):
-        """Returns domain results for the specified term for today or the specified number of days_back.
-        Terms must be setup for monitoring via the web interface: https://research.domaintools.com/phisheye.
-
-        NOTE: Properties of a domain are only provided if we have been able to obtain them.
-              Many domains will have incomplete data because that information isn't available in their Whois records,
-              or they don't have DNS results for a name server or IP address.
-        """
-        return self._results(
-            "phisheye",
-            "/v1/phisheye",
-            query=query,
-            days_back=days_back,
-            items_path=("domains",),
-            **kwargs,
-        )
-
-    def phisheye_term_list(self, include_inactive=False, **kwargs):
-        """Provides a list of terms that are set up for this account.
-        This call is not charged against your API usage limit.
-
-        NOTE: The terms must be configured in the PhishEye web interface: https://research.domaintools.com/phisheye.
-              There is no API call to set up the terms.
-        """
-        return self._results(
-            "phisheye_term_list",
-            "/v1/phisheye/term-list",
-            include_inactive=include_inactive,
-            items_path=("terms",),
-            **kwargs,
-        )
-
     def iris(
         self,
         domain=None,
@@ -1131,3 +1096,20 @@ class API(object):
             cls=FeedsResults,
             **kwargs,
         )
+
+    def noh(self, **kwargs) -> FeedsResults:
+        """Returns back list of the newly observed hostnames feed"""
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"newly-observed-hosts-feed-({source})",
+            f"v1/{endpoint}/noh/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/cli/commands/__init__.py

@@ -2,6 +2,5 @@ from .accounts import *
 from .iris import *
 from .domains import *
 from .ips import *
-from .phisheye import *
 from .detects import *
 from .feeds import *

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/cli/commands/feeds.py

@@ -314,3 +314,83 @@ def feeds_domaindiscovery(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_DOMAINDISCOVERY, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_NOH,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_NOH),
+)
+def feeds_noh(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/cli/constants.py

@@ -39,12 +39,9 @@ IRIS_DETECT_MANAGE_WATCHLIST_DOMAINS = "iris_detect_manage_watchlist_domains"
 IRIS_DETECT_ESCALATE_DOMAINS = "iris_detect_escalate_domains"
 IRIS_DETECT_IGNORED_DOMAINS = "iris_detect_ignored_domains"
 
-# phisheye
-PHISHEYE = "phisheye"
-PHISHEYE_TERM_LIST = "phisheye_term_list"
-
 # feeds
 FEEDS_NAD = "nad"
 FEEDS_NOD = "nod"
+FEEDS_NOH = "noh"
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/cli/utils.py

@@ -29,25 +29,6 @@ def _iris_investigate_helptext():
     """
 
 
-def _phisheye_helptext():
-    return """
-        Returns domain results for the specified term for today or the specified number of days_back. Terms must be setup for monitoring via the web interface: https://research.domaintools.com/phisheye. \n
-
-        NOTE: Properties of a domain are only provided if we have been able to obtain them. \n
-        Many domains will have incomplete data because that information isn't available in their Whois records,
-        or they don't have DNS results for a name server or IP address.
-    """
-
-
-def _phisheye_termlist():
-    return """
-        Provides a list of terms that are set up for this account. This call is not charged against your API usage limit.
-
-        NOTE: The terms must be configured in the PhishEye web interface: https://research.domaintools.com/phisheye. \n
-        There is no API call to set up the terms.
-    """
-
-
 _command_help_mapper = {
     c.ACCOUNT_INFORMATION: "Provides a snapshot of your accounts current API usage.",
     c.AVAILABLE_API_CALLS: "Provides a list of api calls that you can use based on your account information.",
@@ -71,8 +52,6 @@ _command_help_mapper = {
     c.REVERSE_WHOIS: "Provides list of one or more terms to search for in the Whois record, as a Python list or separated with the pipe character ( | ).",
     c.WHOIS: "Pass in a domain name or an IP address to perform a whois lookup.",
     c.WHOIS_HISTORY: "Retrieve historical Whois records of a given domain name.",
-    c.PHISHEYE: _phisheye_helptext(),
-    c.PHISHEYE_TERM_LIST: _phisheye_termlist(),
     c.IRIS: "Performs a search for the provided search terms ANDed together, returning the pivot engine row data for the resulting domains.",
     c.RISK: "Returns back the risk score for a given domain.",
     c.RISK_EVIDENCE: "Returns back the detailed risk evidence associated with a given domain.",
@@ -84,6 +63,7 @@ _command_help_mapper = {
     c.IRIS_DETECT_IGNORED_DOMAINS: "Returns back a list of ignored domains in Iris Detect based on the provided filters.",
     c.FEEDS_NAD: "Returns back newly active domains feed.",
     c.FEEDS_NOD: "Returns back newly observed domains feed.",
+    c.FEEDS_NOH: "Returns back newly observed hosts feed.",
     c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
     c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
 }

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/constants.py

@@ -28,6 +28,8 @@ FEEDS_PRODUCTS_LIST = [
     "newly-active-domains-feed-(s3)",
     "newly-observed-domains-feed-(api)",
     "newly-observed-domains-feed-(s3)",
+    "newly-observed-hosts-feed-(api)",
+    "newly-observed-hosts-feed-(s3)",
     "domain-registration-data-access-protocol-feed-(api)",
     "domain-registration-data-access-protocol-feed-(s3)",
     "real-time-domain-discovery-feed-(api)",

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools/utils.py

@@ -185,4 +185,5 @@ def validate_feeds_parameters(params):
         raise ValueError(f"{format} format is not available in {Endpoint.DOWNLOAD.value} API.")
 
     if endpoint == Endpoint.DOWNLOAD.value and params.get("header_authentication", True):
-        raise ValueError(f"{Endpoint.DOWNLOAD.value} API does not support header authentication. Provide api_key in the parameter")
+        # For download endpoint, header_authentication will be False by default
+        params["header_authentication"] = False

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0/domaintools_api_test_upload.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: domaintools_api_test_upload
-Version: 2.2.2
+Version: 2.4.0
 Summary: DomainTools Official Python API
 Author-email: DomainTools <integrations@domaintools.com>
 License: The MIT License (MIT)
@@ -46,6 +46,7 @@ Requires-Dist: typer
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: mock; extra == "test"
+Dynamic: license-file
 
 ![domaintools](https://github.com/DomainTools/python_api/raw/main/artwork/logo.png)
 ===================
@@ -275,7 +276,7 @@ Custom parameters aside from the common `GET` Request parameters:
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(endpoint="feed", **kwargs)
     ```
-- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so you need to set this to `False` when calling `download` API endpoints.
+- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so this will be defaulted to `False` even without explicitly setting it.
     ```python
     api = API(USERNAME, KEY, always_sign_api_key=False)
     api.nod(header_authentication=False, **kwargs)

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/domaintools_api_test_upload.egg-info/SOURCES.txt

@@ -25,7 +25,6 @@ domaintools/cli/commands/domains.py
 domaintools/cli/commands/feeds.py
 domaintools/cli/commands/ips.py
 domaintools/cli/commands/iris.py
-domaintools/cli/commands/phisheye.py
 domaintools_api_test_upload.egg-info/PKG-INFO
 domaintools_api_test_upload.egg-info/SOURCES.txt
 domaintools_api_test_upload.egg-info/dependency_links.txt

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/tests/test_api.py

@@ -322,16 +322,6 @@ def test_dict_like_behaviour():
         assert isinstance(whois_google.pop("whois", {}), dict)
 
 
-@vcr.use_cassette
-def test_list_like_behaviour():
-    with api.phisheye("google") as data:
-        data.insert(0, {"domain": "woot", "tld": "com"})
-        assert data["term"] == "google"
-        for result in data:
-            assert result["domain"]
-            assert result["tld"]
-
-
 @vcr.use_cassette
 def test_exception_handling():
     exception = None
@@ -386,24 +376,6 @@ def test_formats():
         assert "\n" in str(data.as_list())
 
 
-@vcr.use_cassette
-def test_phisheye():
-    with api.phisheye("google") as data:
-        assert data["term"] == "google"
-        for result in data:
-            assert result["domain"]
-            assert result["tld"]
-
-
-@vcr.use_cassette
-def test_phisheye_term_list():
-    with api.phisheye_term_list() as data:
-        assert data
-        for term in data:
-            assert "term" in term
-            assert type(term["active"]) == bool
-
-
 @vcr.use_cassette
 def test_iris():
     with pytest.raises(ValueError):
@@ -545,6 +517,22 @@ def test_newly_observed_domains_feed():
             assert "domain" in feed_result.keys()
 
 
+@vcr.use_cassette
+def test_newly_observed_hosts_feed():
+    results = feeds_api.noh(after="-60", header_authentication=False)
+    for response in results.response():
+        assert results.status == 200
+
+        rows = response.strip().split("\n")
+        assert response is not None
+        assert len(rows) >= 1
+
+        for row in rows:
+            feed_result = json.loads(row)
+            assert "timestamp" in feed_result.keys()
+            assert "domain" in feed_result.keys()
+
+
 @vcr.use_cassette
 def test_newly_observed_domains_feed_pagination():
     results = feeds_api.nod(sessionID="integrations-testing", after="2025-01-16T10:20:00Z")
@@ -646,11 +634,19 @@ def test_verify_response_is_a_generator():
 
 
 @vcr.use_cassette
-def test_feeds_endpoint_should_raise_error_if_download_api_using_header_auth():
-    with pytest.raises(ValueError) as excinfo:
-        feeds_api.domaindiscovery(after="-60", endpoint="download")
+def test_feeds_endpoint_should_non_header_auth_be_the_default():
+    results = feeds_api.domaindiscovery(after="-60", endpoint="download")
+    for response in results.response():
+        assert results.status == 200
+
+        rows = response.strip().split("\n")
+        assert response is not None
+        assert len(rows) >= 1
 
-    assert str(excinfo.value) == "download API does not support header authentication. Provide api_key in the parameter"
+        for row in rows:
+            feed_result = json.loads(row)
+            assert "download_name" in feed_result["response"].keys()
+            assert "files" in feed_result["response"].keys()
 
 
 @vcr.use_cassette

{domaintools_api_test_upload-2.2.2 → domaintools_api_test_upload-2.4.0}/tests/test_utils.py

@@ -119,13 +119,15 @@ def test_get_pivots():
     assert pivots == [["IP ADDRESS", ("199.30.228.112", 4)], ["IP ASN", (17318, 111)], ["IP ISP", ("DomainTools LLC", 222)]]
 
 
-def test_validate_feeds_parameters_should_raise_error_if_download_api_using_header_auth(test_feeds_params):
+def test_validate_feeds_parameters_update_header_auth_to_False(test_feeds_params):
     test_feeds_params["output_format"] = "jsonl"
 
-    with pytest.raises(ValueError) as excinfo:
-        utils.validate_feeds_parameters(test_feeds_params)
+    assert test_feeds_params.get("header_authentication", True)  # header_authentication is True whether existing or not
+    assert test_feeds_params["endpoint"] == "download"
+
+    utils.validate_feeds_parameters(test_feeds_params)
 
-    assert str(excinfo.value) == "download API does not support header authentication. Provide api_key in the parameter"
+    assert not test_feeds_params["header_authentication"]  # header_authentication will be defaulted to False if endpoint is download
 
 
 def test_validate_feeds_parameters_should_raise_error_if_no_required_params(test_feeds_params):

domaintools_api_test_upload-2.2.2/VERSION

@@ -1 +0,0 @@
-2.2.2

domaintools_api_test_upload-2.2.2/domaintools/cli/commands/phisheye.py

@@ -1,100 +0,0 @@
-import sys
-import typer
-
-
-from domaintools.cli.main import dt_cli
-from domaintools.cli.api import DTCLICommand
-from domaintools.cli.utils import get_cli_helptext_by_name
-from domaintools.cli import constants as c
-
-
-@dt_cli.command(
-    name=c.PHISHEYE,
-    help=get_cli_helptext_by_name(command_name=c.PHISHEYE),
-)
-def phisheye(
-    ctx: typer.Context,
-    query: str = typer.Option(..., "-q", "--query", help="The domain to query."),
-    days_back: str = typer.Option(
-        None,
-        "--days-back",
-        help="Use this parameter in exceptional circumstances where you need to search domains registered up to six days prior to the current date. Set the value to an integer in the range of 1-6.",
-    ),
-    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
-    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
-    creds_file: str = typer.Option(
-        "~/.dtapi",
-        "-c",
-        "--credfile",
-        help="Optional file with API username and API key, one per line.",
-    ),
-    rate_limit: bool = typer.Option(
-        False,
-        "-l",
-        "--rate-limit",
-        help="Rate limit API calls against the API based on per minute limits.",
-    ),
-    format: str = typer.Option(
-        "json",
-        "-f",
-        "--format",
-        help="Output format in {'list', 'json', 'xml', 'html'}",
-        callback=DTCLICommand.validate_format_input,
-    ),
-    out_file: typer.FileTextWrite = typer.Option(
-        sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"
-    ),
-    no_verify_ssl: bool = typer.Option(
-        False,
-        "--no-verify-ssl",
-        help="Skip verification of SSL certificate when making HTTPs API calls",
-    ),
-):
-    DTCLICommand.run(name=c.PHISHEYE, params=ctx.params)
-
-
-@dt_cli.command(
-    name=c.PHISHEYE_TERM_LIST,
-    help=get_cli_helptext_by_name(command_name=c.PHISHEYE_TERM_LIST),
-)
-def phisheye_termlist(
-    ctx: typer.Context,
-    include_inactive: bool = typer.Option(
-        False,
-        "--include-inactive",
-        help="Use this parameter to display terms that may have been inactivated in users' lists.",
-    ),
-    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
-    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
-    creds_file: str = typer.Option(
-        "~/.dtapi",
-        "-c",
-        "--credfile",
-        help="Optional file with API username and API key, one per line.",
-    ),
-    rate_limit: bool = typer.Option(
-        False,
-        "-l",
-        "--rate-limit",
-        help="Rate limit API calls against the API based on per minute limits.",
-    ),
-    format: str = typer.Option(
-        "json",
-        "-f",
-        "--format",
-        help="Output format in {'list', 'json', 'xml', 'html'}",
-        callback=DTCLICommand.validate_format_input,
-    ),
-    out_file: typer.FileTextWrite = typer.Option(
-        sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"
-    ),
-    no_verify_ssl: bool = typer.Option(
-        False,
-        "--no-verify-ssl",
-        help="Skip verification of SSL certificate when making HTTPs API calls",
-    ),
-):
-    DTCLICommand.run(name=c.PHISHEYE_TERM_LIST, params=ctx.params)
-
-
-__all__ = ["phisheye", "phisheye_termlist"]