domaintools-api-test-upload 2.4.0__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {domaintools_api_test_upload-2.4.0/domaintools_api_test_upload.egg-info → domaintools_api_test_upload-2.5.0}/PKG-INFO +42 -2
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/README.md +41 -1
- domaintools_api_test_upload-2.5.0/VERSION +1 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/_version.py +1 -1
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/api.py +149 -8
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/api.py +1 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/feeds.py +160 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/constants.py +2 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/utils.py +2 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/constants.py +4 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0/domaintools_api_test_upload.egg-info}/PKG-INFO +42 -2
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_api_test_upload.egg-info/requires.txt +1 -1
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/pyproject.toml +5 -1
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_api.py +51 -1
- domaintools_api_test_upload-2.4.0/VERSION +0 -1
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/LICENSE +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/__init__.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/base_results.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/__init__.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/__init__.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/accounts.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/detects.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/domains.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/ips.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/commands/iris.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/main.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/exceptions.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/filters.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/results.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/utils.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_api_test_upload.egg-info/SOURCES.txt +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_api_test_upload.egg-info/dependency_links.txt +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_api_test_upload.egg-info/entry_points.txt +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_api_test_upload.egg-info/top_level.txt +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools_async/__init__.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/setup.cfg +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/setup.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_async.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_cli.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_filters.py +0 -0
- {domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: domaintools_api_test_upload
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: DomainTools Official Python API
|
|
5
5
|
Author-email: DomainTools <integrations@domaintools.com>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -40,7 +40,7 @@ Classifier: Topic :: Utilities
|
|
|
40
40
|
Requires-Python: >=3.9
|
|
41
41
|
Description-Content-Type: text/markdown
|
|
42
42
|
License-File: LICENSE
|
|
43
|
-
Requires-Dist: httpx
|
|
43
|
+
Requires-Dist: httpx==0.28.1
|
|
44
44
|
Requires-Dist: rich
|
|
45
45
|
Requires-Dist: typer
|
|
46
46
|
Provides-Extra: test
|
|
@@ -321,3 +321,43 @@ results = api.nod(sessionID="my-session-id", after=-7200)
|
|
|
321
321
|
for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
|
|
322
322
|
# do things to partial_result
|
|
323
323
|
```
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
Running E2E Tests Locally
|
|
327
|
+
===================
|
|
328
|
+
For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
|
|
329
|
+
To add more e2e tests, put these in the `../tests/e2e` folder.
|
|
330
|
+
|
|
331
|
+
## Preparation
|
|
332
|
+
- Create virtual environment.
|
|
333
|
+
```bash
|
|
334
|
+
python3 -m venv venv
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
- Activate virtual environment
|
|
338
|
+
```bash
|
|
339
|
+
source venv/bin/activate
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
- Install dependencies.
|
|
343
|
+
```bash
|
|
344
|
+
pip install -r requirements/development.txt
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
- From the python_api project root directory, install the package.
|
|
348
|
+
```bash
|
|
349
|
+
pip install -e .
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
- Export api credentials to use.
|
|
353
|
+
```bash
|
|
354
|
+
export TEST_USER=<user-key>
|
|
355
|
+
export TEST_KEY=<api-key>
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
## Run the end-to-end test script
|
|
359
|
+
- Before running the test, be sure that docker is running.
|
|
360
|
+
- Execute the e2e test script .
|
|
361
|
+
```bash
|
|
362
|
+
sh tests/e2e/scripts/test_e2e_runner.sh
|
|
363
|
+
```
|
|
@@ -270,4 +270,44 @@ results = api.nod(sessionID="my-session-id", after=-7200)
|
|
|
270
270
|
|
|
271
271
|
for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
|
|
272
272
|
# do things to partial_result
|
|
273
|
-
```
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
Running E2E Tests Locally
|
|
277
|
+
===================
|
|
278
|
+
For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
|
|
279
|
+
To add more e2e tests, put these in the `../tests/e2e` folder.
|
|
280
|
+
|
|
281
|
+
## Preparation
|
|
282
|
+
- Create virtual environment.
|
|
283
|
+
```bash
|
|
284
|
+
python3 -m venv venv
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
- Activate virtual environment
|
|
288
|
+
```bash
|
|
289
|
+
source venv/bin/activate
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
- Install dependencies.
|
|
293
|
+
```bash
|
|
294
|
+
pip install -r requirements/development.txt
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
- From the python_api project root directory, install the package.
|
|
298
|
+
```bash
|
|
299
|
+
pip install -e .
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
- Export api credentials to use.
|
|
303
|
+
```bash
|
|
304
|
+
export TEST_USER=<user-key>
|
|
305
|
+
export TEST_KEY=<api-key>
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
## Run the end-to-end test script
|
|
309
|
+
- Before running the test, be sure that docker is running.
|
|
310
|
+
- Execute the e2e test script .
|
|
311
|
+
```bash
|
|
312
|
+
sh tests/e2e/scripts/test_e2e_runner.sh
|
|
313
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.5.0
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from datetime import datetime, timedelta, timezone
|
|
2
|
-
from hashlib import sha1, sha256
|
|
2
|
+
from hashlib import sha1, sha256
|
|
3
3
|
from hmac import new as hmac
|
|
4
|
+
from typing import Union
|
|
4
5
|
|
|
5
6
|
import re
|
|
7
|
+
import ssl
|
|
6
8
|
|
|
7
9
|
from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat
|
|
8
10
|
from domaintools._version import current as version
|
|
@@ -24,7 +26,7 @@ from domaintools.filters import (
|
|
|
24
26
|
from domaintools.utils import validate_feeds_parameters
|
|
25
27
|
|
|
26
28
|
|
|
27
|
-
AVAILABLE_KEY_SIGN_HASHES = ["sha1", "sha256"
|
|
29
|
+
AVAILABLE_KEY_SIGN_HASHES = ["sha1", "sha256"]
|
|
28
30
|
|
|
29
31
|
|
|
30
32
|
def delimited(items, character="|"):
|
|
@@ -76,7 +78,7 @@ class API(object):
|
|
|
76
78
|
self.username = username
|
|
77
79
|
self.key = key
|
|
78
80
|
self.https = https
|
|
79
|
-
self.verify_ssl = verify_ssl
|
|
81
|
+
self.verify_ssl = self._get_ssl_default_context(verify_ssl)
|
|
80
82
|
self.rate_limit = rate_limit
|
|
81
83
|
self.proxy_url = proxy_url
|
|
82
84
|
self.extra_request_params = {}
|
|
@@ -92,6 +94,9 @@ class API(object):
|
|
|
92
94
|
if proxy_url and not isinstance(proxy_url, str):
|
|
93
95
|
raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
|
|
94
96
|
|
|
97
|
+
def _get_ssl_default_context(self, verify_ssl: Union[str, bool]):
|
|
98
|
+
return ssl.create_default_context(cafile=verify_ssl) if isinstance(verify_ssl, str) else verify_ssl
|
|
99
|
+
|
|
95
100
|
def _build_api_url(self, api_url=None, api_port=None):
|
|
96
101
|
"""Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`"""
|
|
97
102
|
rest_api_url = "https://api.domaintools.com"
|
|
@@ -1033,7 +1038,21 @@ class API(object):
|
|
|
1033
1038
|
)
|
|
1034
1039
|
|
|
1035
1040
|
def nod(self, **kwargs) -> FeedsResults:
|
|
1036
|
-
"""Returns back list of the newly observed domains feed
|
|
1041
|
+
"""Returns back list of the newly observed domains feed.
|
|
1042
|
+
Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.
|
|
1043
|
+
|
|
1044
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1045
|
+
|
|
1046
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1047
|
+
|
|
1048
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1049
|
+
|
|
1050
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1051
|
+
|
|
1052
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1053
|
+
|
|
1054
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1055
|
+
"""
|
|
1037
1056
|
validate_feeds_parameters(kwargs)
|
|
1038
1057
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1039
1058
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
|
|
@@ -1050,7 +1069,22 @@ class API(object):
|
|
|
1050
1069
|
)
|
|
1051
1070
|
|
|
1052
1071
|
def nad(self, **kwargs) -> FeedsResults:
|
|
1053
|
-
"""Returns back list of the newly active domains feed
|
|
1072
|
+
"""Returns back list of the newly active domains feed. Contains domains that have been observed after having not been seen for at least 10 days in passive DNS.
|
|
1073
|
+
Apex-level domains (e.g. example.com but not www.example.com) that we observe based on the latest lifecycle of the domain. A domain may be seen either for the first time ever, or again after at least 10 days of inactivity (no observed resolutions in DNS).
|
|
1074
|
+
Populated with our global passive DNS (pDNS) sensor network.
|
|
1075
|
+
|
|
1076
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1077
|
+
|
|
1078
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1079
|
+
|
|
1080
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1081
|
+
|
|
1082
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1083
|
+
|
|
1084
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1085
|
+
|
|
1086
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1087
|
+
"""
|
|
1054
1088
|
validate_feeds_parameters(kwargs)
|
|
1055
1089
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1056
1090
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
@@ -1067,7 +1101,22 @@ class API(object):
|
|
|
1067
1101
|
)
|
|
1068
1102
|
|
|
1069
1103
|
def domainrdap(self, **kwargs) -> FeedsResults:
|
|
1070
|
-
"""Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP)
|
|
1104
|
+
"""Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).
|
|
1105
|
+
Compliments the 5-Minute WHOIS Feed as registries and registrars switch from Whois to RDAP.
|
|
1106
|
+
Contains parsed and raw RDAP-format domain registration data, emitted as soon as they are collected and parsed into a normalized structure.
|
|
1107
|
+
|
|
1108
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1109
|
+
|
|
1110
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1111
|
+
|
|
1112
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1113
|
+
|
|
1114
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1115
|
+
|
|
1116
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1117
|
+
|
|
1118
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1119
|
+
"""
|
|
1071
1120
|
validate_feeds_parameters(kwargs)
|
|
1072
1121
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1073
1122
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
@@ -1081,7 +1130,22 @@ class API(object):
|
|
|
1081
1130
|
)
|
|
1082
1131
|
|
|
1083
1132
|
def domaindiscovery(self, **kwargs) -> FeedsResults:
|
|
1084
|
-
"""Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties"
|
|
1133
|
+
"""Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties".
|
|
1134
|
+
Contains domains that are newly-discovered by Domain Tools in both passive and active DNS sources, emitted as soon as they are first observed.
|
|
1135
|
+
New domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.
|
|
1136
|
+
|
|
1137
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1138
|
+
|
|
1139
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1140
|
+
|
|
1141
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1142
|
+
|
|
1143
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1144
|
+
|
|
1145
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1146
|
+
|
|
1147
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1148
|
+
"""
|
|
1085
1149
|
validate_feeds_parameters(kwargs)
|
|
1086
1150
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1087
1151
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
@@ -1098,7 +1162,22 @@ class API(object):
|
|
|
1098
1162
|
)
|
|
1099
1163
|
|
|
1100
1164
|
def noh(self, **kwargs) -> FeedsResults:
|
|
1101
|
-
"""Returns back list of the newly observed hostnames feed
|
|
1165
|
+
"""Returns back list of the newly observed hostnames feed.
|
|
1166
|
+
Contains fully qualified domain names (i.e. host names) that have never been seen before in passive DNS, emitted as soon as they are first observed.
|
|
1167
|
+
Hostname resolutions that we observe for the first time with our global DNS sensor network.
|
|
1168
|
+
|
|
1169
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1170
|
+
|
|
1171
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1172
|
+
|
|
1173
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1174
|
+
|
|
1175
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1176
|
+
|
|
1177
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1178
|
+
|
|
1179
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1180
|
+
"""
|
|
1102
1181
|
validate_feeds_parameters(kwargs)
|
|
1103
1182
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1104
1183
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
@@ -1113,3 +1192,65 @@ class API(object):
|
|
|
1113
1192
|
cls=FeedsResults,
|
|
1114
1193
|
**kwargs,
|
|
1115
1194
|
)
|
|
1195
|
+
|
|
1196
|
+
def realtime_domain_risk(self, **kwargs) -> FeedsResults:
|
|
1197
|
+
"""Returns back list of the realtime domain risk feed.
|
|
1198
|
+
Contains realtime domain risk information for apex-level domains, regardless of observed traffic.
|
|
1199
|
+
|
|
1200
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1201
|
+
|
|
1202
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1203
|
+
|
|
1204
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1205
|
+
|
|
1206
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1207
|
+
|
|
1208
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1209
|
+
|
|
1210
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1211
|
+
"""
|
|
1212
|
+
validate_feeds_parameters(kwargs)
|
|
1213
|
+
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1214
|
+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1215
|
+
if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
|
|
1216
|
+
# headers param is allowed only in Feed API and CSV format
|
|
1217
|
+
kwargs.pop("headers", None)
|
|
1218
|
+
|
|
1219
|
+
return self._results(
|
|
1220
|
+
f"domain-risk-({source})",
|
|
1221
|
+
f"v1/{endpoint}/domainrisk/",
|
|
1222
|
+
response_path=(),
|
|
1223
|
+
cls=FeedsResults,
|
|
1224
|
+
**kwargs,
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
def domainhotlist(self, **kwargs) -> FeedsResults:
|
|
1228
|
+
"""Returns back list of domain hotlist feed.
|
|
1229
|
+
Contains high-risk, apex-level domains that are observed by DomainTools' global sensor network to be active within 24 hours.
|
|
1230
|
+
|
|
1231
|
+
domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
|
|
1232
|
+
|
|
1233
|
+
before: str: Filter for records before the given time value inclusive or time offset relative to now
|
|
1234
|
+
|
|
1235
|
+
after: str: Filter for records after the given time value inclusive or time offset relative to now
|
|
1236
|
+
|
|
1237
|
+
headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
|
|
1238
|
+
|
|
1239
|
+
sessionID: str: A custom string to distinguish between different sessions
|
|
1240
|
+
|
|
1241
|
+
top: int: Limit the number of results to the top N, where N is the value of this parameter.
|
|
1242
|
+
"""
|
|
1243
|
+
validate_feeds_parameters(kwargs)
|
|
1244
|
+
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1245
|
+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1246
|
+
if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
|
|
1247
|
+
# headers param is allowed only in Feed API and CSV format
|
|
1248
|
+
kwargs.pop("headers", None)
|
|
1249
|
+
|
|
1250
|
+
return self._results(
|
|
1251
|
+
f"domain-hotlist-feed-({source})",
|
|
1252
|
+
f"v1/{endpoint}/domainhotlist/",
|
|
1253
|
+
response_path=(),
|
|
1254
|
+
cls=FeedsResults,
|
|
1255
|
+
**kwargs,
|
|
1256
|
+
)
|
|
@@ -394,3 +394,163 @@ def feeds_noh(
|
|
|
394
394
|
),
|
|
395
395
|
):
|
|
396
396
|
DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
@dt_cli.command(
|
|
400
|
+
name=c.FEEDS_DOMAINHOTLIST,
|
|
401
|
+
help=get_cli_helptext_by_name(command_name=c.FEEDS_DOMAINHOTLIST),
|
|
402
|
+
)
|
|
403
|
+
def feeds_domainhotlist(
|
|
404
|
+
ctx: typer.Context,
|
|
405
|
+
user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
|
|
406
|
+
key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
|
|
407
|
+
creds_file: str = typer.Option(
|
|
408
|
+
"~/.dtapi",
|
|
409
|
+
"-c",
|
|
410
|
+
"--credfile",
|
|
411
|
+
help="Optional file with API username and API key, one per line.",
|
|
412
|
+
),
|
|
413
|
+
no_verify_ssl: bool = typer.Option(
|
|
414
|
+
False,
|
|
415
|
+
"--no-verify-ssl",
|
|
416
|
+
help="Skip verification of SSL certificate when making HTTPs API calls",
|
|
417
|
+
),
|
|
418
|
+
no_sign_api_key: bool = typer.Option(
|
|
419
|
+
False,
|
|
420
|
+
"--no-sign-api-key",
|
|
421
|
+
help="Skip signing of api key",
|
|
422
|
+
),
|
|
423
|
+
header_authentication: bool = typer.Option(
|
|
424
|
+
True,
|
|
425
|
+
"--no-header-auth",
|
|
426
|
+
help="Don't use header authentication",
|
|
427
|
+
),
|
|
428
|
+
output_format: str = typer.Option(
|
|
429
|
+
"jsonl",
|
|
430
|
+
"-f",
|
|
431
|
+
"--format",
|
|
432
|
+
help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
|
|
433
|
+
callback=DTCLICommand.validate_feeds_format_input,
|
|
434
|
+
),
|
|
435
|
+
endpoint: str = typer.Option(
|
|
436
|
+
Endpoint.FEED.value,
|
|
437
|
+
"-e",
|
|
438
|
+
"--endpoint",
|
|
439
|
+
help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
|
|
440
|
+
callback=DTCLICommand.validate_endpoint_input,
|
|
441
|
+
),
|
|
442
|
+
sessionID: str = typer.Option(
|
|
443
|
+
None,
|
|
444
|
+
"--session-id",
|
|
445
|
+
help="Unique identifier for the session",
|
|
446
|
+
),
|
|
447
|
+
after: str = typer.Option(
|
|
448
|
+
None,
|
|
449
|
+
"--after",
|
|
450
|
+
help="Start of the time window, relative to the current time in seconds, for which data will be provided",
|
|
451
|
+
callback=DTCLICommand.validate_after_or_before_input,
|
|
452
|
+
),
|
|
453
|
+
before: str = typer.Option(
|
|
454
|
+
None,
|
|
455
|
+
"--before",
|
|
456
|
+
help="The end of the query window in seconds, relative to the current time, inclusive",
|
|
457
|
+
callback=DTCLICommand.validate_after_or_before_input,
|
|
458
|
+
),
|
|
459
|
+
domain: str = typer.Option(
|
|
460
|
+
None,
|
|
461
|
+
"-d",
|
|
462
|
+
"--domain",
|
|
463
|
+
help="A string value used to filter feed results",
|
|
464
|
+
),
|
|
465
|
+
headers: bool = typer.Option(
|
|
466
|
+
False,
|
|
467
|
+
"--headers",
|
|
468
|
+
help="Adds a header to the first line of response when text/csv is set in header parameters",
|
|
469
|
+
),
|
|
470
|
+
top: str = typer.Option(
|
|
471
|
+
None,
|
|
472
|
+
"--top",
|
|
473
|
+
help="Number of results to return in the response payload. This is ignored in download endpoint",
|
|
474
|
+
),
|
|
475
|
+
):
|
|
476
|
+
DTCLICommand.run(name=c.FEEDS_DOMAINHOTLIST, params=ctx.params)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
@dt_cli.command(
|
|
480
|
+
name=c.FEEDS_REALTIME_DOMAIN_RISK,
|
|
481
|
+
help=get_cli_helptext_by_name(command_name=c.FEEDS_REALTIME_DOMAIN_RISK),
|
|
482
|
+
)
|
|
483
|
+
def feeds_realtime_domain_risk(
|
|
484
|
+
ctx: typer.Context,
|
|
485
|
+
user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
|
|
486
|
+
key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
|
|
487
|
+
creds_file: str = typer.Option(
|
|
488
|
+
"~/.dtapi",
|
|
489
|
+
"-c",
|
|
490
|
+
"--credfile",
|
|
491
|
+
help="Optional file with API username and API key, one per line.",
|
|
492
|
+
),
|
|
493
|
+
no_verify_ssl: bool = typer.Option(
|
|
494
|
+
False,
|
|
495
|
+
"--no-verify-ssl",
|
|
496
|
+
help="Skip verification of SSL certificate when making HTTPs API calls",
|
|
497
|
+
),
|
|
498
|
+
no_sign_api_key: bool = typer.Option(
|
|
499
|
+
False,
|
|
500
|
+
"--no-sign-api-key",
|
|
501
|
+
help="Skip signing of api key",
|
|
502
|
+
),
|
|
503
|
+
header_authentication: bool = typer.Option(
|
|
504
|
+
True,
|
|
505
|
+
"--no-header-auth",
|
|
506
|
+
help="Don't use header authentication",
|
|
507
|
+
),
|
|
508
|
+
output_format: str = typer.Option(
|
|
509
|
+
"jsonl",
|
|
510
|
+
"-f",
|
|
511
|
+
"--format",
|
|
512
|
+
help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
|
|
513
|
+
callback=DTCLICommand.validate_feeds_format_input,
|
|
514
|
+
),
|
|
515
|
+
endpoint: str = typer.Option(
|
|
516
|
+
Endpoint.FEED.value,
|
|
517
|
+
"-e",
|
|
518
|
+
"--endpoint",
|
|
519
|
+
help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
|
|
520
|
+
callback=DTCLICommand.validate_endpoint_input,
|
|
521
|
+
),
|
|
522
|
+
sessionID: str = typer.Option(
|
|
523
|
+
None,
|
|
524
|
+
"--session-id",
|
|
525
|
+
help="Unique identifier for the session",
|
|
526
|
+
),
|
|
527
|
+
after: str = typer.Option(
|
|
528
|
+
None,
|
|
529
|
+
"--after",
|
|
530
|
+
help="Start of the time window, relative to the current time in seconds, for which data will be provided",
|
|
531
|
+
callback=DTCLICommand.validate_after_or_before_input,
|
|
532
|
+
),
|
|
533
|
+
before: str = typer.Option(
|
|
534
|
+
None,
|
|
535
|
+
"--before",
|
|
536
|
+
help="The end of the query window in seconds, relative to the current time, inclusive",
|
|
537
|
+
callback=DTCLICommand.validate_after_or_before_input,
|
|
538
|
+
),
|
|
539
|
+
domain: str = typer.Option(
|
|
540
|
+
None,
|
|
541
|
+
"-d",
|
|
542
|
+
"--domain",
|
|
543
|
+
help="A string value used to filter feed results",
|
|
544
|
+
),
|
|
545
|
+
headers: bool = typer.Option(
|
|
546
|
+
False,
|
|
547
|
+
"--headers",
|
|
548
|
+
help="Adds a header to the first line of response when text/csv is set in header parameters",
|
|
549
|
+
),
|
|
550
|
+
top: str = typer.Option(
|
|
551
|
+
None,
|
|
552
|
+
"--top",
|
|
553
|
+
help="Number of results to return in the response payload. This is ignored in download endpoint",
|
|
554
|
+
),
|
|
555
|
+
):
|
|
556
|
+
DTCLICommand.run(name=c.FEEDS_REALTIME_DOMAIN_RISK, params=ctx.params)
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/constants.py
RENAMED
|
@@ -43,5 +43,7 @@ IRIS_DETECT_IGNORED_DOMAINS = "iris_detect_ignored_domains"
|
|
|
43
43
|
FEEDS_NAD = "nad"
|
|
44
44
|
FEEDS_NOD = "nod"
|
|
45
45
|
FEEDS_NOH = "noh"
|
|
46
|
+
FEEDS_DOMAINHOTLIST = "domainhotlist"
|
|
46
47
|
FEEDS_DOMAINRDAP = "domainrdap"
|
|
47
48
|
FEEDS_DOMAINDISCOVERY = "domaindiscovery"
|
|
49
|
+
FEEDS_REALTIME_DOMAIN_RISK = "realtime_domain_risk"
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/utils.py
RENAMED
|
@@ -64,8 +64,10 @@ _command_help_mapper = {
|
|
|
64
64
|
c.FEEDS_NAD: "Returns back newly active domains feed.",
|
|
65
65
|
c.FEEDS_NOD: "Returns back newly observed domains feed.",
|
|
66
66
|
c.FEEDS_NOH: "Returns back newly observed hosts feed.",
|
|
67
|
+
c.FEEDS_DOMAINHOTLIST: "Returns domaint hotlist feed.",
|
|
67
68
|
c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
|
|
68
69
|
c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
|
|
70
|
+
c.FEEDS_REALTIME_DOMAIN_RISK: "Returns realtime domain risk information for apex-level domains, regardless of observed traffic.",
|
|
69
71
|
}
|
|
70
72
|
|
|
71
73
|
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/constants.py
RENAMED
|
@@ -30,8 +30,12 @@ FEEDS_PRODUCTS_LIST = [
|
|
|
30
30
|
"newly-observed-domains-feed-(s3)",
|
|
31
31
|
"newly-observed-hosts-feed-(api)",
|
|
32
32
|
"newly-observed-hosts-feed-(s3)",
|
|
33
|
+
"domain-hotlist-feed-(api)",
|
|
34
|
+
"domain-hotlist-feed-(s3)",
|
|
33
35
|
"domain-registration-data-access-protocol-feed-(api)",
|
|
34
36
|
"domain-registration-data-access-protocol-feed-(s3)",
|
|
37
|
+
"domain-risk-feed-(api)",
|
|
38
|
+
"domain-risk-feed-(s3)",
|
|
35
39
|
"real-time-domain-discovery-feed-(api)",
|
|
36
40
|
"real-time-domain-discovery-feed-(s3)",
|
|
37
41
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: domaintools_api_test_upload
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: DomainTools Official Python API
|
|
5
5
|
Author-email: DomainTools <integrations@domaintools.com>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -40,7 +40,7 @@ Classifier: Topic :: Utilities
|
|
|
40
40
|
Requires-Python: >=3.9
|
|
41
41
|
Description-Content-Type: text/markdown
|
|
42
42
|
License-File: LICENSE
|
|
43
|
-
Requires-Dist: httpx
|
|
43
|
+
Requires-Dist: httpx==0.28.1
|
|
44
44
|
Requires-Dist: rich
|
|
45
45
|
Requires-Dist: typer
|
|
46
46
|
Provides-Extra: test
|
|
@@ -321,3 +321,43 @@ results = api.nod(sessionID="my-session-id", after=-7200)
|
|
|
321
321
|
for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
|
|
322
322
|
# do things to partial_result
|
|
323
323
|
```
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
Running E2E Tests Locally
|
|
327
|
+
===================
|
|
328
|
+
For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
|
|
329
|
+
To add more e2e tests, put these in the `../tests/e2e` folder.
|
|
330
|
+
|
|
331
|
+
## Preparation
|
|
332
|
+
- Create virtual environment.
|
|
333
|
+
```bash
|
|
334
|
+
python3 -m venv venv
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
- Activate virtual environment
|
|
338
|
+
```bash
|
|
339
|
+
source venv/bin/activate
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
- Install dependencies.
|
|
343
|
+
```bash
|
|
344
|
+
pip install -r requirements/development.txt
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
- From the python_api project root directory, install the package.
|
|
348
|
+
```bash
|
|
349
|
+
pip install -e .
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
- Export api credentials to use.
|
|
353
|
+
```bash
|
|
354
|
+
export TEST_USER=<user-key>
|
|
355
|
+
export TEST_KEY=<api-key>
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
## Run the end-to-end test script
|
|
359
|
+
- Before running the test, be sure that docker is running.
|
|
360
|
+
- Execute the e2e test script .
|
|
361
|
+
```bash
|
|
362
|
+
sh tests/e2e/scripts/test_e2e_runner.sh
|
|
363
|
+
```
|
|
@@ -4,7 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "domaintools_api_test_upload"
|
|
7
|
-
dependencies = [
|
|
7
|
+
dependencies = [
|
|
8
|
+
"httpx==0.28.1",
|
|
9
|
+
"rich",
|
|
10
|
+
"typer"
|
|
11
|
+
]
|
|
8
12
|
requires-python = ">= 3.9"
|
|
9
13
|
authors = [{ name = "DomainTools", email = "integrations@domaintools.com" }]
|
|
10
14
|
description = "DomainTools Official Python API"
|
|
@@ -342,11 +342,20 @@ def test_exception_handling():
|
|
|
342
342
|
API("notauser", "notakey").domain_search("amazon").data()
|
|
343
343
|
with pytest.raises(
|
|
344
344
|
ValueError,
|
|
345
|
-
match=r"Invalid value 'notahash' for 'key_sign_hash'. Values available are sha1,sha256
|
|
345
|
+
match=r"Invalid value 'notahash' for 'key_sign_hash'. Values available are sha1,sha256",
|
|
346
346
|
):
|
|
347
347
|
API("notauser", "notakey", always_sign_api_key=True, key_sign_hash="notahash").domain_search("amazon")
|
|
348
348
|
|
|
349
349
|
|
|
350
|
+
@vcr.use_cassette
|
|
351
|
+
def test_md5_is_not_supported():
|
|
352
|
+
with pytest.raises(
|
|
353
|
+
ValueError,
|
|
354
|
+
match=r"Invalid value 'md5' for 'key_sign_hash'. Values available are sha1,sha256",
|
|
355
|
+
):
|
|
356
|
+
API("notauser", "notakey", always_sign_api_key=True, key_sign_hash="md5").domain_search("amazon")
|
|
357
|
+
|
|
358
|
+
|
|
350
359
|
@vcr.use_cassette
|
|
351
360
|
def test_rate_limiting():
|
|
352
361
|
domain_searches = ["google"] * 31
|
|
@@ -377,6 +386,7 @@ def test_formats():
|
|
|
377
386
|
|
|
378
387
|
|
|
379
388
|
@vcr.use_cassette
|
|
389
|
+
@pytest.mark.skip(reason="Iris endpoint is already deprecated in light of enrich and investigate")
|
|
380
390
|
def test_iris():
|
|
381
391
|
with pytest.raises(ValueError):
|
|
382
392
|
api.iris()
|
|
@@ -663,3 +673,43 @@ def test_feeds_endpoint_should_raise_error_if_asked_csv_format_for_download_api(
|
|
|
663
673
|
feeds_api.domaindiscovery(after="-60", output_format="csv", endpoint="download")
|
|
664
674
|
|
|
665
675
|
assert str(excinfo.value) == "csv format is not available in download API."
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
@vcr.use_cassette
|
|
679
|
+
def test_realtime_domain_risk():
|
|
680
|
+
results = feeds_api.realtime_domain_risk(after="-60", header_authentication=False)
|
|
681
|
+
for response in results.response():
|
|
682
|
+
assert results.status == 200
|
|
683
|
+
|
|
684
|
+
rows = response.strip().split("\n")
|
|
685
|
+
assert response is not None
|
|
686
|
+
assert len(rows) >= 1
|
|
687
|
+
|
|
688
|
+
for row in rows:
|
|
689
|
+
feed_result = json.loads(row)
|
|
690
|
+
assert "timestamp" in feed_result.keys()
|
|
691
|
+
assert "domain" in feed_result.keys()
|
|
692
|
+
assert "phishing_risk" in feed_result.keys()
|
|
693
|
+
assert "malware_risk" in feed_result.keys()
|
|
694
|
+
assert "proximity_risk" in feed_result.keys()
|
|
695
|
+
assert "overall_risk" in feed_result.keys()
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
@vcr.use_cassette
|
|
699
|
+
def test_domain_hotlist():
|
|
700
|
+
results = feeds_api.domainhotlist(after="-60", header_authentication=False)
|
|
701
|
+
for response in results.response():
|
|
702
|
+
assert results.status == 200
|
|
703
|
+
|
|
704
|
+
rows = response.strip().split("\n")
|
|
705
|
+
assert response is not None
|
|
706
|
+
assert len(rows) >= 1
|
|
707
|
+
|
|
708
|
+
for row in rows:
|
|
709
|
+
feed_result = json.loads(row)
|
|
710
|
+
assert "timestamp" in feed_result.keys()
|
|
711
|
+
assert "domain" in feed_result.keys()
|
|
712
|
+
assert "phishing_risk" in feed_result.keys()
|
|
713
|
+
assert "malware_risk" in feed_result.keys()
|
|
714
|
+
assert "proximity_risk" in feed_result.keys()
|
|
715
|
+
assert "overall_risk" in feed_result.keys()
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
2.4.0
|
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/__init__.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/base_results.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/cli/main.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/exceptions.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/filters.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/results.py
RENAMED
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/domaintools/utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{domaintools_api_test_upload-2.4.0 → domaintools_api_test_upload-2.5.0}/tests/test_filters.py
RENAMED
|
File without changes
|
|
File without changes
|