domaintools-api-test-upload 2.5.4__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {domaintools_api_test_upload-2.5.4/domaintools_api_test_upload.egg-info → domaintools_api_test_upload-2.6.0}/PKG-INFO +2 -2
- domaintools_api_test_upload-2.6.0/VERSION +1 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/_version.py +1 -1
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/api.py +64 -11
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/base_results.py +27 -8
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/api.py +23 -9
- domaintools_api_test_upload-2.6.0/domaintools/results.py +301 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0/domaintools_api_test_upload.egg-info}/PKG-INFO +2 -2
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/pyproject.toml +4 -1
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/setup.py +1 -1
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/tests/test_api.py +71 -72
- domaintools_api_test_upload-2.6.0/tests/test_async.py +50 -0
- domaintools_api_test_upload-2.5.4/VERSION +0 -1
- domaintools_api_test_upload-2.5.4/domaintools/results.py +0 -170
- domaintools_api_test_upload-2.5.4/tests/test_async.py +0 -64
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/LICENSE +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/README.md +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/__init__.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/__init__.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/__init__.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/accounts.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/detects.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/domains.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/feeds.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/ips.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/commands/iris.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/constants.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/main.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/utils.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/constants.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/exceptions.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/filters.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/utils.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_api_test_upload.egg-info/SOURCES.txt +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_api_test_upload.egg-info/dependency_links.txt +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_api_test_upload.egg-info/entry_points.txt +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_api_test_upload.egg-info/requires.txt +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_api_test_upload.egg-info/top_level.txt +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools_async/__init__.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/setup.cfg +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/tests/test_cli.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/tests/test_filters.py +0 -0
- {domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/tests/test_utils.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.6.0
|
|
@@ -92,7 +92,9 @@ class API(object):
|
|
|
92
92
|
self._build_api_url(api_url, api_port)
|
|
93
93
|
|
|
94
94
|
if not https:
|
|
95
|
-
raise Exception(
|
|
95
|
+
raise Exception(
|
|
96
|
+
"The DomainTools API endpoints no longer support http traffic. Please make sure https=True."
|
|
97
|
+
)
|
|
96
98
|
if proxy_url and not isinstance(proxy_url, str):
|
|
97
99
|
raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
|
|
98
100
|
|
|
@@ -126,7 +128,13 @@ class API(object):
|
|
|
126
128
|
def _results(self, product, path, cls=Results, **kwargs):
|
|
127
129
|
"""Returns _results for the specified API path with the specified **kwargs parameters"""
|
|
128
130
|
if product != "account-information" and self.rate_limit and not self.limits_set and not self.limits:
|
|
131
|
+
always_sign_api_key_previous_value = self.always_sign_api_key
|
|
132
|
+
header_authentication_previous_value = self.header_authentication
|
|
129
133
|
self._rate_limit()
|
|
134
|
+
# Reset always_sign_api_key and header_authentication to its original
|
|
135
|
+
# User-set values as these might be affected when self.account_information() was executed
|
|
136
|
+
self.always_sign_api_key = always_sign_api_key_previous_value
|
|
137
|
+
self.header_authentication = header_authentication_previous_value
|
|
130
138
|
|
|
131
139
|
uri = "/".join((self._rest_api_url, path.lstrip("/")))
|
|
132
140
|
parameters = self.default_parameters.copy()
|
|
@@ -134,7 +142,13 @@ class API(object):
|
|
|
134
142
|
is_rttf_product = product in RTTF_PRODUCTS_LIST
|
|
135
143
|
self._handle_api_key_parameters(is_rttf_product)
|
|
136
144
|
self.handle_api_key(is_rttf_product, path, parameters)
|
|
137
|
-
parameters.update(
|
|
145
|
+
parameters.update(
|
|
146
|
+
{
|
|
147
|
+
key: str(value).lower() if value in (True, False) else value
|
|
148
|
+
for key, value in kwargs.items()
|
|
149
|
+
if value is not None
|
|
150
|
+
}
|
|
151
|
+
)
|
|
138
152
|
|
|
139
153
|
return cls(self, product, uri, **parameters)
|
|
140
154
|
|
|
@@ -184,8 +198,20 @@ class API(object):
|
|
|
184
198
|
string[1:],
|
|
185
199
|
)
|
|
186
200
|
|
|
187
|
-
api_calls = tuple(
|
|
188
|
-
|
|
201
|
+
api_calls = tuple(
|
|
202
|
+
(
|
|
203
|
+
api_call
|
|
204
|
+
for api_call in dir(API)
|
|
205
|
+
if not api_call.startswith("_") and callable(getattr(API, api_call, None))
|
|
206
|
+
)
|
|
207
|
+
)
|
|
208
|
+
return sorted(
|
|
209
|
+
[
|
|
210
|
+
snakecase(p["id"])
|
|
211
|
+
for p in self.account_information()["products"]
|
|
212
|
+
if snakecase(p["id"]) in api_calls
|
|
213
|
+
]
|
|
214
|
+
)
|
|
189
215
|
|
|
190
216
|
def brand_monitor(self, query, exclude=None, domain_status=None, days_back=None, **kwargs):
|
|
191
217
|
"""Pass in one or more terms as a list or separated by the pipe character ( | )"""
|
|
@@ -440,7 +466,16 @@ class API(object):
|
|
|
440
466
|
"""Performs a search for the provided search terms ANDed together,
|
|
441
467
|
returning the pivot engine row data for the resulting domains.
|
|
442
468
|
"""
|
|
443
|
-
if
|
|
469
|
+
if (
|
|
470
|
+
not domain
|
|
471
|
+
and not ip
|
|
472
|
+
and not email
|
|
473
|
+
and not nameserver
|
|
474
|
+
and not registrar
|
|
475
|
+
and not registrant
|
|
476
|
+
and not registrant_org
|
|
477
|
+
and not kwargs
|
|
478
|
+
):
|
|
444
479
|
raise ValueError("At least one search term must be specified")
|
|
445
480
|
|
|
446
481
|
return self._results(
|
|
@@ -1064,7 +1099,10 @@ class API(object):
|
|
|
1064
1099
|
validate_feeds_parameters(kwargs)
|
|
1065
1100
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1066
1101
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
|
|
1067
|
-
if
|
|
1102
|
+
if (
|
|
1103
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1104
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1105
|
+
):
|
|
1068
1106
|
# headers param is allowed only in Feed API and CSV format
|
|
1069
1107
|
kwargs.pop("headers", None)
|
|
1070
1108
|
|
|
@@ -1096,7 +1134,10 @@ class API(object):
|
|
|
1096
1134
|
validate_feeds_parameters(kwargs)
|
|
1097
1135
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1098
1136
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1099
|
-
if
|
|
1137
|
+
if (
|
|
1138
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1139
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1140
|
+
):
|
|
1100
1141
|
# headers param is allowed only in Feed API and CSV format
|
|
1101
1142
|
kwargs.pop("headers", None)
|
|
1102
1143
|
|
|
@@ -1157,7 +1198,10 @@ class API(object):
|
|
|
1157
1198
|
validate_feeds_parameters(kwargs)
|
|
1158
1199
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1159
1200
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1160
|
-
if
|
|
1201
|
+
if (
|
|
1202
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1203
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1204
|
+
):
|
|
1161
1205
|
# headers param is allowed only in Feed API and CSV format
|
|
1162
1206
|
kwargs.pop("headers", None)
|
|
1163
1207
|
|
|
@@ -1189,7 +1233,10 @@ class API(object):
|
|
|
1189
1233
|
validate_feeds_parameters(kwargs)
|
|
1190
1234
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1191
1235
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1192
|
-
if
|
|
1236
|
+
if (
|
|
1237
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1238
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1239
|
+
):
|
|
1193
1240
|
# headers param is allowed only in Feed API and CSV format
|
|
1194
1241
|
kwargs.pop("headers", None)
|
|
1195
1242
|
|
|
@@ -1220,7 +1267,10 @@ class API(object):
|
|
|
1220
1267
|
validate_feeds_parameters(kwargs)
|
|
1221
1268
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1222
1269
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1223
|
-
if
|
|
1270
|
+
if (
|
|
1271
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1272
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1273
|
+
):
|
|
1224
1274
|
# headers param is allowed only in Feed API and CSV format
|
|
1225
1275
|
kwargs.pop("headers", None)
|
|
1226
1276
|
|
|
@@ -1251,7 +1301,10 @@ class API(object):
|
|
|
1251
1301
|
validate_feeds_parameters(kwargs)
|
|
1252
1302
|
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
|
|
1253
1303
|
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
|
|
1254
|
-
if
|
|
1304
|
+
if (
|
|
1305
|
+
endpoint == Endpoint.DOWNLOAD.value
|
|
1306
|
+
or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
|
|
1307
|
+
):
|
|
1255
1308
|
# headers param is allowed only in Feed API and CSV format
|
|
1256
1309
|
kwargs.pop("headers", None)
|
|
1257
1310
|
|
{domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/base_results.py
RENAMED
|
@@ -9,7 +9,11 @@ from copy import deepcopy
|
|
|
9
9
|
from datetime import datetime
|
|
10
10
|
from httpx import Client
|
|
11
11
|
|
|
12
|
-
from domaintools.constants import
|
|
12
|
+
from domaintools.constants import (
|
|
13
|
+
RTTF_PRODUCTS_LIST,
|
|
14
|
+
OutputFormat,
|
|
15
|
+
HEADER_ACCEPT_KEY_CSV_FORMAT,
|
|
16
|
+
)
|
|
13
17
|
from domaintools.exceptions import (
|
|
14
18
|
BadRequestException,
|
|
15
19
|
InternalServerErrorException,
|
|
@@ -53,6 +57,7 @@ class Results(MutableMapping, MutableSequence):
|
|
|
53
57
|
self._response = None
|
|
54
58
|
self._items_list = None
|
|
55
59
|
self._data = None
|
|
60
|
+
self._status = None
|
|
56
61
|
|
|
57
62
|
def _wait_time(self):
|
|
58
63
|
if not self.api.rate_limit or not self.product in self.api.limits:
|
|
@@ -92,10 +97,10 @@ class Results(MutableMapping, MutableSequence):
|
|
|
92
97
|
header_key_for_api_key = "X-Api-Key" if is_rttf_product else "X-API-Key"
|
|
93
98
|
headers[header_key_for_api_key] = self.api.key
|
|
94
99
|
|
|
95
|
-
|
|
100
|
+
session_param_and_headers = {"parameters": parameters, "headers": headers}
|
|
101
|
+
return session_param_and_headers
|
|
96
102
|
|
|
97
103
|
def _make_request(self):
|
|
98
|
-
|
|
99
104
|
with Client(verify=self.api.verify_ssl, proxy=self.api.proxy_url, timeout=None) as session:
|
|
100
105
|
session_params_and_headers = self._get_session_params_and_headers()
|
|
101
106
|
headers = session_params_and_headers.get("headers")
|
|
@@ -113,7 +118,12 @@ class Results(MutableMapping, MutableSequence):
|
|
|
113
118
|
return session.patch(url=self.url, json=patch_data, headers=headers)
|
|
114
119
|
else:
|
|
115
120
|
parameters = session_params_and_headers.get("parameters")
|
|
116
|
-
return session.get(
|
|
121
|
+
return session.get(
|
|
122
|
+
url=self.url,
|
|
123
|
+
params=parameters,
|
|
124
|
+
headers=headers,
|
|
125
|
+
**self.api.extra_request_params,
|
|
126
|
+
)
|
|
117
127
|
|
|
118
128
|
def _get_results(self):
|
|
119
129
|
wait_for = self._wait_time()
|
|
@@ -152,7 +162,9 @@ class Results(MutableMapping, MutableSequence):
|
|
|
152
162
|
def check_limit_exceeded(self):
|
|
153
163
|
limit_exceeded, reason = False, ""
|
|
154
164
|
if isinstance(self._data, dict) and (
|
|
155
|
-
"response" in self._data
|
|
165
|
+
"response" in self._data
|
|
166
|
+
and "limit_exceeded" in self._data["response"]
|
|
167
|
+
and self._data["response"]["limit_exceeded"] is True
|
|
156
168
|
):
|
|
157
169
|
limit_exceeded, reason = True, self._data["response"]["message"]
|
|
158
170
|
elif "response" in self._data and "limit_exceeded" in self._data:
|
|
@@ -163,12 +175,12 @@ class Results(MutableMapping, MutableSequence):
|
|
|
163
175
|
|
|
164
176
|
@property
|
|
165
177
|
def status(self):
|
|
166
|
-
if not getattr(self, "_status", None):
|
|
178
|
+
if not getattr(self, "_status", None) and not self.product in RTTF_PRODUCTS_LIST:
|
|
167
179
|
self._status = self._get_results().status_code
|
|
168
180
|
|
|
169
181
|
return self._status
|
|
170
182
|
|
|
171
|
-
def setStatus(self, code, response=None):
|
|
183
|
+
def setStatus(self, code, response=None, reason_text=None):
|
|
172
184
|
self._status = code
|
|
173
185
|
if code == 200 or (self.product in RTTF_PRODUCTS_LIST and code == 206):
|
|
174
186
|
return
|
|
@@ -181,6 +193,9 @@ class Results(MutableMapping, MutableSequence):
|
|
|
181
193
|
reason = response.text
|
|
182
194
|
if callable(reason):
|
|
183
195
|
reason = reason()
|
|
196
|
+
else: # optionally pass a customize reason of error for better traceback
|
|
197
|
+
if reason_text is not None:
|
|
198
|
+
reason = reason_text
|
|
184
199
|
|
|
185
200
|
if code in (400, 422):
|
|
186
201
|
raise BadRequestException(code, reason)
|
|
@@ -330,4 +345,8 @@ class Results(MutableMapping, MutableSequence):
|
|
|
330
345
|
return "\n".join([json.dumps(item, indent=4, separators=(",", ": ")) for item in self._items()])
|
|
331
346
|
|
|
332
347
|
def __str__(self):
|
|
333
|
-
return str(
|
|
348
|
+
return str(
|
|
349
|
+
json.dumps(self.data(), indent=4, separators=(",", ": "))
|
|
350
|
+
if self.kwargs.get("format", "json") == "json"
|
|
351
|
+
else self.data()
|
|
352
|
+
)
|
{domaintools_api_test_upload-2.5.4 → domaintools_api_test_upload-2.6.0}/domaintools/cli/api.py
RENAMED
|
@@ -58,7 +58,9 @@ class DTCLICommand:
|
|
|
58
58
|
datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
|
|
59
59
|
return value
|
|
60
60
|
except:
|
|
61
|
-
raise typer.BadParameter(
|
|
61
|
+
raise typer.BadParameter(
|
|
62
|
+
f"{value} is neither an integer or a valid ISO 8601 datetime string in UTC form"
|
|
63
|
+
)
|
|
62
64
|
|
|
63
65
|
@staticmethod
|
|
64
66
|
def validate_source_file_extension(value: str):
|
|
@@ -78,7 +80,9 @@ class DTCLICommand:
|
|
|
78
80
|
ext = get_file_extension(value)
|
|
79
81
|
|
|
80
82
|
if ext.lower() not in VALID_EXTENSIONS:
|
|
81
|
-
raise typer.BadParameter(
|
|
83
|
+
raise typer.BadParameter(
|
|
84
|
+
f"{value} is not in valid extensions. Valid file extensions: {VALID_EXTENSIONS}"
|
|
85
|
+
)
|
|
82
86
|
|
|
83
87
|
return value
|
|
84
88
|
|
|
@@ -111,7 +115,7 @@ class DTCLICommand:
|
|
|
111
115
|
if cmd_name in ("available_api_calls",):
|
|
112
116
|
return "\n".join(response)
|
|
113
117
|
if response.product in RTTF_PRODUCTS_LIST:
|
|
114
|
-
|
|
118
|
+
pass # do nothing
|
|
115
119
|
return str(getattr(response, out_format) if out_format != "list" else response.as_list())
|
|
116
120
|
|
|
117
121
|
@classmethod
|
|
@@ -203,7 +207,7 @@ class DTCLICommand:
|
|
|
203
207
|
transient=True,
|
|
204
208
|
) as progress:
|
|
205
209
|
|
|
206
|
-
progress.add_task(
|
|
210
|
+
task_id = progress.add_task(
|
|
207
211
|
description=f"Using api credentials with a username of: [cyan]{user}[/cyan]\nExecuting [green]{name}[/green] api call...",
|
|
208
212
|
total=None,
|
|
209
213
|
)
|
|
@@ -222,23 +226,33 @@ class DTCLICommand:
|
|
|
222
226
|
params = params | kwargs
|
|
223
227
|
|
|
224
228
|
response = dt_api_func(**params)
|
|
225
|
-
progress.
|
|
229
|
+
progress.update(
|
|
230
|
+
task_id,
|
|
226
231
|
description=f"Preparing results with format of {response_format}...",
|
|
227
|
-
total=None,
|
|
228
232
|
)
|
|
229
233
|
|
|
230
|
-
output = cls._get_formatted_output(
|
|
234
|
+
output = cls._get_formatted_output(
|
|
235
|
+
cmd_name=name, response=response, out_format=response_format
|
|
236
|
+
)
|
|
231
237
|
|
|
232
238
|
if isinstance(out_file, _io.TextIOWrapper):
|
|
239
|
+
progress.update(
|
|
240
|
+
task_id,
|
|
241
|
+
description=f"Printing the results with format of {response_format}...",
|
|
242
|
+
)
|
|
233
243
|
# use rich `print` command to prettify the ouput in sys.stdout
|
|
234
244
|
if response.product in RTTF_PRODUCTS_LIST:
|
|
235
|
-
|
|
245
|
+
for feeds in response.response():
|
|
246
|
+
print(feeds)
|
|
236
247
|
else:
|
|
237
248
|
print(response)
|
|
238
249
|
else:
|
|
250
|
+
progress.update(
|
|
251
|
+
task_id,
|
|
252
|
+
description=f"Writing results to {out_file}",
|
|
253
|
+
)
|
|
239
254
|
# if it's a file then write
|
|
240
255
|
out_file.write(output if output.endswith("\n") else output + "\n")
|
|
241
|
-
time.sleep(0.25)
|
|
242
256
|
except Exception as e:
|
|
243
257
|
if isinstance(e, ServiceException):
|
|
244
258
|
code = typer.style(getattr(e, "code", 400), fg=typer.colors.BRIGHT_RED)
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Defines the used Result object based on the current versions and/or features available to Python runtime
|
|
2
|
+
|
|
3
|
+
Additionally, defines any custom result objects that may be used to enable more Pythonic interaction with endpoints.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import httpx
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
try: # pragma: no cover
|
|
12
|
+
from collections import OrderedDict
|
|
13
|
+
except ImportError: # pragma: no cover
|
|
14
|
+
from ordereddict import OrderedDict
|
|
15
|
+
|
|
16
|
+
from itertools import zip_longest, chain
|
|
17
|
+
from typing import Generator
|
|
18
|
+
from httpx import Client
|
|
19
|
+
|
|
20
|
+
from domaintools_async import AsyncResults as Results
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
log = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Reputation(Results):
|
|
27
|
+
"""Returns the reputation results in a format that can quickly be converted into floats / ints"""
|
|
28
|
+
|
|
29
|
+
def __float__(self):
|
|
30
|
+
return float(self["risk_score"])
|
|
31
|
+
|
|
32
|
+
def __int__(self):
|
|
33
|
+
return int(self["risk_score"])
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class GroupedIterable(Results):
|
|
37
|
+
"""Returns a results item in a format that allows for grouped iteration of mulpitle result lists"""
|
|
38
|
+
|
|
39
|
+
def _items(self):
|
|
40
|
+
if self._items_list is None:
|
|
41
|
+
self._items_list = chain(
|
|
42
|
+
*[
|
|
43
|
+
zip_longest([], value, fillvalue=key)
|
|
44
|
+
for key, value in self.response().items()
|
|
45
|
+
if type(value) in (list, tuple)
|
|
46
|
+
]
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
return self._items_list
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ParsedWhois(Results):
|
|
53
|
+
"""Returns the parsed whois results in a format that can quickly be flattened"""
|
|
54
|
+
|
|
55
|
+
def flattened(self):
|
|
56
|
+
"""Returns a flattened version of the parsed whois data"""
|
|
57
|
+
parsed = self["parsed_whois"]
|
|
58
|
+
flat = OrderedDict()
|
|
59
|
+
for key in (
|
|
60
|
+
"domain",
|
|
61
|
+
"created_date",
|
|
62
|
+
"updated_date",
|
|
63
|
+
"expired_date",
|
|
64
|
+
"statuses",
|
|
65
|
+
"name_servers",
|
|
66
|
+
):
|
|
67
|
+
if key in parsed:
|
|
68
|
+
value = parsed[key]
|
|
69
|
+
flat[key] = " | ".join(value) if type(value) in (list, tuple) else value
|
|
70
|
+
|
|
71
|
+
registrar = parsed.get("registrar", {})
|
|
72
|
+
for key in (
|
|
73
|
+
"name",
|
|
74
|
+
"abuse_contact_phone",
|
|
75
|
+
"abuse_contact_email",
|
|
76
|
+
"iana_id",
|
|
77
|
+
"url",
|
|
78
|
+
"whois_server",
|
|
79
|
+
):
|
|
80
|
+
if key in registrar:
|
|
81
|
+
flat["registrar_{0}".format(key)] = registrar[key]
|
|
82
|
+
|
|
83
|
+
if "networks" in parsed:
|
|
84
|
+
networks = parsed.get("networks")
|
|
85
|
+
for network in networks:
|
|
86
|
+
id = network.get("id")
|
|
87
|
+
for key in (
|
|
88
|
+
"range",
|
|
89
|
+
"asn",
|
|
90
|
+
"org",
|
|
91
|
+
"parent",
|
|
92
|
+
"customer",
|
|
93
|
+
"country",
|
|
94
|
+
"phone",
|
|
95
|
+
"status",
|
|
96
|
+
"source",
|
|
97
|
+
"updated_date",
|
|
98
|
+
"created_date",
|
|
99
|
+
):
|
|
100
|
+
if key in network:
|
|
101
|
+
value = network[key]
|
|
102
|
+
flat["network_{0}".format(id)] = (
|
|
103
|
+
" ".join(value) if type(value) in (list, tuple) else value
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
if "contacts" in parsed:
|
|
107
|
+
contacts = parsed.get("contacts")
|
|
108
|
+
if type(contacts) is list:
|
|
109
|
+
# handle IP-style contacts, which show up as a list
|
|
110
|
+
for contact in contacts:
|
|
111
|
+
contact_type = contact.get("type")
|
|
112
|
+
for key in (
|
|
113
|
+
"name",
|
|
114
|
+
"email",
|
|
115
|
+
"org",
|
|
116
|
+
"abuse_mailbos",
|
|
117
|
+
"address",
|
|
118
|
+
"street",
|
|
119
|
+
"city",
|
|
120
|
+
"state",
|
|
121
|
+
"postal",
|
|
122
|
+
"country",
|
|
123
|
+
"phone",
|
|
124
|
+
"fax",
|
|
125
|
+
):
|
|
126
|
+
if key in contact:
|
|
127
|
+
value = contact[key]
|
|
128
|
+
flat["{0}_{1}".format(contact_type, key)] = (
|
|
129
|
+
" ".join(value) if type(value) in (list, tuple) else value
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
elif type(contacts) is dict:
|
|
133
|
+
for contact_type in ("registrant", "admin", "tech", "billing"):
|
|
134
|
+
contact = contacts.get(contact_type, {})
|
|
135
|
+
for key in (
|
|
136
|
+
"name",
|
|
137
|
+
"email",
|
|
138
|
+
"org",
|
|
139
|
+
"street",
|
|
140
|
+
"city",
|
|
141
|
+
"state",
|
|
142
|
+
"postal",
|
|
143
|
+
"country",
|
|
144
|
+
"phone",
|
|
145
|
+
"fax",
|
|
146
|
+
):
|
|
147
|
+
if key in contact:
|
|
148
|
+
value = contact[key]
|
|
149
|
+
flat["{0}_{1}".format(contact_type, key)] = (
|
|
150
|
+
" ".join(value) if type(value) in (list, tuple) else value
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
return flat
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class ParsedDomainRdap(Results):
|
|
157
|
+
"""Returns the parsed domain rdap results in a format that can quickly be flattened"""
|
|
158
|
+
|
|
159
|
+
def flattened(self):
|
|
160
|
+
"""Returns a flattened version of the parsed domain rdap data"""
|
|
161
|
+
parsed = self["parsed_domain_rdap"]
|
|
162
|
+
flat = OrderedDict()
|
|
163
|
+
for key in (
|
|
164
|
+
"domain",
|
|
165
|
+
"handle",
|
|
166
|
+
"domain_statuses",
|
|
167
|
+
"creation_date",
|
|
168
|
+
"last_changed_date",
|
|
169
|
+
"expiration_date",
|
|
170
|
+
"dnssec",
|
|
171
|
+
"nameservers",
|
|
172
|
+
"conformance",
|
|
173
|
+
"emails",
|
|
174
|
+
"email_domains",
|
|
175
|
+
"unclassified_emails",
|
|
176
|
+
):
|
|
177
|
+
if key in parsed:
|
|
178
|
+
value = parsed[key]
|
|
179
|
+
flat[key] = " | ".join(value) if type(value) in (list, tuple) else value
|
|
180
|
+
|
|
181
|
+
registrar = parsed.get("registrar", {})
|
|
182
|
+
for registrar_key, registrar_value in registrar.items():
|
|
183
|
+
if registrar_key == "contacts":
|
|
184
|
+
for i, contact in enumerate(registrar_value, start=1):
|
|
185
|
+
for contact_key, contact_value in contact.items():
|
|
186
|
+
flat[f"registrar_contacts_{contact_key}"] = (
|
|
187
|
+
" | ".join(contact_value)
|
|
188
|
+
if type(contact_value) in (list, tuple)
|
|
189
|
+
else contact_value
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
continue
|
|
193
|
+
flat[f"registrar_{registrar_key}"] = registrar_value
|
|
194
|
+
|
|
195
|
+
contacts = parsed.get("contacts")
|
|
196
|
+
if contacts:
|
|
197
|
+
for i, contact in enumerate(contacts, start=1):
|
|
198
|
+
for contact_key, contact_value in contact.items():
|
|
199
|
+
flat[f"contact_{contact_key}_{i}"] = (
|
|
200
|
+
" | ".join(contact_value) if type(contact_value) in (list, tuple) else contact_value
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
return flat
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class FeedsResults(Results):
|
|
207
|
+
"""
|
|
208
|
+
Real Time Threat Feeds (RTTF) returns an application/ndjson stream.
|
|
209
|
+
With this we use httpx stream to process each JSON object efficiently.
|
|
210
|
+
|
|
211
|
+
Highlevel process:
|
|
212
|
+
|
|
213
|
+
httpx stream -> yield each json line -> check status code -> yield back data to client
|
|
214
|
+
|
|
215
|
+
Returns the generator object for feeds results.
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
latest_feeds_status_code = None
|
|
219
|
+
|
|
220
|
+
def _make_request(self) -> Generator:
|
|
221
|
+
"""
|
|
222
|
+
Creates and manages the httpx stream request, yielding data line by line.
|
|
223
|
+
This is the core generator that communicates with the DT frontend API server.
|
|
224
|
+
"""
|
|
225
|
+
session_info = self._get_session_params_and_headers()
|
|
226
|
+
headers = session_info.get("headers")
|
|
227
|
+
parameters = session_info.get("parameters")
|
|
228
|
+
|
|
229
|
+
try:
|
|
230
|
+
with httpx.stream(
|
|
231
|
+
"GET",
|
|
232
|
+
self.url,
|
|
233
|
+
headers=headers,
|
|
234
|
+
params=parameters,
|
|
235
|
+
verify=self.api.verify_ssl,
|
|
236
|
+
proxy=self.api.proxy_url,
|
|
237
|
+
timeout=None,
|
|
238
|
+
) as response:
|
|
239
|
+
self.latest_feeds_status_code = response.status_code
|
|
240
|
+
# assigned the status code to `latest_feeds_status_code`
|
|
241
|
+
# started the process
|
|
242
|
+
yield {"status_ready": True}
|
|
243
|
+
|
|
244
|
+
for line in response.iter_lines():
|
|
245
|
+
if line:
|
|
246
|
+
yield line
|
|
247
|
+
except Exception as e:
|
|
248
|
+
self.latest_feeds_status_code = 500
|
|
249
|
+
yield {"status_ready": True, "error": str(e)}
|
|
250
|
+
raise
|
|
251
|
+
|
|
252
|
+
def _get_results(self) -> Generator:
|
|
253
|
+
try:
|
|
254
|
+
wait_for = self._wait_time()
|
|
255
|
+
feeds_generator = self._make_request()
|
|
256
|
+
|
|
257
|
+
next(feeds_generator) # to start the generator process
|
|
258
|
+
self.setStatus(self.latest_feeds_status_code) # set the status already
|
|
259
|
+
|
|
260
|
+
should_wait = (
|
|
261
|
+
wait_for
|
|
262
|
+
and wait_for > 0
|
|
263
|
+
and not (self.api.rate_limit and (self.product == "account-information"))
|
|
264
|
+
)
|
|
265
|
+
if should_wait:
|
|
266
|
+
log.info(f"Sleeping for {wait_for}s.")
|
|
267
|
+
time.sleep(wait_for)
|
|
268
|
+
|
|
269
|
+
# yield the rest of the feeds
|
|
270
|
+
yield from feeds_generator
|
|
271
|
+
except Exception as e:
|
|
272
|
+
log.error(f"FATAL: Failed to start the feed generator in _make_request. Reason: {e}")
|
|
273
|
+
self.latest_feeds_status_code = 500
|
|
274
|
+
self.setStatus(500, reason_text=f"Reason: {e}")
|
|
275
|
+
return
|
|
276
|
+
|
|
277
|
+
def data(self) -> Generator:
|
|
278
|
+
self._data = self._get_results()
|
|
279
|
+
return self._data
|
|
280
|
+
|
|
281
|
+
def response(self) -> Generator:
|
|
282
|
+
status_code = None
|
|
283
|
+
while status_code != 200:
|
|
284
|
+
try:
|
|
285
|
+
feed_response_generator = self.data()
|
|
286
|
+
|
|
287
|
+
yield from feed_response_generator
|
|
288
|
+
status_code = self.status
|
|
289
|
+
self._data = None # clear the data here
|
|
290
|
+
|
|
291
|
+
if not self.kwargs.get("sessionID"):
|
|
292
|
+
# we'll only do iterative request for queries that has sessionID.
|
|
293
|
+
# Otherwise, we will have an infinite request if sessionID was not provided but the required data asked is more than the maximum (1 hour of data)
|
|
294
|
+
break
|
|
295
|
+
except Exception as e:
|
|
296
|
+
self.latest_feeds_status_code = 500
|
|
297
|
+
self.setStatus(500, reason_text=f"Reason: {e}")
|
|
298
|
+
break # safely close the while loop if there's any error above
|
|
299
|
+
|
|
300
|
+
def __str__(self):
|
|
301
|
+
return f"{self.__class__.__name__} - {self.product}"
|
|
@@ -3,7 +3,7 @@ requires = ["setuptools"]
|
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
|
-
name = "
|
|
6
|
+
name = "domaintools-api-test-upload"
|
|
7
7
|
dependencies = [
|
|
8
8
|
"httpx==0.28.1",
|
|
9
9
|
"rich",
|
|
@@ -49,3 +49,6 @@ packages = [
|
|
|
49
49
|
|
|
50
50
|
[tool.setuptools.dynamic]
|
|
51
51
|
version = { "file" = "VERSION" }
|
|
52
|
+
|
|
53
|
+
[tool.black]
|
|
54
|
+
line-length = 110
|