domaintools-api 2.0.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {domaintools_api-2.0.0/domaintools_api.egg-info → domaintools_api-2.2.0}/PKG-INFO +2 -7
- domaintools_api-2.2.0/VERSION +1 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/_version.py +1 -1
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/api.py +72 -107
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/base_results.py +95 -48
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/api.py +6 -4
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/__init__.py +1 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/domains.py +60 -63
- domaintools_api-2.2.0/domaintools/cli/commands/feeds.py +138 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/constants.py +5 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/utils.py +3 -0
- domaintools_api-2.2.0/domaintools/results.py +143 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/utils.py +7 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0/domaintools_api.egg-info}/PKG-INFO +2 -7
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_api.egg-info/SOURCES.txt +1 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_async/__init__.py +11 -5
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/pyproject.toml +1 -6
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/setup.py +1 -1
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/tests/test_api.py +63 -5
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/tests/test_cli.py +1 -1
- domaintools_api-2.0.0/VERSION +0 -1
- domaintools_api-2.0.0/domaintools/results.py +0 -81
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/LICENSE +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/README.md +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/__init__.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/__init__.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/accounts.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/detects.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/ips.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/iris.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/commands/phisheye.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/cli/main.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/exceptions.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools/filters.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_api.egg-info/dependency_links.txt +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_api.egg-info/entry_points.txt +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_api.egg-info/requires.txt +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/domaintools_api.egg-info/top_level.txt +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/setup.cfg +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/tests/test_async.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/tests/test_filters.py +0 -0
- {domaintools_api-2.0.0 → domaintools_api-2.2.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: domaintools_api
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: DomainTools Official Python API
|
|
5
5
|
Author-email: DomainTools <integrations@domaintools.com>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -33,16 +33,11 @@ Classifier: Natural Language :: English
|
|
|
33
33
|
Classifier: Environment :: Console
|
|
34
34
|
Classifier: License :: OSI Approved :: MIT License
|
|
35
35
|
Classifier: Programming Language :: Python
|
|
36
|
-
Classifier: Programming Language :: Python :: 3
|
|
37
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
38
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
39
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
40
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
41
36
|
Classifier: Programming Language :: Python :: 3.9
|
|
42
37
|
Classifier: Programming Language :: Python :: 3.10
|
|
43
38
|
Classifier: Topic :: Software Development :: Libraries
|
|
44
39
|
Classifier: Topic :: Utilities
|
|
45
|
-
Requires-Python: >=3.
|
|
40
|
+
Requires-Python: >=3.9
|
|
46
41
|
Description-Content-Type: text/markdown
|
|
47
42
|
License-File: LICENSE
|
|
48
43
|
Requires-Dist: httpx
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.2.0
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
from datetime import datetime, timedelta, timezone
|
|
2
2
|
from hashlib import sha1, sha256, md5
|
|
3
3
|
from hmac import new as hmac
|
|
4
|
-
from types import MethodType
|
|
5
4
|
import re
|
|
6
5
|
|
|
7
6
|
from domaintools._version import current as version
|
|
8
|
-
from domaintools.results import
|
|
7
|
+
from domaintools.results import (
|
|
8
|
+
GroupedIterable,
|
|
9
|
+
ParsedWhois,
|
|
10
|
+
ParsedDomainRdap,
|
|
11
|
+
Reputation,
|
|
12
|
+
Results,
|
|
13
|
+
)
|
|
9
14
|
from domaintools.filters import (
|
|
10
15
|
filter_by_riskscore,
|
|
11
16
|
filter_by_expire_date,
|
|
@@ -51,7 +56,7 @@ class API(object):
|
|
|
51
56
|
verify_ssl=True,
|
|
52
57
|
rate_limit=True,
|
|
53
58
|
proxy_url=None,
|
|
54
|
-
always_sign_api_key=
|
|
59
|
+
always_sign_api_key=True,
|
|
55
60
|
key_sign_hash="sha256",
|
|
56
61
|
app_name="python_wrapper",
|
|
57
62
|
app_version=version,
|
|
@@ -78,16 +83,12 @@ class API(object):
|
|
|
78
83
|
self._build_api_url(api_url, api_port)
|
|
79
84
|
|
|
80
85
|
if not https:
|
|
81
|
-
raise Exception(
|
|
82
|
-
"The DomainTools API endpoints no longer support http traffic. Please make sure https=True."
|
|
83
|
-
)
|
|
86
|
+
raise Exception("The DomainTools API endpoints no longer support http traffic. Please make sure https=True.")
|
|
84
87
|
if proxy_url:
|
|
85
88
|
if isinstance(proxy_url, str):
|
|
86
89
|
self.proxy_url = {"http://": proxy_url, "https://": proxy_url}
|
|
87
90
|
else:
|
|
88
|
-
raise Exception(
|
|
89
|
-
"Proxy URL must be a string. For example: '127.0.0.1:8888'"
|
|
90
|
-
)
|
|
91
|
+
raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
|
|
91
92
|
|
|
92
93
|
def _build_api_url(self, api_url=None, api_port=None):
|
|
93
94
|
"""Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`"""
|
|
@@ -111,31 +112,18 @@ class API(object):
|
|
|
111
112
|
hours = limit_hours and 3600 / float(limit_hours)
|
|
112
113
|
minutes = limit_minutes and 60 / float(limit_minutes)
|
|
113
114
|
|
|
114
|
-
self.limits[product["id"]] = {
|
|
115
|
-
"interval": timedelta(seconds=minutes or hours or default)
|
|
116
|
-
}
|
|
115
|
+
self.limits[product["id"]] = {"interval": timedelta(seconds=minutes or hours or default)}
|
|
117
116
|
|
|
118
117
|
def _results(self, product, path, cls=Results, **kwargs):
|
|
119
118
|
"""Returns _results for the specified API path with the specified **kwargs parameters"""
|
|
120
|
-
if
|
|
121
|
-
product != "account-information"
|
|
122
|
-
and self.rate_limit
|
|
123
|
-
and not self.limits_set
|
|
124
|
-
and not self.limits
|
|
125
|
-
):
|
|
119
|
+
if product != "account-information" and self.rate_limit and not self.limits_set and not self.limits:
|
|
126
120
|
self._rate_limit()
|
|
127
121
|
|
|
128
122
|
uri = "/".join((self._rest_api_url, path.lstrip("/")))
|
|
129
123
|
parameters = self.default_parameters.copy()
|
|
130
124
|
parameters["api_username"] = self.username
|
|
131
125
|
self.handle_api_key(path, parameters)
|
|
132
|
-
parameters.update(
|
|
133
|
-
{
|
|
134
|
-
key: str(value).lower() if value in (True, False) else value
|
|
135
|
-
for key, value in kwargs.items()
|
|
136
|
-
if value is not None
|
|
137
|
-
}
|
|
138
|
-
)
|
|
126
|
+
parameters.update({key: str(value).lower() if value in (True, False) else value for key, value in kwargs.items() if value is not None})
|
|
139
127
|
|
|
140
128
|
return cls(self, product, uri, **parameters)
|
|
141
129
|
|
|
@@ -148,14 +136,10 @@ class API(object):
|
|
|
148
136
|
else:
|
|
149
137
|
raise ValueError(
|
|
150
138
|
"Invalid value '{0}' for 'key_sign_hash'. "
|
|
151
|
-
"Values available are {1}".format(
|
|
152
|
-
self.key_sign_hash, ",".join(AVAILABLE_KEY_SIGN_HASHES)
|
|
153
|
-
)
|
|
139
|
+
"Values available are {1}".format(self.key_sign_hash, ",".join(AVAILABLE_KEY_SIGN_HASHES))
|
|
154
140
|
)
|
|
155
141
|
|
|
156
|
-
parameters["timestamp"] = datetime.now(timezone.utc).strftime(
|
|
157
|
-
"%Y-%m-%dT%H:%M:%SZ"
|
|
158
|
-
)
|
|
142
|
+
parameters["timestamp"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
159
143
|
parameters["signature"] = hmac(
|
|
160
144
|
self.key.encode("utf8"),
|
|
161
145
|
"".join([self.username, parameters["timestamp"], path]).encode("utf8"),
|
|
@@ -164,9 +148,7 @@ class API(object):
|
|
|
164
148
|
|
|
165
149
|
def account_information(self, **kwargs):
|
|
166
150
|
"""Provides a snapshot of your accounts current API usage"""
|
|
167
|
-
return self._results(
|
|
168
|
-
"account-information", "/v1/account", items_path=("products",), **kwargs
|
|
169
|
-
)
|
|
151
|
+
return self._results("account-information", "/v1/account", items_path=("products",), **kwargs)
|
|
170
152
|
|
|
171
153
|
def available_api_calls(self):
|
|
172
154
|
"""Provides a list of api calls that you can use based on your account information."""
|
|
@@ -181,25 +163,10 @@ class API(object):
|
|
|
181
163
|
string[1:],
|
|
182
164
|
)
|
|
183
165
|
|
|
184
|
-
api_calls = tuple(
|
|
185
|
-
|
|
186
|
-
api_call
|
|
187
|
-
for api_call in dir(API)
|
|
188
|
-
if not api_call.startswith("_")
|
|
189
|
-
and callable(getattr(API, api_call, None))
|
|
190
|
-
)
|
|
191
|
-
)
|
|
192
|
-
return sorted(
|
|
193
|
-
[
|
|
194
|
-
snakecase(p["id"])
|
|
195
|
-
for p in self.account_information()["products"]
|
|
196
|
-
if snakecase(p["id"]) in api_calls
|
|
197
|
-
]
|
|
198
|
-
)
|
|
166
|
+
api_calls = tuple((api_call for api_call in dir(API) if not api_call.startswith("_") and callable(getattr(API, api_call, None))))
|
|
167
|
+
return sorted([snakecase(p["id"]) for p in self.account_information()["products"] if snakecase(p["id"]) in api_calls])
|
|
199
168
|
|
|
200
|
-
def brand_monitor(
|
|
201
|
-
self, query, exclude=None, domain_status=None, days_back=None, **kwargs
|
|
202
|
-
):
|
|
169
|
+
def brand_monitor(self, query, exclude=None, domain_status=None, days_back=None, **kwargs):
|
|
203
170
|
"""Pass in one or more terms as a list or separated by the pipe character ( | )"""
|
|
204
171
|
if exclude is None:
|
|
205
172
|
exclude = []
|
|
@@ -325,9 +292,16 @@ class API(object):
|
|
|
325
292
|
**kwargs,
|
|
326
293
|
)
|
|
327
294
|
|
|
328
|
-
def
|
|
329
|
-
|
|
330
|
-
|
|
295
|
+
def parsed_domain_rdap(self, query, **kwargs):
|
|
296
|
+
"""Pass in a domain name to see the most recent Domain-RDAP registration record"""
|
|
297
|
+
return self._results(
|
|
298
|
+
"parsed-domain-rdap",
|
|
299
|
+
"/v1/{0}/rdap/parsed/".format(query),
|
|
300
|
+
cls=ParsedDomainRdap,
|
|
301
|
+
**kwargs,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
def registrant_monitor(self, query, exclude=None, days_back=0, limit=None, **kwargs):
|
|
331
305
|
"""One or more terms as a Python list or separated by the pipe character ( | )."""
|
|
332
306
|
if exclude is None:
|
|
333
307
|
exclude = []
|
|
@@ -355,15 +329,11 @@ class API(object):
|
|
|
355
329
|
|
|
356
330
|
def reverse_ip(self, domain=None, limit=None, **kwargs):
|
|
357
331
|
"""Pass in a domain name."""
|
|
358
|
-
return self._results(
|
|
359
|
-
"reverse-ip", "/v1/{0}/reverse-ip".format(domain), limit=limit, **kwargs
|
|
360
|
-
)
|
|
332
|
+
return self._results("reverse-ip", "/v1/{0}/reverse-ip".format(domain), limit=limit, **kwargs)
|
|
361
333
|
|
|
362
334
|
def host_domains(self, ip=None, limit=None, **kwargs):
|
|
363
335
|
"""Pass in an IP address."""
|
|
364
|
-
return self._results(
|
|
365
|
-
"reverse-ip", "/v1/{0}/host-domains".format(ip), limit=limit, **kwargs
|
|
366
|
-
)
|
|
336
|
+
return self._results("reverse-ip", "/v1/{0}/host-domains".format(ip), limit=limit, **kwargs)
|
|
367
337
|
|
|
368
338
|
def reverse_ip_whois(
|
|
369
339
|
self,
|
|
@@ -402,9 +372,7 @@ class API(object):
|
|
|
402
372
|
**kwargs,
|
|
403
373
|
)
|
|
404
374
|
|
|
405
|
-
def reverse_whois(
|
|
406
|
-
self, query, exclude=None, scope="current", mode="purchase", **kwargs
|
|
407
|
-
):
|
|
375
|
+
def reverse_whois(self, query, exclude=None, scope="current", mode="purchase", **kwargs):
|
|
408
376
|
"""List of one or more terms to search for in the Whois record,
|
|
409
377
|
as a Python list or separated with the pipe character ( | ).
|
|
410
378
|
"""
|
|
@@ -424,9 +392,7 @@ class API(object):
|
|
|
424
392
|
"""Pass in a domain name or an IP address to perform a whois lookup."""
|
|
425
393
|
return self._results("whois", "/v1/{0}/whois".format(query), **kwargs)
|
|
426
394
|
|
|
427
|
-
def whois_history(
|
|
428
|
-
self, query, mode=None, sort=None, offset=None, limit=None, **kwargs
|
|
429
|
-
):
|
|
395
|
+
def whois_history(self, query, mode=None, sort=None, offset=None, limit=None, **kwargs):
|
|
430
396
|
"""Pass in a domain name."""
|
|
431
397
|
return self._results(
|
|
432
398
|
"whois-history",
|
|
@@ -485,16 +451,7 @@ class API(object):
|
|
|
485
451
|
"""Performs a search for the provided search terms ANDed together,
|
|
486
452
|
returning the pivot engine row data for the resulting domains.
|
|
487
453
|
"""
|
|
488
|
-
if
|
|
489
|
-
not domain
|
|
490
|
-
and not ip
|
|
491
|
-
and not email
|
|
492
|
-
and not nameserver
|
|
493
|
-
and not registrar
|
|
494
|
-
and not registrant
|
|
495
|
-
and not registrant_org
|
|
496
|
-
and not kwargs
|
|
497
|
-
):
|
|
454
|
+
if not domain and not ip and not email and not nameserver and not registrar and not registrant and not registrant_org and not kwargs:
|
|
498
455
|
raise ValueError("At least one search term must be specified")
|
|
499
456
|
|
|
500
457
|
return self._results(
|
|
@@ -569,12 +526,8 @@ class API(object):
|
|
|
569
526
|
younger_than_date = kwargs.pop("younger_than_date", {}) or None
|
|
570
527
|
older_than_date = kwargs.pop("older_than_date", {}) or None
|
|
571
528
|
updated_after = kwargs.pop("updated_after", {}) or None
|
|
572
|
-
include_domains_with_missing_field = (
|
|
573
|
-
|
|
574
|
-
)
|
|
575
|
-
exclude_domains_with_missing_field = (
|
|
576
|
-
kwargs.pop("exclude_domains_with_missing_field", {}) or None
|
|
577
|
-
)
|
|
529
|
+
include_domains_with_missing_field = kwargs.pop("include_domains_with_missing_field", {}) or None
|
|
530
|
+
exclude_domains_with_missing_field = kwargs.pop("exclude_domains_with_missing_field", {}) or None
|
|
578
531
|
|
|
579
532
|
filtered_results = DTResultFilter(result_set=results).by(
|
|
580
533
|
[
|
|
@@ -582,12 +535,8 @@ class API(object):
|
|
|
582
535
|
filter_by_expire_date(date=younger_than_date, lookup_type="before"),
|
|
583
536
|
filter_by_expire_date(date=older_than_date, lookup_type="after"),
|
|
584
537
|
filter_by_date_updated_after(date=updated_after),
|
|
585
|
-
filter_by_field(
|
|
586
|
-
|
|
587
|
-
),
|
|
588
|
-
filter_by_field(
|
|
589
|
-
field=exclude_domains_with_missing_field, filter_type="exclude"
|
|
590
|
-
),
|
|
538
|
+
filter_by_field(field=include_domains_with_missing_field, filter_type="include"),
|
|
539
|
+
filter_by_field(field=exclude_domains_with_missing_field, filter_type="exclude"),
|
|
591
540
|
]
|
|
592
541
|
)
|
|
593
542
|
|
|
@@ -692,9 +641,7 @@ class API(object):
|
|
|
692
641
|
kwargs["search_hash"] = search_hash
|
|
693
642
|
|
|
694
643
|
if not (kwargs or domains):
|
|
695
|
-
raise ValueError(
|
|
696
|
-
"Need to define investigation using kwarg filters or domains"
|
|
697
|
-
)
|
|
644
|
+
raise ValueError("Need to define investigation using kwarg filters or domains")
|
|
698
645
|
|
|
699
646
|
if isinstance(domains, (list, tuple)):
|
|
700
647
|
domains = ",".join(domains)
|
|
@@ -724,12 +671,8 @@ class API(object):
|
|
|
724
671
|
filter_by_expire_date(date=younger_than_date, lookup_type="before"),
|
|
725
672
|
filter_by_expire_date(date=older_than_date, lookup_type="after"),
|
|
726
673
|
filter_by_date_updated_after(date=updated_after),
|
|
727
|
-
filter_by_field(
|
|
728
|
-
|
|
729
|
-
),
|
|
730
|
-
filter_by_field(
|
|
731
|
-
field=exclude_domains_with_missing_field, filter_type="exclude"
|
|
732
|
-
),
|
|
674
|
+
filter_by_field(field=include_domains_with_missing_field, filter_type="include"),
|
|
675
|
+
filter_by_field(field=exclude_domains_with_missing_field, filter_type="exclude"),
|
|
733
676
|
]
|
|
734
677
|
)
|
|
735
678
|
|
|
@@ -769,9 +712,7 @@ class API(object):
|
|
|
769
712
|
|
|
770
713
|
if include_counts:
|
|
771
714
|
if not datetime_counts_since:
|
|
772
|
-
raise ValueError(
|
|
773
|
-
"Need to define datetime_counts_since when include_counts is True"
|
|
774
|
-
)
|
|
715
|
+
raise ValueError("Need to define datetime_counts_since when include_counts is True")
|
|
775
716
|
if isinstance(datetime_counts_since, datetime):
|
|
776
717
|
datetime_counts_since = str(datetime_counts_since.astimezone())
|
|
777
718
|
elif isinstance(datetime_counts_since, str):
|
|
@@ -979,9 +920,7 @@ class API(object):
|
|
|
979
920
|
**kwargs,
|
|
980
921
|
)
|
|
981
922
|
|
|
982
|
-
def iris_detect_manage_watchlist_domains(
|
|
983
|
-
self, watchlist_domain_ids, state, **kwargs
|
|
984
|
-
):
|
|
923
|
+
def iris_detect_manage_watchlist_domains(self, watchlist_domain_ids, state, **kwargs):
|
|
985
924
|
"""Changes the watch state of a list of domains by their Iris Detect domain ID.
|
|
986
925
|
|
|
987
926
|
watchlist_domain_ids: List[str]: required. List of Iris Detect domain IDs to manage.
|
|
@@ -1000,9 +939,7 @@ class API(object):
|
|
|
1000
939
|
**kwargs,
|
|
1001
940
|
)
|
|
1002
941
|
|
|
1003
|
-
def iris_detect_escalate_domains(
|
|
1004
|
-
self, watchlist_domain_ids, escalation_type, **kwargs
|
|
1005
|
-
):
|
|
942
|
+
def iris_detect_escalate_domains(self, watchlist_domain_ids, escalation_type, **kwargs):
|
|
1006
943
|
"""Changes the escalation type of a list of domains by their Iris Detect domain ID.
|
|
1007
944
|
|
|
1008
945
|
watchlist_domain_ids: List[str]: required. List of Iris Detect domain IDs to escalate.
|
|
@@ -1120,3 +1057,31 @@ class API(object):
|
|
|
1120
1057
|
response_path=(),
|
|
1121
1058
|
**kwargs,
|
|
1122
1059
|
)
|
|
1060
|
+
|
|
1061
|
+
def nod(self, **kwargs):
|
|
1062
|
+
"""Returns back list of the newly observed domains feed"""
|
|
1063
|
+
sessionID = kwargs.get("sessionID")
|
|
1064
|
+
after = kwargs.get("after")
|
|
1065
|
+
if not (sessionID or after):
|
|
1066
|
+
raise ValueError("sessionID or after (can be both) must be defined")
|
|
1067
|
+
|
|
1068
|
+
return self._results(
|
|
1069
|
+
"newly-observed-domains-feed-(api)",
|
|
1070
|
+
"v1/feed/nod/",
|
|
1071
|
+
response_path=(),
|
|
1072
|
+
**kwargs,
|
|
1073
|
+
)
|
|
1074
|
+
|
|
1075
|
+
def nad(self, **kwargs):
|
|
1076
|
+
"""Returns back list of the newly active domains feed"""
|
|
1077
|
+
sessionID = kwargs.get("sessionID")
|
|
1078
|
+
after = kwargs.get("after")
|
|
1079
|
+
if not (sessionID or after):
|
|
1080
|
+
raise ValueError("sessionID or after (can be both) must be defined")
|
|
1081
|
+
|
|
1082
|
+
return self._results(
|
|
1083
|
+
"newly-active-domains-feed-(api)",
|
|
1084
|
+
"v1/feed/nad/",
|
|
1085
|
+
response_path=(),
|
|
1086
|
+
**kwargs,
|
|
1087
|
+
)
|
|
@@ -1,27 +1,46 @@
|
|
|
1
1
|
"""Defines the base result object - which specifies how DomainTools API endpoints will be interacted with"""
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
5
|
import time
|
|
6
6
|
import logging
|
|
7
7
|
from datetime import datetime
|
|
8
8
|
|
|
9
|
-
from domaintools.exceptions import (
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
from domaintools.exceptions import (
|
|
10
|
+
BadRequestException,
|
|
11
|
+
InternalServerErrorException,
|
|
12
|
+
NotAuthorizedException,
|
|
13
|
+
NotFoundException,
|
|
14
|
+
ServiceException,
|
|
15
|
+
ServiceUnavailableException,
|
|
16
|
+
IncompleteResponseException,
|
|
17
|
+
RequestUriTooLongException,
|
|
18
|
+
)
|
|
19
|
+
from domaintools.utils import get_feeds_products_list
|
|
20
|
+
|
|
12
21
|
from httpx import Client
|
|
13
22
|
|
|
14
|
-
try:
|
|
23
|
+
try: # pragma: no cover
|
|
15
24
|
from collections.abc import MutableMapping, MutableSequence
|
|
16
|
-
except ImportError:
|
|
25
|
+
except ImportError: # pragma: no cover
|
|
17
26
|
from collections import MutableMapping, MutableSequence
|
|
18
27
|
|
|
19
28
|
log = logging.getLogger(__name__)
|
|
20
29
|
|
|
30
|
+
|
|
21
31
|
class Results(MutableMapping, MutableSequence):
|
|
22
32
|
"""The base (abstract) DomainTools result definition"""
|
|
23
33
|
|
|
24
|
-
def __init__(
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
api,
|
|
37
|
+
product,
|
|
38
|
+
url,
|
|
39
|
+
items_path=(),
|
|
40
|
+
response_path=("response",),
|
|
41
|
+
proxy_url=None,
|
|
42
|
+
**kwargs,
|
|
43
|
+
):
|
|
25
44
|
self.api = api
|
|
26
45
|
self.product = product
|
|
27
46
|
self.url = url
|
|
@@ -41,29 +60,33 @@ class Results(MutableMapping, MutableSequence):
|
|
|
41
60
|
|
|
42
61
|
now = datetime.now()
|
|
43
62
|
limit = self.api.limits[self.product]
|
|
44
|
-
if
|
|
45
|
-
limit[
|
|
63
|
+
if "last_scheduled" not in limit:
|
|
64
|
+
limit["last_scheduled"] = now
|
|
46
65
|
return None
|
|
47
66
|
|
|
48
|
-
safe_after = limit[
|
|
67
|
+
safe_after = limit["last_scheduled"] + limit["interval"]
|
|
49
68
|
wait_for = 0
|
|
50
69
|
if now < safe_after:
|
|
51
70
|
wait_for = safe_after - now
|
|
52
71
|
wait_for = float(wait_for.seconds) + (float(wait_for.microseconds) / 1000000.0)
|
|
53
|
-
limit[
|
|
72
|
+
limit["last_scheduled"] = safe_after
|
|
54
73
|
else:
|
|
55
|
-
limit[
|
|
74
|
+
limit["last_scheduled"] = now
|
|
56
75
|
|
|
57
76
|
return wait_for
|
|
58
77
|
|
|
59
78
|
def _make_request(self):
|
|
60
79
|
|
|
61
|
-
with Client(verify=self.api.verify_ssl,
|
|
62
|
-
if self.product in [
|
|
80
|
+
with Client(verify=self.api.verify_ssl, proxy=self.api.proxy_url, timeout=None) as session:
|
|
81
|
+
if self.product in [
|
|
82
|
+
"iris-investigate",
|
|
83
|
+
"iris-enrich",
|
|
84
|
+
"iris-detect-escalate-domains",
|
|
85
|
+
]:
|
|
63
86
|
post_data = self.kwargs.copy()
|
|
64
87
|
post_data.update(self.api.extra_request_params)
|
|
65
88
|
return session.post(url=self.url, data=post_data)
|
|
66
|
-
elif self.product in [
|
|
89
|
+
elif self.product in ["iris-detect-manage-watchlist-domains"]:
|
|
67
90
|
patch_data = self.kwargs.copy()
|
|
68
91
|
patch_data.update(self.api.extra_request_params)
|
|
69
92
|
return session.patch(url=self.url, json=patch_data)
|
|
@@ -72,20 +95,22 @@ class Results(MutableMapping, MutableSequence):
|
|
|
72
95
|
|
|
73
96
|
def _get_results(self):
|
|
74
97
|
wait_for = self._wait_time()
|
|
75
|
-
if self.api.rate_limit and (wait_for is None or self.product ==
|
|
98
|
+
if self.api.rate_limit and (wait_for is None or self.product == "account-information"):
|
|
76
99
|
data = self._make_request()
|
|
77
|
-
if data.status_code == 503:
|
|
100
|
+
if data.status_code == 503: # pragma: no cover
|
|
78
101
|
sleeptime = 60
|
|
79
|
-
log.info(
|
|
80
|
-
|
|
102
|
+
log.info(
|
|
103
|
+
"503 encountered for [%s] - sleeping [%s] seconds before retrying request.",
|
|
104
|
+
self.product,
|
|
105
|
+
sleeptime,
|
|
106
|
+
)
|
|
81
107
|
time.sleep(sleeptime)
|
|
82
108
|
self._wait_time()
|
|
83
109
|
data = self._make_request()
|
|
84
110
|
return data
|
|
85
111
|
|
|
86
112
|
if wait_for > 0:
|
|
87
|
-
log.info(
|
|
88
|
-
wait_for, self.product)
|
|
113
|
+
log.info("Sleeping for [%s] prior to requesting [%s].", wait_for, self.product)
|
|
89
114
|
time.sleep(wait_for)
|
|
90
115
|
return self._make_request()
|
|
91
116
|
|
|
@@ -93,7 +118,11 @@ class Results(MutableMapping, MutableSequence):
|
|
|
93
118
|
if self._data is None:
|
|
94
119
|
results = self._get_results()
|
|
95
120
|
self.setStatus(results.status_code, results)
|
|
96
|
-
if
|
|
121
|
+
if (
|
|
122
|
+
self.kwargs.get("format", "json") == "json"
|
|
123
|
+
and self.product
|
|
124
|
+
not in get_feeds_products_list() # Special handling of feeds products' data to preserve the result in jsonline format
|
|
125
|
+
):
|
|
97
126
|
self._data = results.json()
|
|
98
127
|
else:
|
|
99
128
|
self._data = results.text
|
|
@@ -109,11 +138,9 @@ class Results(MutableMapping, MutableSequence):
|
|
|
109
138
|
return self._data
|
|
110
139
|
|
|
111
140
|
def check_limit_exceeded(self):
|
|
112
|
-
if self.kwargs.get(
|
|
113
|
-
if
|
|
114
|
-
|
|
115
|
-
self._data['response']['limit_exceeded'] is True):
|
|
116
|
-
return True, self._data['response']['message']
|
|
141
|
+
if self.kwargs.get("format", "json") == "json":
|
|
142
|
+
if "response" in self._data and "limit_exceeded" in self._data["response"] and self._data["response"]["limit_exceeded"] is True:
|
|
143
|
+
return True, self._data["response"]["message"]
|
|
117
144
|
# TODO: handle html, xml response errors better.
|
|
118
145
|
elif "response" in self._data and "limit_exceeded" in self._data:
|
|
119
146
|
return True, "limit exceeded"
|
|
@@ -121,7 +148,7 @@ class Results(MutableMapping, MutableSequence):
|
|
|
121
148
|
|
|
122
149
|
@property
|
|
123
150
|
def status(self):
|
|
124
|
-
if not getattr(self,
|
|
151
|
+
if not getattr(self, "_status", None):
|
|
125
152
|
self._status = self._get_results().status_code
|
|
126
153
|
|
|
127
154
|
return self._status
|
|
@@ -135,7 +162,7 @@ class Results(MutableMapping, MutableSequence):
|
|
|
135
162
|
if response is not None:
|
|
136
163
|
try:
|
|
137
164
|
reason = response.json()
|
|
138
|
-
except Exception:
|
|
165
|
+
except Exception: # pragma: no cover
|
|
139
166
|
reason = response.text
|
|
140
167
|
if callable(reason):
|
|
141
168
|
reason = reason()
|
|
@@ -146,16 +173,16 @@ class Results(MutableMapping, MutableSequence):
|
|
|
146
173
|
raise NotAuthorizedException(code, reason)
|
|
147
174
|
elif code == 404:
|
|
148
175
|
raise NotFoundException(code, reason)
|
|
149
|
-
elif code == 500:
|
|
176
|
+
elif code == 500: # pragma: no cover
|
|
150
177
|
raise InternalServerErrorException(code, reason)
|
|
151
|
-
elif code == 503:
|
|
178
|
+
elif code == 503: # pragma: no cover
|
|
152
179
|
raise ServiceUnavailableException(code, reason)
|
|
153
|
-
elif code == 206:
|
|
180
|
+
elif code == 206: # pragma: no cover
|
|
154
181
|
raise IncompleteResponseException(code, reason)
|
|
155
|
-
elif code == 414:
|
|
182
|
+
elif code == 414: # pragma: no cover
|
|
156
183
|
raise RequestUriTooLongException(code, reason)
|
|
157
|
-
else:
|
|
158
|
-
raise ServiceException(code,
|
|
184
|
+
else: # pragma: no cover
|
|
185
|
+
raise ServiceException(code, "Unknown Exception")
|
|
159
186
|
|
|
160
187
|
def response(self):
|
|
161
188
|
if self._response is None:
|
|
@@ -171,7 +198,7 @@ class Results(MutableMapping, MutableSequence):
|
|
|
171
198
|
|
|
172
199
|
def emails(self):
|
|
173
200
|
"""Find and returns all emails mentioned in the response"""
|
|
174
|
-
return set(re.findall(r
|
|
201
|
+
return set(re.findall(r"[\w\.-]+@[\w\.-]+", str(self.response())))
|
|
175
202
|
|
|
176
203
|
def _items(self):
|
|
177
204
|
if self._items_list is None:
|
|
@@ -221,25 +248,45 @@ class Results(MutableMapping, MutableSequence):
|
|
|
221
248
|
|
|
222
249
|
@property
|
|
223
250
|
def json(self):
|
|
224
|
-
self.kwargs.pop(
|
|
225
|
-
return self.__class__(
|
|
226
|
-
|
|
251
|
+
self.kwargs.pop("format", None)
|
|
252
|
+
return self.__class__(
|
|
253
|
+
format="json",
|
|
254
|
+
product=self.product,
|
|
255
|
+
url=self.url,
|
|
256
|
+
items_path=self.items_path,
|
|
257
|
+
response_path=self.response_path,
|
|
258
|
+
api=self.api,
|
|
259
|
+
**self.kwargs,
|
|
260
|
+
)
|
|
227
261
|
|
|
228
262
|
@property
|
|
229
263
|
def xml(self):
|
|
230
|
-
self.kwargs.pop(
|
|
231
|
-
return self.__class__(
|
|
232
|
-
|
|
264
|
+
self.kwargs.pop("format", None)
|
|
265
|
+
return self.__class__(
|
|
266
|
+
format="xml",
|
|
267
|
+
product=self.product,
|
|
268
|
+
url=self.url,
|
|
269
|
+
items_path=self.items_path,
|
|
270
|
+
response_path=self.response_path,
|
|
271
|
+
api=self.api,
|
|
272
|
+
**self.kwargs,
|
|
273
|
+
)
|
|
233
274
|
|
|
234
275
|
@property
|
|
235
276
|
def html(self):
|
|
236
|
-
self.kwargs.pop(
|
|
237
|
-
return self.__class__(
|
|
238
|
-
|
|
277
|
+
self.kwargs.pop("format", None)
|
|
278
|
+
return self.__class__(
|
|
279
|
+
api=self.api,
|
|
280
|
+
product=self.product,
|
|
281
|
+
url=self.url,
|
|
282
|
+
items_path=self.items_path,
|
|
283
|
+
response_path=self.response_path,
|
|
284
|
+
format="html",
|
|
285
|
+
**self.kwargs,
|
|
286
|
+
)
|
|
239
287
|
|
|
240
288
|
def as_list(self):
|
|
241
|
-
return
|
|
289
|
+
return "\n".join([json.dumps(item, indent=4, separators=(",", ": ")) for item in self._items()])
|
|
242
290
|
|
|
243
291
|
def __str__(self):
|
|
244
|
-
return str(json.dumps(self.data(), indent=4,
|
|
245
|
-
separators=(',', ': ')) if self.kwargs.get('format', 'json') == 'json' else self.data())
|
|
292
|
+
return str(json.dumps(self.data(), indent=4, separators=(",", ": ")) if self.kwargs.get("format", "json") == "json" else self.data())
|
|
@@ -148,10 +148,11 @@ class DTCLICommand:
|
|
|
148
148
|
kwargs (Optional[Dict], optional): The command available kwargs to pass in domaintools API
|
|
149
149
|
"""
|
|
150
150
|
try:
|
|
151
|
-
rate_limit = params.pop("rate_limit"
|
|
152
|
-
response_format = params.pop("format"
|
|
153
|
-
out_file = params.pop("out_file"
|
|
154
|
-
verify_ssl = params.pop("no_verify_ssl"
|
|
151
|
+
rate_limit = params.pop("rate_limit", False)
|
|
152
|
+
response_format = params.pop("format", "json")
|
|
153
|
+
out_file = params.pop("out_file", sys.stdout)
|
|
154
|
+
verify_ssl = params.pop("no_verify_ssl", False)
|
|
155
|
+
always_sign_api_key = params.pop("no_sign_api_key", False)
|
|
155
156
|
source = None
|
|
156
157
|
|
|
157
158
|
if "src_file" in params:
|
|
@@ -185,6 +186,7 @@ class DTCLICommand:
|
|
|
185
186
|
app_name=cls.APP_PARTNER_NAME,
|
|
186
187
|
verify_ssl=verify_ssl,
|
|
187
188
|
rate_limit=rate_limit,
|
|
189
|
+
always_sign_api_key=always_sign_api_key,
|
|
188
190
|
)
|
|
189
191
|
dt_api_func = getattr(dt_api, name)
|
|
190
192
|
|