datalake-scripts 2.7.1__tar.gz → 2.7.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/PKG-INFO +13 -3
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/README.md +12 -2
- datalake-scripts-2.7.3/datalake/common/token_manager.py +147 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/datalake.py +21 -3
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/endpoint.py +5 -2
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/filtered_tag_subcategory.py +12 -1
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/cli.py +1 -1
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/common/base_engine.py +2 -1
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/common/mixins.py +4 -1
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/PKG-INFO +13 -3
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/requires.txt +1 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/setup.py +1 -1
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/common/fixture.py +6 -0
- datalake-scripts-2.7.3/tests/lib/test_auth_via_tokens.py +332 -0
- datalake-scripts-2.7.1/datalake/common/token_manager.py +0 -99
- datalake-scripts-2.7.1/tests/lib/test_auth_via_tokens.py +0 -181
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/api_objects/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/api_objects/bulk_search_task.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/atom.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/atom_type.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/config.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/logger.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/output.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/throttler.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/utils.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/common/warn.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/advanced_search.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/bulk_search.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/sightings.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/tags.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/miscellaneous/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/common/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/common/base_script.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/engines/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/engines/get_engine.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/engines/post_engine.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/SOURCES.txt +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/entry_points.txt +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/setup.cfg +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/common/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/common/test_output.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/common/test_utils.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/conftest.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_advanced_search.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_atom_type.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_bulk_search.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_filtered_tag_subcategory.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_search_watch.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_sightings.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_tags.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/__init__.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/test_base_engine.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/test_cli.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/test_csv_builder.py +0 -0
- {datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/scripts/test_endpoint_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.3
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -57,10 +57,20 @@ You can also use a script directly by using the following command: `<script_name
|
|
|
57
57
|
## Environment variables
|
|
58
58
|
|
|
59
59
|
#### Authentication
|
|
60
|
-
|
|
60
|
+
|
|
61
|
+
There are two methods of authentication:
|
|
62
|
+
- The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
|
|
63
|
+
- The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
|
|
64
|
+
|
|
65
|
+
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
66
|
+
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
67
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
|
|
68
|
+
|
|
69
|
+
or
|
|
70
|
+
|
|
61
71
|
* `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
|
|
62
72
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
63
|
-
>
|
|
73
|
+
> These last two are independent and one can be used without the other if you wish.
|
|
64
74
|
|
|
65
75
|
#### Throttling
|
|
66
76
|
For throttling the request, those two environment variable can be used:
|
|
@@ -37,10 +37,20 @@ You can also use a script directly by using the following command: `<script_name
|
|
|
37
37
|
## Environment variables
|
|
38
38
|
|
|
39
39
|
#### Authentication
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
There are two methods of authentication:
|
|
42
|
+
- The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
|
|
43
|
+
- The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
|
|
44
|
+
|
|
45
|
+
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
46
|
+
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
47
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
|
|
48
|
+
|
|
49
|
+
or
|
|
50
|
+
|
|
41
51
|
* `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
|
|
42
52
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
43
|
-
>
|
|
53
|
+
> These last two are independent and one can be used without the other if you wish.
|
|
44
54
|
|
|
45
55
|
#### Throttling
|
|
46
56
|
For throttling the request, those two environment variable can be used:
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Token manager will manage tokens for the scripts.
|
|
3
|
+
"""
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from getpass import getpass
|
|
7
|
+
from urllib.parse import urljoin
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from datalake.common.logger import logger
|
|
12
|
+
from datalake.common.utils import get_error_message
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TokenManager:
|
|
16
|
+
"""
|
|
17
|
+
Use it to generate token access to the API
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
endpoint_config: dict,
|
|
23
|
+
*,
|
|
24
|
+
environment: str,
|
|
25
|
+
username=None,
|
|
26
|
+
password=None,
|
|
27
|
+
longterm_token=None,
|
|
28
|
+
):
|
|
29
|
+
"""environment can be either prod or preprod"""
|
|
30
|
+
base_url = urljoin(
|
|
31
|
+
endpoint_config["main"][environment], endpoint_config["api_version"]
|
|
32
|
+
)
|
|
33
|
+
endpoints = endpoint_config["endpoints"]
|
|
34
|
+
|
|
35
|
+
self.url_token = urljoin(base_url, endpoints["token"], allow_fragments=True)
|
|
36
|
+
self.url_refresh = urljoin(
|
|
37
|
+
base_url, endpoints["refresh_token"], allow_fragments=True
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
self.username = username
|
|
41
|
+
self.password = password
|
|
42
|
+
self.longterm_token = longterm_token
|
|
43
|
+
self.access_token = None
|
|
44
|
+
self.refresh_token = None
|
|
45
|
+
self.get_token()
|
|
46
|
+
|
|
47
|
+
def get_token(self):
|
|
48
|
+
"""
|
|
49
|
+
Generate token from user input
|
|
50
|
+
by default, we use the values given as args,
|
|
51
|
+
then in second choice, we try the environment variables,
|
|
52
|
+
and finally we ask for user input
|
|
53
|
+
Priority is given to the long term token if it is provided at the same place (args, environment variable or prompt) than the username/password )
|
|
54
|
+
"""
|
|
55
|
+
if self.username is None and self.password is None:
|
|
56
|
+
"""We do not ignore the values username and password in favor of long term token if the two credentials are given explicitly when instancing Datalake"""
|
|
57
|
+
self.longterm_token = self.longterm_token or os.getenv(
|
|
58
|
+
"OCD_DTL_LONGTERM_TOKEN"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
self.username = self.username or os.getenv("OCD_DTL_USERNAME")
|
|
62
|
+
self.password = self.password or os.getenv("OCD_DTL_PASSWORD")
|
|
63
|
+
|
|
64
|
+
if self.longterm_token is None and (
|
|
65
|
+
self.username is None or self.password is None
|
|
66
|
+
):
|
|
67
|
+
"""I only ask in prompt if no other choice"""
|
|
68
|
+
self.username = self.username or input("Email: ")
|
|
69
|
+
self.password = self.password or getpass()
|
|
70
|
+
self.longterm_token = self.longterm_token or input(
|
|
71
|
+
"Long term token (Default : None): "
|
|
72
|
+
)
|
|
73
|
+
print()
|
|
74
|
+
|
|
75
|
+
if self.longterm_token:
|
|
76
|
+
self.longterm_token = f"Token {self.longterm_token}"
|
|
77
|
+
if self.username or self.password:
|
|
78
|
+
logger.warning(
|
|
79
|
+
f"Using provided Long Term Token for Authentication to the Datalake API. Ignoring username and/or password."
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
data = {"email": self.username, "password": self.password}
|
|
83
|
+
|
|
84
|
+
response = requests.post(url=self.url_token, json=data)
|
|
85
|
+
json_response = json.loads(response.text)
|
|
86
|
+
try:
|
|
87
|
+
self.access_token = f'Token {json_response["access_token"]}'
|
|
88
|
+
self.refresh_token = f'Token {json_response["refresh_token"]}'
|
|
89
|
+
except KeyError:
|
|
90
|
+
logger.error(
|
|
91
|
+
f"An error occurred while retrieving an access token, for URL: {self.url_token}\n"
|
|
92
|
+
f"response of the API: {response.text}"
|
|
93
|
+
)
|
|
94
|
+
raise ValueError(f"Could not login: {response.text}")
|
|
95
|
+
|
|
96
|
+
def fetch_new_token(self):
|
|
97
|
+
logger.debug("Token will be refreshed")
|
|
98
|
+
headers = {"Authorization": self.refresh_token}
|
|
99
|
+
response = requests.post(url=self.url_refresh, headers=headers)
|
|
100
|
+
|
|
101
|
+
json_response = response.json()
|
|
102
|
+
if (
|
|
103
|
+
response.status_code == 401
|
|
104
|
+
and json_response.get("msg") == "Token has expired"
|
|
105
|
+
):
|
|
106
|
+
logger.info("Refreshing the refresh token")
|
|
107
|
+
# Refresh token is also expired, we need to restart the authentication from scratch
|
|
108
|
+
self.get_token()
|
|
109
|
+
elif "access_token" in json_response:
|
|
110
|
+
self.access_token = f'Token {json_response["access_token"]}'
|
|
111
|
+
else: # an error occurred
|
|
112
|
+
logger.error(
|
|
113
|
+
f"An error occurred while refreshing the refresh token, for URL: {self.url_refresh}\n"
|
|
114
|
+
f"response of the API: {response.text}"
|
|
115
|
+
)
|
|
116
|
+
raise ValueError(f"Could not refresh the token: {response.text}")
|
|
117
|
+
|
|
118
|
+
def process_auth_error(self, json_resp: dict):
|
|
119
|
+
"""
|
|
120
|
+
Allow to update token when API response is either Missing Authorization Header or Token has expired.
|
|
121
|
+
"""
|
|
122
|
+
error_msg = get_error_message(json_resp)
|
|
123
|
+
if error_msg in (
|
|
124
|
+
"Missing Authorization Header",
|
|
125
|
+
"Bad Authorization header. Expected value 'Token <JWT>'",
|
|
126
|
+
"Missing 'Token' type in 'Authorization' header. Expected 'Authorization: Token <JWT>'",
|
|
127
|
+
):
|
|
128
|
+
self.get_token()
|
|
129
|
+
elif error_msg == "Token has expired":
|
|
130
|
+
if self.longterm_token:
|
|
131
|
+
raise ValueError(f"Long term token has expired")
|
|
132
|
+
else:
|
|
133
|
+
self.fetch_new_token()
|
|
134
|
+
elif error_msg == "Token has been revoked":
|
|
135
|
+
raise ValueError(f"Long term token has been revoked")
|
|
136
|
+
elif error_msg == "Fresh token required":
|
|
137
|
+
raise ValueError(
|
|
138
|
+
f"You cannot use Long term token with this endpoint, please use only the credentials username and password to init the Datalake instance for this request"
|
|
139
|
+
)
|
|
140
|
+
elif (
|
|
141
|
+
error_msg
|
|
142
|
+
in ("Invalid token", "Not enough segments", "Signature verification failed")
|
|
143
|
+
and self.longterm_token
|
|
144
|
+
):
|
|
145
|
+
raise ValueError(f"Long term token is invalid")
|
|
146
|
+
else:
|
|
147
|
+
raise ValueError(f"Unexpected msg: {error_msg}")
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import sys
|
|
2
3
|
|
|
3
4
|
from datalake import AtomType, SightingType, Visibility, Output
|
|
4
5
|
from datalake.common.config import Config
|
|
@@ -18,21 +19,38 @@ class Datalake:
|
|
|
18
19
|
|
|
19
20
|
Usage:
|
|
20
21
|
>>> dtl = Datalake(username='some username', password='some password')
|
|
22
|
+
or
|
|
23
|
+
>>> dtl = Datalake(longterm_token='some longterm token')
|
|
24
|
+
then
|
|
21
25
|
>>> dtl.Threats.lookup(atom_value='mayoclinic.org', atom_type=AtomType.DOMAIN, hashkey_only=False)
|
|
26
|
+
|
|
22
27
|
"""
|
|
23
28
|
|
|
24
29
|
def __init__(
|
|
25
30
|
self,
|
|
26
31
|
username: str = None,
|
|
27
32
|
password: str = None,
|
|
33
|
+
longterm_token: str = None,
|
|
28
34
|
env="prod",
|
|
29
35
|
log_level=logging.WARNING,
|
|
30
36
|
):
|
|
31
37
|
configure_logging(log_level)
|
|
32
38
|
endpoint_config = Config().load_config()
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
39
|
+
try:
|
|
40
|
+
token_manager = TokenManager(
|
|
41
|
+
endpoint_config,
|
|
42
|
+
environment=env,
|
|
43
|
+
username=username,
|
|
44
|
+
password=password,
|
|
45
|
+
longterm_token=longterm_token,
|
|
46
|
+
)
|
|
47
|
+
except Exception as e:
|
|
48
|
+
if "Failed to resolve" in str(e) or "Failed to establish" in str(e):
|
|
49
|
+
raise ConnectionError(
|
|
50
|
+
"Unable to access Datalake. Please check your network settings/connection"
|
|
51
|
+
)
|
|
52
|
+
else:
|
|
53
|
+
raise
|
|
36
54
|
|
|
37
55
|
self.Threats = Threats(endpoint_config, env, token_manager)
|
|
38
56
|
self.BulkSearch = BulkSearch(endpoint_config, env, token_manager)
|
|
@@ -65,7 +65,9 @@ class Endpoint:
|
|
|
65
65
|
tries_left = self.SET_MAX_RETRY
|
|
66
66
|
|
|
67
67
|
while tries_left > 0:
|
|
68
|
-
headers["Authorization"] =
|
|
68
|
+
headers["Authorization"] = (
|
|
69
|
+
self.token_manager.access_token or self.token_manager.longterm_token
|
|
70
|
+
)
|
|
69
71
|
logger.debug(self._pretty_debug_request(url, method, post_body, headers))
|
|
70
72
|
|
|
71
73
|
response = self._send_request(
|
|
@@ -77,7 +79,7 @@ class Endpoint:
|
|
|
77
79
|
logger.debug("API response:\n%s", response.text)
|
|
78
80
|
if response.status_code == 401:
|
|
79
81
|
logger.warning(
|
|
80
|
-
"
|
|
82
|
+
"Missing authorization header or Token Error. Updating token"
|
|
81
83
|
)
|
|
82
84
|
self.token_manager.process_auth_error(response.json())
|
|
83
85
|
elif response.status_code == 422:
|
|
@@ -148,6 +150,7 @@ class Endpoint:
|
|
|
148
150
|
+ f" - data: \n{data}\n"
|
|
149
151
|
+ f" - token: \n{self.token_manager.access_token}\n"
|
|
150
152
|
+ f" - refresh_token: \n{self.token_manager.refresh_token}\n"
|
|
153
|
+
+ f" - longterm_token: \n{self.token_manager.longterm_token}\n"
|
|
151
154
|
+ "-" * self.terminal_size
|
|
152
155
|
)
|
|
153
156
|
return debug
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake/endpoints/filtered_tag_subcategory.py
RENAMED
|
@@ -73,7 +73,18 @@ class FilteredTagSubcategory(Endpoint):
|
|
|
73
73
|
)
|
|
74
74
|
if limit_match:
|
|
75
75
|
print("The 'limit' parameter must be >= 0 and <= 5000.")
|
|
76
|
-
|
|
76
|
+
token_match = re.search(
|
|
77
|
+
r"token",
|
|
78
|
+
error_message,
|
|
79
|
+
)
|
|
80
|
+
if token_match:
|
|
81
|
+
raise ve
|
|
82
|
+
if (
|
|
83
|
+
not tag_category_match
|
|
84
|
+
and not ordering_match
|
|
85
|
+
and not limit_match
|
|
86
|
+
and not token_match
|
|
87
|
+
):
|
|
77
88
|
print(
|
|
78
89
|
"An error occurred, but no specific pattern was matched in the 422 HTTP error message."
|
|
79
90
|
)
|
|
@@ -50,7 +50,8 @@ class BaseEngine:
|
|
|
50
50
|
"""
|
|
51
51
|
try:
|
|
52
52
|
response = self.endpoint.datalake_requests(url, method, headers, post_body)
|
|
53
|
-
except ValueError:
|
|
53
|
+
except ValueError as ve:
|
|
54
|
+
logger.error(f"Error while attempting to request datalake : {ve}")
|
|
54
55
|
logger.error("Request failed: Will return nothing for this request")
|
|
55
56
|
return {}
|
|
56
57
|
if (
|
|
@@ -47,7 +47,10 @@ class HandleBulkTaskMixin(BaseEngine):
|
|
|
47
47
|
|
|
48
48
|
json_response = None
|
|
49
49
|
while not json_response:
|
|
50
|
-
headers = {
|
|
50
|
+
headers = {
|
|
51
|
+
"Authorization": self.token_manager.access_token
|
|
52
|
+
or self.token_manager.longterm_token
|
|
53
|
+
}
|
|
51
54
|
response = requests.get(
|
|
52
55
|
url=retrieve_bulk_result_url,
|
|
53
56
|
headers=headers,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.3
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -57,10 +57,20 @@ You can also use a script directly by using the following command: `<script_name
|
|
|
57
57
|
## Environment variables
|
|
58
58
|
|
|
59
59
|
#### Authentication
|
|
60
|
-
|
|
60
|
+
|
|
61
|
+
There are two methods of authentication:
|
|
62
|
+
- The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
|
|
63
|
+
- The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
|
|
64
|
+
|
|
65
|
+
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
66
|
+
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
67
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
|
|
68
|
+
|
|
69
|
+
or
|
|
70
|
+
|
|
61
71
|
* `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
|
|
62
72
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
63
|
-
>
|
|
73
|
+
> These last two are independent and one can be used without the other if you wish.
|
|
64
74
|
|
|
65
75
|
#### Throttling
|
|
66
76
|
For throttling the request, those two environment variable can be used:
|
|
@@ -49,3 +49,9 @@ def datalake():
|
|
|
49
49
|
|
|
50
50
|
responses.add(responses.POST, url, json=auth_response, status=200)
|
|
51
51
|
return Datalake(username="username", password="password")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.fixture
|
|
55
|
+
@responses.activate
|
|
56
|
+
def datalake_longterm_token():
|
|
57
|
+
return Datalake(longterm_token="longterm_token1234")
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from functools import partial
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
import responses
|
|
7
|
+
|
|
8
|
+
from datalake import Datalake, AtomType
|
|
9
|
+
from tests.common.fixture import (
|
|
10
|
+
datalake,
|
|
11
|
+
datalake_longterm_token,
|
|
12
|
+
) # noqa needed fixture import
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@responses.activate
|
|
16
|
+
def test_token_auth(datalake):
|
|
17
|
+
expected_tokens = {
|
|
18
|
+
"longterm_token": None,
|
|
19
|
+
"access_token": "12345",
|
|
20
|
+
"refresh_token": "123456",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
token_manager = datalake.Threats.token_manager
|
|
24
|
+
|
|
25
|
+
assert token_manager.longterm_token == None
|
|
26
|
+
assert token_manager.access_token == f"Token {expected_tokens['access_token']}"
|
|
27
|
+
assert token_manager.refresh_token == f"Token {expected_tokens['refresh_token']}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@responses.activate
|
|
31
|
+
def test_invalid_credentials(caplog):
|
|
32
|
+
url = "https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/"
|
|
33
|
+
|
|
34
|
+
api_error_msg = "Wrong credentials provided"
|
|
35
|
+
api_response = {"message": api_error_msg}
|
|
36
|
+
responses.add(responses.POST, url, json=api_response, status=401)
|
|
37
|
+
with caplog.at_level(logging.ERROR):
|
|
38
|
+
with pytest.raises(ValueError) as ve:
|
|
39
|
+
Datalake(username="username@wow.com", password="password")
|
|
40
|
+
assert str(ve.value) == f'Could not login: {{"message": "{api_error_msg}"}}'
|
|
41
|
+
assert caplog.messages == [
|
|
42
|
+
f"An error occurred while retrieving an access token, for URL: {url}\n"
|
|
43
|
+
f'response of the API: {{"message": "{api_error_msg}"}}'
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def lookup_callback(
|
|
48
|
+
request,
|
|
49
|
+
expired_token,
|
|
50
|
+
valid_token,
|
|
51
|
+
response_on_valid_token,
|
|
52
|
+
response_on_expired_token=None,
|
|
53
|
+
):
|
|
54
|
+
headers = {}
|
|
55
|
+
if not response_on_expired_token:
|
|
56
|
+
response_on_expired_token = {"msg": "Token has expired"} # Default value
|
|
57
|
+
if request.headers["Authorization"] == f"Token {expired_token}":
|
|
58
|
+
return 401, headers, json.dumps(response_on_expired_token)
|
|
59
|
+
elif request.headers["Authorization"] == f"Token {valid_token}":
|
|
60
|
+
return 200, headers, json.dumps(response_on_valid_token)
|
|
61
|
+
else:
|
|
62
|
+
raise Exception()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@responses.activate
|
|
66
|
+
def test_access_token_expired(datalake, caplog):
|
|
67
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
68
|
+
expected_json = {"wow.com": "bad"}
|
|
69
|
+
|
|
70
|
+
responses.add_callback(
|
|
71
|
+
responses.GET,
|
|
72
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
73
|
+
callback=partial(
|
|
74
|
+
lookup_callback,
|
|
75
|
+
expired_token="12345",
|
|
76
|
+
valid_token="refreshed_token",
|
|
77
|
+
response_on_valid_token=expected_json,
|
|
78
|
+
),
|
|
79
|
+
content_type="application/json",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def refresh_token_callback(request):
|
|
83
|
+
headers = {}
|
|
84
|
+
assert (
|
|
85
|
+
request.headers["Authorization"] == "Token 123456"
|
|
86
|
+
), "token passed is not the refresh token"
|
|
87
|
+
return 200, headers, json.dumps({"access_token": "refreshed_token"})
|
|
88
|
+
|
|
89
|
+
responses.add_callback(
|
|
90
|
+
responses.POST,
|
|
91
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/refresh-token/",
|
|
92
|
+
callback=refresh_token_callback,
|
|
93
|
+
content_type="application/json",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
assert (
|
|
97
|
+
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
98
|
+
== expected_json
|
|
99
|
+
)
|
|
100
|
+
assert caplog.messages == [
|
|
101
|
+
"Missing authorization header or Token Error. Updating token"
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@responses.activate
|
|
106
|
+
def test_refresh_token_expired(datalake, caplog):
|
|
107
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
108
|
+
expected_json = {"wow.com": "bad"}
|
|
109
|
+
|
|
110
|
+
responses.add_callback(
|
|
111
|
+
responses.GET,
|
|
112
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
113
|
+
callback=partial(
|
|
114
|
+
lookup_callback,
|
|
115
|
+
expired_token="12345",
|
|
116
|
+
valid_token="new_token",
|
|
117
|
+
response_on_valid_token=expected_json,
|
|
118
|
+
),
|
|
119
|
+
content_type="application/json",
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def refresh_token_callback(request):
|
|
123
|
+
headers = {}
|
|
124
|
+
assert (
|
|
125
|
+
request.headers["Authorization"] == "Token 123456"
|
|
126
|
+
), "token passed is not the refresh token"
|
|
127
|
+
return 401, headers, json.dumps({"msg": "Token has expired"})
|
|
128
|
+
|
|
129
|
+
responses.add_callback(
|
|
130
|
+
responses.POST,
|
|
131
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/refresh-token/",
|
|
132
|
+
callback=refresh_token_callback,
|
|
133
|
+
content_type="application/json",
|
|
134
|
+
)
|
|
135
|
+
responses.post(
|
|
136
|
+
url="https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/",
|
|
137
|
+
json={"access_token": "new_token", "refresh_token": ""},
|
|
138
|
+
status=200,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
assert (
|
|
142
|
+
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
143
|
+
== expected_json
|
|
144
|
+
)
|
|
145
|
+
assert caplog.messages == [
|
|
146
|
+
"Missing authorization header or Token Error. Updating token",
|
|
147
|
+
"Refreshing the refresh token",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@responses.activate
|
|
152
|
+
def test_invalid_token(datalake, caplog):
|
|
153
|
+
expected_json = {"wow.com": "bad"}
|
|
154
|
+
invalid_access_token = "not-valid-token"
|
|
155
|
+
valid_access_token = "valid_token"
|
|
156
|
+
|
|
157
|
+
responses.add_callback(
|
|
158
|
+
responses.GET,
|
|
159
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
160
|
+
callback=partial(
|
|
161
|
+
lookup_callback,
|
|
162
|
+
expired_token=invalid_access_token,
|
|
163
|
+
valid_token=valid_access_token,
|
|
164
|
+
response_on_valid_token=expected_json,
|
|
165
|
+
response_on_expired_token={
|
|
166
|
+
"msg": "Missing 'Token' type in 'Authorization' header. Expected 'Authorization: Token <JWT>'"
|
|
167
|
+
},
|
|
168
|
+
),
|
|
169
|
+
content_type="application/json",
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
responses.post(
|
|
173
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/",
|
|
174
|
+
json={"access_token": valid_access_token, "refresh_token": ""},
|
|
175
|
+
content_type="application/json",
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
datalake.Threats.token_manager.access_token = f"Token {invalid_access_token}"
|
|
179
|
+
|
|
180
|
+
assert (
|
|
181
|
+
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
182
|
+
== expected_json
|
|
183
|
+
)
|
|
184
|
+
assert caplog.messages == [
|
|
185
|
+
"Missing authorization header or Token Error. Updating token"
|
|
186
|
+
]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def test_active_longterm_token(datalake_longterm_token):
|
|
190
|
+
token_manager = datalake_longterm_token.Threats.token_manager
|
|
191
|
+
expected_tokens = {
|
|
192
|
+
"longterm_token": "longterm_token1234",
|
|
193
|
+
"access_token": None,
|
|
194
|
+
"refresh_token": None,
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
assert token_manager.longterm_token == f"Token {expected_tokens['longterm_token']}"
|
|
198
|
+
assert token_manager.access_token == None
|
|
199
|
+
assert token_manager.refresh_token == None
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def lookup_callback_longterm_token(
|
|
203
|
+
request,
|
|
204
|
+
expired_longterm_token,
|
|
205
|
+
disabled_longterm_token,
|
|
206
|
+
not_fresh_token,
|
|
207
|
+
response_on_expired_longterm_token={"msg": "Token has expired"},
|
|
208
|
+
response_on_disabled_longterm_token={"msg": "Token has been revoked"},
|
|
209
|
+
response_on_not_fresh_token={"msg": "Fresh token required"},
|
|
210
|
+
response_on_invalid_token={"msg": "Invalid token"},
|
|
211
|
+
):
|
|
212
|
+
headers = {}
|
|
213
|
+
if request.headers["Authorization"] == f"Token {expired_longterm_token}":
|
|
214
|
+
return 401, headers, json.dumps(response_on_expired_longterm_token)
|
|
215
|
+
elif request.headers["Authorization"] == f"Token {disabled_longterm_token}":
|
|
216
|
+
return 401, headers, json.dumps(response_on_disabled_longterm_token)
|
|
217
|
+
elif request.headers["Authorization"] == f"Token {not_fresh_token}":
|
|
218
|
+
return 401, headers, json.dumps(response_on_not_fresh_token)
|
|
219
|
+
else:
|
|
220
|
+
return 401, headers, json.dumps(response_on_invalid_token)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@responses.activate
|
|
224
|
+
def test_disabled_longterm_token(datalake_longterm_token, caplog):
|
|
225
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
226
|
+
|
|
227
|
+
responses.add_callback(
|
|
228
|
+
responses.GET,
|
|
229
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
230
|
+
callback=partial(
|
|
231
|
+
lookup_callback_longterm_token,
|
|
232
|
+
expired_longterm_token="expired_longterm_token_1234",
|
|
233
|
+
disabled_longterm_token="longterm_token1234",
|
|
234
|
+
not_fresh_token="not_fresh_token_1234",
|
|
235
|
+
),
|
|
236
|
+
content_type="application/json",
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
with caplog.at_level(logging.WARNING):
|
|
240
|
+
with pytest.raises(ValueError) as ve:
|
|
241
|
+
r = datalake_longterm_token.Threats.lookup(
|
|
242
|
+
atom_value="wow.com", atom_type=AtomType.DOMAIN
|
|
243
|
+
)
|
|
244
|
+
assert str(ve.value) == "Long term token has been revoked"
|
|
245
|
+
assert caplog.messages == [
|
|
246
|
+
"Missing authorization header or Token Error. Updating token"
|
|
247
|
+
]
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@responses.activate
|
|
251
|
+
def test_expired_longterm_token(datalake_longterm_token, caplog):
|
|
252
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
253
|
+
|
|
254
|
+
responses.add_callback(
|
|
255
|
+
responses.GET,
|
|
256
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
257
|
+
callback=partial(
|
|
258
|
+
lookup_callback_longterm_token,
|
|
259
|
+
expired_longterm_token="longterm_token1234",
|
|
260
|
+
disabled_longterm_token="disabled_token1234",
|
|
261
|
+
not_fresh_token="not_fresh_token_1234",
|
|
262
|
+
),
|
|
263
|
+
content_type="application/json",
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
with caplog.at_level(logging.WARNING):
|
|
267
|
+
with pytest.raises(ValueError) as ve:
|
|
268
|
+
r = datalake_longterm_token.Threats.lookup(
|
|
269
|
+
atom_value="wow.com", atom_type=AtomType.DOMAIN
|
|
270
|
+
)
|
|
271
|
+
assert str(ve.value) == "Long term token has expired"
|
|
272
|
+
assert caplog.messages == [
|
|
273
|
+
"Missing authorization header or Token Error. Updating token"
|
|
274
|
+
]
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# test long test token on endpoint requiring fresh token
|
|
278
|
+
@responses.activate
|
|
279
|
+
def test_requires_fresh_token(datalake_longterm_token, caplog):
|
|
280
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
281
|
+
|
|
282
|
+
responses.add_callback(
|
|
283
|
+
responses.GET,
|
|
284
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
285
|
+
callback=partial(
|
|
286
|
+
lookup_callback_longterm_token,
|
|
287
|
+
expired_longterm_token="expired_longterm_token_1234",
|
|
288
|
+
disabled_longterm_token="disabled_token1234",
|
|
289
|
+
not_fresh_token="longterm_token1234",
|
|
290
|
+
),
|
|
291
|
+
content_type="application/json",
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
with caplog.at_level(logging.WARNING):
|
|
295
|
+
with pytest.raises(ValueError) as ve:
|
|
296
|
+
r = datalake_longterm_token.Threats.lookup(
|
|
297
|
+
atom_value="wow.com", atom_type=AtomType.DOMAIN
|
|
298
|
+
)
|
|
299
|
+
assert (
|
|
300
|
+
str(ve.value)
|
|
301
|
+
== "You cannot use Long term token with this endpoint, please use only the credentials username and password to init the Datalake instance for this request"
|
|
302
|
+
)
|
|
303
|
+
assert caplog.messages == [
|
|
304
|
+
"Missing authorization header or Token Error. Updating token"
|
|
305
|
+
]
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@responses.activate
|
|
309
|
+
def test_invalid_longterm_token(datalake_longterm_token, caplog):
|
|
310
|
+
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
311
|
+
|
|
312
|
+
responses.add_callback(
|
|
313
|
+
responses.GET,
|
|
314
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
315
|
+
callback=partial(
|
|
316
|
+
lookup_callback_longterm_token,
|
|
317
|
+
expired_longterm_token="expired_longterm_token_1234",
|
|
318
|
+
disabled_longterm_token="disabled_token1234",
|
|
319
|
+
not_fresh_token="not_fresh_token_1234",
|
|
320
|
+
),
|
|
321
|
+
content_type="application/json",
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
with caplog.at_level(logging.WARNING):
|
|
325
|
+
with pytest.raises(ValueError) as ve:
|
|
326
|
+
r = datalake_longterm_token.Threats.lookup(
|
|
327
|
+
atom_value="wow.com", atom_type=AtomType.DOMAIN
|
|
328
|
+
)
|
|
329
|
+
assert str(ve.value) == "Long term token is invalid"
|
|
330
|
+
assert caplog.messages == [
|
|
331
|
+
"Missing authorization header or Token Error. Updating token"
|
|
332
|
+
]
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Token manager will manage tokens for the scripts.
|
|
3
|
-
"""
|
|
4
|
-
import json
|
|
5
|
-
import os
|
|
6
|
-
from getpass import getpass
|
|
7
|
-
from urllib.parse import urljoin
|
|
8
|
-
|
|
9
|
-
import requests
|
|
10
|
-
|
|
11
|
-
from datalake.common.logger import logger
|
|
12
|
-
from datalake.common.utils import get_error_message
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class TokenManager:
|
|
16
|
-
"""
|
|
17
|
-
Use it to generate token access to the API
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
def __init__(
|
|
21
|
-
self, endpoint_config: dict, *, environment: str, username=None, password=None
|
|
22
|
-
):
|
|
23
|
-
"""environment can be either prod or preprod"""
|
|
24
|
-
base_url = urljoin(
|
|
25
|
-
endpoint_config["main"][environment], endpoint_config["api_version"]
|
|
26
|
-
)
|
|
27
|
-
endpoints = endpoint_config["endpoints"]
|
|
28
|
-
|
|
29
|
-
self.url_token = urljoin(base_url, endpoints["token"], allow_fragments=True)
|
|
30
|
-
self.url_refresh = urljoin(
|
|
31
|
-
base_url, endpoints["refresh_token"], allow_fragments=True
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
self.username = username
|
|
35
|
-
self.password = password
|
|
36
|
-
self.access_token = None
|
|
37
|
-
self.refresh_token = None
|
|
38
|
-
self.get_token()
|
|
39
|
-
|
|
40
|
-
def get_token(self):
|
|
41
|
-
"""
|
|
42
|
-
Generate token from user input, with email and password
|
|
43
|
-
"""
|
|
44
|
-
self.username = (
|
|
45
|
-
self.username or os.getenv("OCD_DTL_USERNAME") or input("Email: ")
|
|
46
|
-
)
|
|
47
|
-
self.password = self.password or os.getenv("OCD_DTL_PASSWORD") or getpass()
|
|
48
|
-
print()
|
|
49
|
-
data = {"email": self.username, "password": self.password}
|
|
50
|
-
|
|
51
|
-
response = requests.post(url=self.url_token, json=data)
|
|
52
|
-
json_response = json.loads(response.text)
|
|
53
|
-
try:
|
|
54
|
-
self.access_token = f'Token {json_response["access_token"]}'
|
|
55
|
-
self.refresh_token = f'Token {json_response["refresh_token"]}'
|
|
56
|
-
except KeyError:
|
|
57
|
-
logger.error(
|
|
58
|
-
f"An error occurred while retrieving an access token, for URL: {self.url_token}\n"
|
|
59
|
-
f"response of the API: {response.text}"
|
|
60
|
-
)
|
|
61
|
-
raise ValueError(f"Could not login: {response.text}")
|
|
62
|
-
|
|
63
|
-
def fetch_new_token(self):
|
|
64
|
-
logger.debug("Token will be refreshed")
|
|
65
|
-
headers = {"Authorization": self.refresh_token}
|
|
66
|
-
response = requests.post(url=self.url_refresh, headers=headers)
|
|
67
|
-
|
|
68
|
-
json_response = response.json()
|
|
69
|
-
if (
|
|
70
|
-
response.status_code == 401
|
|
71
|
-
and json_response.get("msg") == "Token has expired"
|
|
72
|
-
):
|
|
73
|
-
logger.info("Refreshing the refresh token")
|
|
74
|
-
# Refresh token is also expired, we need to restart the authentication from scratch
|
|
75
|
-
self.get_token()
|
|
76
|
-
elif "access_token" in json_response:
|
|
77
|
-
self.access_token = f'Token {json_response["access_token"]}'
|
|
78
|
-
else: # an error occurred
|
|
79
|
-
logger.error(
|
|
80
|
-
f"An error occurred while refreshing the refresh token, for URL: {self.url_refresh}\n"
|
|
81
|
-
f"response of the API: {response.text}"
|
|
82
|
-
)
|
|
83
|
-
raise ValueError(f"Could not refresh the token: {response.text}")
|
|
84
|
-
|
|
85
|
-
def process_auth_error(self, json_resp: dict):
|
|
86
|
-
"""
|
|
87
|
-
Allow to update token when API response is either Missing Authorization Header or Token has expired.
|
|
88
|
-
"""
|
|
89
|
-
error_msg = get_error_message(json_resp)
|
|
90
|
-
if error_msg in (
|
|
91
|
-
"Missing Authorization Header",
|
|
92
|
-
"Bad Authorization header. Expected value 'Token <JWT>'",
|
|
93
|
-
"Missing 'Token' type in 'Authorization' header. Expected 'Authorization: Token <JWT>'",
|
|
94
|
-
):
|
|
95
|
-
self.get_token()
|
|
96
|
-
elif error_msg == "Token has expired":
|
|
97
|
-
self.fetch_new_token()
|
|
98
|
-
else:
|
|
99
|
-
raise ValueError(f"Unexpected msg: {error_msg}")
|
|
@@ -1,181 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import logging
|
|
3
|
-
from functools import partial
|
|
4
|
-
|
|
5
|
-
import pytest
|
|
6
|
-
import responses
|
|
7
|
-
|
|
8
|
-
from datalake import Datalake, AtomType
|
|
9
|
-
from tests.common.fixture import datalake # noqa needed fixture import
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
@responses.activate
|
|
13
|
-
def test_token_auth(datalake):
|
|
14
|
-
expected_tokens = {
|
|
15
|
-
"access_token": "12345",
|
|
16
|
-
"refresh_token": "123456",
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
token_manager = datalake.Threats.token_manager
|
|
20
|
-
|
|
21
|
-
assert token_manager.access_token == f"Token {expected_tokens['access_token']}"
|
|
22
|
-
assert token_manager.refresh_token == f"Token {expected_tokens['refresh_token']}"
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
@responses.activate
|
|
26
|
-
def test_invalid_credentials(caplog):
|
|
27
|
-
url = "https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/"
|
|
28
|
-
|
|
29
|
-
api_error_msg = "Wrong credentials provided"
|
|
30
|
-
api_response = {"message": api_error_msg}
|
|
31
|
-
responses.add(responses.POST, url, json=api_response, status=401)
|
|
32
|
-
with caplog.at_level(logging.ERROR):
|
|
33
|
-
with pytest.raises(ValueError) as ve:
|
|
34
|
-
Datalake(username="username@wow.com", password="password")
|
|
35
|
-
assert str(ve.value) == f'Could not login: {{"message": "{api_error_msg}"}}'
|
|
36
|
-
assert caplog.messages == [
|
|
37
|
-
f"An error occurred while retrieving an access token, for URL: {url}\n"
|
|
38
|
-
f'response of the API: {{"message": "{api_error_msg}"}}'
|
|
39
|
-
]
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def lookup_callback(
|
|
43
|
-
request,
|
|
44
|
-
expired_token,
|
|
45
|
-
valid_token,
|
|
46
|
-
response_on_valid_token,
|
|
47
|
-
response_on_expired_token=None,
|
|
48
|
-
):
|
|
49
|
-
headers = {}
|
|
50
|
-
if not response_on_expired_token:
|
|
51
|
-
response_on_expired_token = {"msg": "Token has expired"} # Default value
|
|
52
|
-
if request.headers["Authorization"] == f"Token {expired_token}":
|
|
53
|
-
return 401, headers, json.dumps(response_on_expired_token)
|
|
54
|
-
elif request.headers["Authorization"] == f"Token {valid_token}":
|
|
55
|
-
return 200, headers, json.dumps(response_on_valid_token)
|
|
56
|
-
else:
|
|
57
|
-
raise Exception()
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
@responses.activate
|
|
61
|
-
def test_access_token_expired(datalake, caplog):
|
|
62
|
-
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
63
|
-
expected_json = {"wow.com": "bad"}
|
|
64
|
-
|
|
65
|
-
responses.add_callback(
|
|
66
|
-
responses.GET,
|
|
67
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
68
|
-
callback=partial(
|
|
69
|
-
lookup_callback,
|
|
70
|
-
expired_token="12345",
|
|
71
|
-
valid_token="refreshed_token",
|
|
72
|
-
response_on_valid_token=expected_json,
|
|
73
|
-
),
|
|
74
|
-
content_type="application/json",
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
def refresh_token_callback(request):
|
|
78
|
-
headers = {}
|
|
79
|
-
assert (
|
|
80
|
-
request.headers["Authorization"] == "Token 123456"
|
|
81
|
-
), "token passed is not the refresh token"
|
|
82
|
-
return 200, headers, json.dumps({"access_token": "refreshed_token"})
|
|
83
|
-
|
|
84
|
-
responses.add_callback(
|
|
85
|
-
responses.POST,
|
|
86
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/refresh-token/",
|
|
87
|
-
callback=refresh_token_callback,
|
|
88
|
-
content_type="application/json",
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
assert (
|
|
92
|
-
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
93
|
-
== expected_json
|
|
94
|
-
)
|
|
95
|
-
assert caplog.messages == [
|
|
96
|
-
"Token expired or Missing authorization header. Updating token"
|
|
97
|
-
]
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
@responses.activate
|
|
101
|
-
def test_refresh_token_expired(datalake, caplog):
|
|
102
|
-
caplog.set_level(level=logging.INFO, logger="OCD_DTL")
|
|
103
|
-
expected_json = {"wow.com": "bad"}
|
|
104
|
-
|
|
105
|
-
responses.add_callback(
|
|
106
|
-
responses.GET,
|
|
107
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
108
|
-
callback=partial(
|
|
109
|
-
lookup_callback,
|
|
110
|
-
expired_token="12345",
|
|
111
|
-
valid_token="new_token",
|
|
112
|
-
response_on_valid_token=expected_json,
|
|
113
|
-
),
|
|
114
|
-
content_type="application/json",
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
def refresh_token_callback(request):
|
|
118
|
-
headers = {}
|
|
119
|
-
assert (
|
|
120
|
-
request.headers["Authorization"] == "Token 123456"
|
|
121
|
-
), "token passed is not the refresh token"
|
|
122
|
-
return 401, headers, json.dumps({"msg": "Token has expired"})
|
|
123
|
-
|
|
124
|
-
responses.add_callback(
|
|
125
|
-
responses.POST,
|
|
126
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/refresh-token/",
|
|
127
|
-
callback=refresh_token_callback,
|
|
128
|
-
content_type="application/json",
|
|
129
|
-
)
|
|
130
|
-
responses.post(
|
|
131
|
-
url="https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/",
|
|
132
|
-
json={"access_token": "new_token", "refresh_token": ""},
|
|
133
|
-
status=200,
|
|
134
|
-
)
|
|
135
|
-
|
|
136
|
-
assert (
|
|
137
|
-
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
138
|
-
== expected_json
|
|
139
|
-
)
|
|
140
|
-
assert caplog.messages == [
|
|
141
|
-
"Token expired or Missing authorization header. Updating token",
|
|
142
|
-
"Refreshing the refresh token",
|
|
143
|
-
]
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
@responses.activate
|
|
147
|
-
def test_invalid_token(datalake, caplog):
|
|
148
|
-
expected_json = {"wow.com": "bad"}
|
|
149
|
-
invalid_access_token = "not-valid-token"
|
|
150
|
-
valid_access_token = "valid_token"
|
|
151
|
-
|
|
152
|
-
responses.add_callback(
|
|
153
|
-
responses.GET,
|
|
154
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/threats/lookup/",
|
|
155
|
-
callback=partial(
|
|
156
|
-
lookup_callback,
|
|
157
|
-
expired_token=invalid_access_token,
|
|
158
|
-
valid_token=valid_access_token,
|
|
159
|
-
response_on_valid_token=expected_json,
|
|
160
|
-
response_on_expired_token={
|
|
161
|
-
"msg": "Missing 'Token' type in 'Authorization' header. Expected 'Authorization: Token <JWT>'"
|
|
162
|
-
},
|
|
163
|
-
),
|
|
164
|
-
content_type="application/json",
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
responses.post(
|
|
168
|
-
"https://datalake.cert.orangecyberdefense.com/api/v2/auth/token/",
|
|
169
|
-
json={"access_token": valid_access_token, "refresh_token": ""},
|
|
170
|
-
content_type="application/json",
|
|
171
|
-
)
|
|
172
|
-
|
|
173
|
-
datalake.Threats.token_manager.access_token = f"Token {invalid_access_token}"
|
|
174
|
-
|
|
175
|
-
assert (
|
|
176
|
-
datalake.Threats.lookup(atom_value="wow.com", atom_type=AtomType.DOMAIN)
|
|
177
|
-
== expected_json
|
|
178
|
-
)
|
|
179
|
-
assert caplog.messages == [
|
|
180
|
-
"Token expired or Missing authorization header. Updating token"
|
|
181
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/helper_scripts/output_builder.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/bulk_lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/get_threats_by_hashkey.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/datalake_scripts.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.1 → datalake-scripts-2.7.3}/tests/lib/test_filtered_tag_subcategory.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|