datalake-scripts 2.7.2__tar.gz → 2.7.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/PKG-INFO +37 -4
  2. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/README.md +36 -3
  3. datalake-scripts-2.7.4/datalake/common/token_manager.py +158 -0
  4. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/datalake.py +30 -7
  5. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/endpoint.py +30 -5
  6. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/filtered_tag_subcategory.py +12 -1
  7. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/cli.py +1 -1
  8. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/common/base_engine.py +17 -4
  9. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/common/mixins.py +5 -1
  10. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/PKG-INFO +37 -4
  11. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/common/fixture.py +6 -0
  12. datalake-scripts-2.7.4/tests/lib/test_auth_via_tokens.py +332 -0
  13. datalake-scripts-2.7.2/datalake/common/token_manager.py +0 -99
  14. datalake-scripts-2.7.2/tests/lib/test_auth_via_tokens.py +0 -181
  15. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/__init__.py +0 -0
  16. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/api_objects/__init__.py +0 -0
  17. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/api_objects/bulk_search_task.py +0 -0
  18. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/__init__.py +0 -0
  19. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/atom.py +0 -0
  20. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/atom_type.py +0 -0
  21. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/config.py +0 -0
  22. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/logger.py +0 -0
  23. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/output.py +0 -0
  24. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/throttler.py +0 -0
  25. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/utils.py +0 -0
  26. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/common/warn.py +0 -0
  27. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/__init__.py +0 -0
  28. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/advanced_search.py +0 -0
  29. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/bulk_search.py +0 -0
  30. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/sightings.py +0 -0
  31. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/tags.py +0 -0
  32. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/endpoints/threats.py +0 -0
  33. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/miscellaneous/__init__.py +0 -0
  34. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake/miscellaneous/search_watch.py +0 -0
  35. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/__init__.py +0 -0
  36. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/common/__init__.py +0 -0
  37. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/common/base_script.py +0 -0
  38. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/engines/__init__.py +0 -0
  39. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/engines/get_engine.py +0 -0
  40. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/engines/post_engine.py +0 -0
  41. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/__init__.py +0 -0
  42. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  43. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/utils.py +0 -0
  44. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/__init__.py +0 -0
  45. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_comments.py +0 -0
  46. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_tags.py +0 -0
  47. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_threats.py +0 -0
  48. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/advanced_search.py +0 -0
  49. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  50. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/edit_score.py +0 -0
  51. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_atom_values.py +0 -0
  52. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -0
  53. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_query_hash.py +0 -0
  54. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
  55. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  56. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/lookup_threats.py +0 -0
  57. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts/scripts/search_watch.py +0 -0
  58. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/SOURCES.txt +0 -0
  59. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  60. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/entry_points.txt +0 -0
  61. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/requires.txt +0 -0
  62. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/top_level.txt +0 -0
  63. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/setup.cfg +0 -0
  64. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/setup.py +0 -0
  65. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/__init__.py +0 -0
  66. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/common/__init__.py +0 -0
  67. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/common/test_output.py +0 -0
  68. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/common/test_utils.py +0 -0
  69. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/conftest.py +0 -0
  70. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/__init__.py +0 -0
  71. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_advanced_search.py +0 -0
  72. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_atom_type.py +0 -0
  73. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_bulk_search.py +0 -0
  74. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_filtered_tag_subcategory.py +0 -0
  75. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_search_watch.py +0 -0
  76. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_sightings.py +0 -0
  77. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_tags.py +0 -0
  78. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/lib/test_threats.py +0 -0
  79. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/__init__.py +0 -0
  80. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/test_add_new_threats.py +0 -0
  81. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/test_base_engine.py +0 -0
  82. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/test_cli.py +0 -0
  83. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/test_csv_builder.py +0 -0
  84. {datalake-scripts-2.7.2 → datalake-scripts-2.7.4}/tests/scripts/test_endpoint_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.7.2
3
+ Version: 2.7.4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
40
40
  $ pip3 install datalake-scripts
41
41
  ```
42
42
  ### Using as a library
43
- see [the following link](tutorial.md)
43
+ ```python
44
+ from datalake import Datalake, AtomType, Output
45
+
46
+ dtl = Datalake(username='username', password='password')
47
+ dtl.Threats.lookup(
48
+ atom_value='mayoclinic.org',
49
+ atom_type=AtomType.DOMAIN,
50
+ hashkey_only=False,
51
+ output=Output.JSON
52
+ )
53
+ ```
54
+
55
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
44
56
 
45
57
  ### Using as a CLI
46
58
 
@@ -57,10 +69,31 @@ You can also use a script directly by using the following command: `<script_name
57
69
  ## Environment variables
58
70
 
59
71
  #### Authentication
60
- In case you don't want to enter credential for each commands and you are on a secured terminal, set those variables:
72
+
73
+ There are two methods of authentication:
74
+ - The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
75
+ - The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
76
+
77
+ In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
78
+ * `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
79
+ Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
80
+
81
+ or
82
+
61
83
  * `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
62
84
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
63
- > They are independent and one can be used without the other if you wish.
85
+ > These last two are independent and one can be used without the other if you wish.
86
+
87
+ #### Using a Proxy
88
+
89
+ You can set up following environment variables :
90
+
91
+ * `HTTP_PROXY`
92
+ * `HTTPS_PROXY`
93
+
94
+ We use the format accepted by the requests python library.
95
+ See its documenation for other possible kinds of proxy to set up.
96
+
64
97
 
65
98
  #### Throttling
66
99
  For throttling the request, those two environment variable can be used:
@@ -20,7 +20,19 @@ $ pip install datalake-scripts
20
20
  $ pip3 install datalake-scripts
21
21
  ```
22
22
  ### Using as a library
23
- see [the following link](tutorial.md)
23
+ ```python
24
+ from datalake import Datalake, AtomType, Output
25
+
26
+ dtl = Datalake(username='username', password='password')
27
+ dtl.Threats.lookup(
28
+ atom_value='mayoclinic.org',
29
+ atom_type=AtomType.DOMAIN,
30
+ hashkey_only=False,
31
+ output=Output.JSON
32
+ )
33
+ ```
34
+
35
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
24
36
 
25
37
  ### Using as a CLI
26
38
 
@@ -37,10 +49,31 @@ You can also use a script directly by using the following command: `<script_name
37
49
  ## Environment variables
38
50
 
39
51
  #### Authentication
40
- In case you don't want to enter credential for each commands and you are on a secured terminal, set those variables:
52
+
53
+ There are two methods of authentication:
54
+ - The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
55
+ - The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
56
+
57
+ In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
58
+ * `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
59
+ Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
60
+
61
+ or
62
+
41
63
  * `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
42
64
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
43
- > They are independent and one can be used without the other if you wish.
65
+ > These last two are independent and one can be used without the other if you wish.
66
+
67
+ #### Using a Proxy
68
+
69
+ You can set up following environment variables :
70
+
71
+ * `HTTP_PROXY`
72
+ * `HTTPS_PROXY`
73
+
74
+ We use the format accepted by the requests python library.
75
+ See its documenation for other possible kinds of proxy to set up.
76
+
44
77
 
45
78
  #### Throttling
46
79
  For throttling the request, those two environment variable can be used:
@@ -0,0 +1,158 @@
1
+ """
2
+ Token manager will manage tokens for the scripts.
3
+ """
4
+ import json
5
+ import os
6
+ from getpass import getpass
7
+ from urllib.parse import urljoin
8
+
9
+ import requests
10
+
11
+ from datalake.common.logger import logger
12
+ from datalake.common.utils import get_error_message
13
+
14
+
15
+ class TokenManager:
16
+ """
17
+ Use it to generate token access to the API
18
+ """
19
+
20
+ def __init__(
21
+ self,
22
+ endpoint_config: dict,
23
+ *,
24
+ environment: str,
25
+ username=None,
26
+ password=None,
27
+ longterm_token=None,
28
+ proxies: dict = None,
29
+ verify: bool = True,
30
+ ):
31
+ """environment can be either prod or preprod"""
32
+ base_url = urljoin(
33
+ endpoint_config["main"][environment], endpoint_config["api_version"]
34
+ )
35
+ endpoints = endpoint_config["endpoints"]
36
+
37
+ self.url_token = urljoin(base_url, endpoints["token"], allow_fragments=True)
38
+ self.url_refresh = urljoin(
39
+ base_url, endpoints["refresh_token"], allow_fragments=True
40
+ )
41
+
42
+ self.username = username
43
+ self.password = password
44
+ self.longterm_token = longterm_token
45
+ self.proxies = proxies
46
+ self.verify = verify
47
+ self.access_token = None
48
+ self.refresh_token = None
49
+ self.get_token()
50
+
51
+ def get_token(self):
52
+ """
53
+ Generate token from user input
54
+ by default, we use the values given as args,
55
+ then in second choice, we try the environment variables,
56
+ and finally we ask for user input
57
+ Priority is given to the long term token if it is provided at the same place (args, environment variable or prompt) than the username/password )
58
+ """
59
+ if self.username is None and self.password is None:
60
+ """We do not ignore the values username and password in favor of long term token if the two credentials are given explicitly when instancing Datalake"""
61
+ self.longterm_token = self.longterm_token or os.getenv(
62
+ "OCD_DTL_LONGTERM_TOKEN"
63
+ )
64
+
65
+ self.username = self.username or os.getenv("OCD_DTL_USERNAME")
66
+ self.password = self.password or os.getenv("OCD_DTL_PASSWORD")
67
+
68
+ if self.longterm_token is None and (
69
+ self.username is None or self.password is None
70
+ ):
71
+ """I only ask in prompt if no other choice"""
72
+ self.username = self.username or input("Email: ")
73
+ self.password = self.password or getpass()
74
+ self.longterm_token = self.longterm_token or input(
75
+ "Long term token (Default : None): "
76
+ )
77
+ print()
78
+
79
+ if self.longterm_token:
80
+ self.longterm_token = f"Token {self.longterm_token}"
81
+ if self.username or self.password:
82
+ logger.warning(
83
+ f"Using provided Long Term Token for Authentication to the Datalake API. Ignoring username and/or password."
84
+ )
85
+ else:
86
+ data = {"email": self.username, "password": self.password}
87
+
88
+ response = requests.post(
89
+ url=self.url_token, json=data, proxies=self.proxies, verify=self.verify
90
+ )
91
+ json_response = json.loads(response.text)
92
+ try:
93
+ self.access_token = f'Token {json_response["access_token"]}'
94
+ self.refresh_token = f'Token {json_response["refresh_token"]}'
95
+ except KeyError:
96
+ logger.error(
97
+ f"An error occurred while retrieving an access token, for URL: {self.url_token}\n"
98
+ f"response of the API: {response.text}"
99
+ )
100
+ raise ValueError(f"Could not login: {response.text}")
101
+
102
+ def fetch_new_token(self):
103
+ logger.debug("Token will be refreshed")
104
+ headers = {"Authorization": self.refresh_token}
105
+ response = requests.post(
106
+ url=self.url_refresh,
107
+ headers=headers,
108
+ proxies=self.proxies,
109
+ verify=self.verify,
110
+ )
111
+
112
+ json_response = response.json()
113
+ if (
114
+ response.status_code == 401
115
+ and json_response.get("msg") == "Token has expired"
116
+ ):
117
+ logger.info("Refreshing the refresh token")
118
+ # Refresh token is also expired, we need to restart the authentication from scratch
119
+ self.get_token()
120
+ elif "access_token" in json_response:
121
+ self.access_token = f'Token {json_response["access_token"]}'
122
+ else: # an error occurred
123
+ logger.error(
124
+ f"An error occurred while refreshing the refresh token, for URL: {self.url_refresh}\n"
125
+ f"response of the API: {response.text}"
126
+ )
127
+ raise ValueError(f"Could not refresh the token: {response.text}")
128
+
129
+ def process_auth_error(self, json_resp: dict):
130
+ """
131
+ Allow to update token when API response is either Missing Authorization Header or Token has expired.
132
+ """
133
+ error_msg = get_error_message(json_resp)
134
+ if error_msg in (
135
+ "Missing Authorization Header",
136
+ "Bad Authorization header. Expected value 'Token <JWT>'",
137
+ "Missing 'Token' type in 'Authorization' header. Expected 'Authorization: Token <JWT>'",
138
+ ):
139
+ self.get_token()
140
+ elif error_msg == "Token has expired":
141
+ if self.longterm_token:
142
+ raise ValueError(f"Long term token has expired")
143
+ else:
144
+ self.fetch_new_token()
145
+ elif error_msg == "Token has been revoked":
146
+ raise ValueError(f"Long term token has been revoked")
147
+ elif error_msg == "Fresh token required":
148
+ raise ValueError(
149
+ f"You cannot use Long term token with this endpoint, please use only the credentials username and password to init the Datalake instance for this request"
150
+ )
151
+ elif (
152
+ error_msg
153
+ in ("Invalid token", "Not enough segments", "Signature verification failed")
154
+ and self.longterm_token
155
+ ):
156
+ raise ValueError(f"Long term token is invalid")
157
+ else:
158
+ raise ValueError(f"Unexpected msg: {error_msg}")
@@ -19,21 +19,34 @@ class Datalake:
19
19
 
20
20
  Usage:
21
21
  >>> dtl = Datalake(username='some username', password='some password')
22
+ or
23
+ >>> dtl = Datalake(longterm_token='some longterm token')
24
+ then
22
25
  >>> dtl.Threats.lookup(atom_value='mayoclinic.org', atom_type=AtomType.DOMAIN, hashkey_only=False)
26
+
23
27
  """
24
28
 
25
29
  def __init__(
26
30
  self,
27
31
  username: str = None,
28
32
  password: str = None,
33
+ longterm_token: str = None,
29
34
  env="prod",
30
35
  log_level=logging.WARNING,
36
+ proxies: dict = None,
37
+ verify: bool = True,
31
38
  ):
32
39
  configure_logging(log_level)
33
40
  endpoint_config = Config().load_config()
34
41
  try:
35
42
  token_manager = TokenManager(
36
- endpoint_config, environment=env, username=username, password=password
43
+ endpoint_config,
44
+ environment=env,
45
+ username=username,
46
+ password=password,
47
+ longterm_token=longterm_token,
48
+ proxies=proxies,
49
+ verify=verify,
37
50
  )
38
51
  except Exception as e:
39
52
  if "Failed to resolve" in str(e) or "Failed to establish" in str(e):
@@ -43,14 +56,24 @@ class Datalake:
43
56
  else:
44
57
  raise
45
58
 
46
- self.Threats = Threats(endpoint_config, env, token_manager)
47
- self.BulkSearch = BulkSearch(endpoint_config, env, token_manager)
48
- self.Tags = Tags(endpoint_config, env, token_manager)
59
+ self.Threats = Threats(
60
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
61
+ )
62
+ self.BulkSearch = BulkSearch(
63
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
64
+ )
65
+ self.Tags = Tags(
66
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
67
+ )
49
68
  self.FilteredTagSubcategory = FilteredTagSubcategory(
50
- endpoint_config, env, token_manager
69
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
70
+ )
71
+ self.AdvancedSearch = AdvancedSearch(
72
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
73
+ )
74
+ self.Sightings = Sightings(
75
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
51
76
  )
52
- self.AdvancedSearch = AdvancedSearch(endpoint_config, env, token_manager)
53
- self.Sightings = Sightings(endpoint_config, env, token_manager)
54
77
  self.SearchWatch = SearchWatch(self.BulkSearch)
55
78
 
56
79
  def sightings_filtered_from_atom_value(
@@ -28,13 +28,20 @@ class Endpoint:
28
28
  SET_MAX_RETRY = 3
29
29
 
30
30
  def __init__(
31
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
31
+ self,
32
+ endpoint_config: dict,
33
+ environment: str,
34
+ token_manager: TokenManager,
35
+ proxies: dict = None,
36
+ verify: bool = True,
32
37
  ):
33
38
  self.endpoint_config = endpoint_config
34
39
  self.environment = environment
35
40
  self.terminal_size = self._get_terminal_size()
36
41
  self.token_manager = token_manager
37
42
  self.SET_MAX_RETRY = 3
43
+ self.proxies = proxies
44
+ self.verify = verify
38
45
 
39
46
  @staticmethod
40
47
  def _get_terminal_size() -> int:
@@ -65,11 +72,19 @@ class Endpoint:
65
72
  tries_left = self.SET_MAX_RETRY
66
73
 
67
74
  while tries_left > 0:
68
- headers["Authorization"] = self.token_manager.access_token
75
+ headers["Authorization"] = (
76
+ self.token_manager.access_token or self.token_manager.longterm_token
77
+ )
69
78
  logger.debug(self._pretty_debug_request(url, method, post_body, headers))
70
79
 
71
80
  response = self._send_request(
72
- url, method, headers, post_body, stream=stream
81
+ url,
82
+ method,
83
+ headers,
84
+ post_body,
85
+ stream=stream,
86
+ proxies=self.proxies,
87
+ verify=self.verify,
73
88
  )
74
89
 
75
90
  if logger.isEnabledFor(logging.DEBUG):
@@ -77,7 +92,7 @@ class Endpoint:
77
92
  logger.debug("API response:\n%s", response.text)
78
93
  if response.status_code == 401:
79
94
  logger.warning(
80
- "Token expired or Missing authorization header. Updating token"
95
+ "Missing authorization header or Token Error. Updating token"
81
96
  )
82
97
  self.token_manager.process_auth_error(response.json())
83
98
  elif response.status_code == 422:
@@ -109,7 +124,13 @@ class Endpoint:
109
124
 
110
125
  @staticmethod
111
126
  def _send_request(
112
- url: str, method: str, headers: dict, data: dict, stream=False
127
+ url: str,
128
+ method: str,
129
+ headers: dict,
130
+ data: dict,
131
+ stream=False,
132
+ proxies: dict = None,
133
+ verify: bool = True,
113
134
  ) -> Response:
114
135
  """
115
136
  Send the correct http request to url from method [get, post, delete, patch, put].
@@ -119,7 +140,10 @@ class Endpoint:
119
140
  "url": url,
120
141
  "headers": headers,
121
142
  "stream": stream,
143
+ "verify": verify,
122
144
  }
145
+ if proxies:
146
+ common_kwargs["proxies"] = proxies
123
147
 
124
148
  if method == "get":
125
149
  api_response = requests.get(**common_kwargs)
@@ -148,6 +172,7 @@ class Endpoint:
148
172
  + f" - data: \n{data}\n"
149
173
  + f" - token: \n{self.token_manager.access_token}\n"
150
174
  + f" - refresh_token: \n{self.token_manager.refresh_token}\n"
175
+ + f" - longterm_token: \n{self.token_manager.longterm_token}\n"
151
176
  + "-" * self.terminal_size
152
177
  )
153
178
  return debug
@@ -73,7 +73,18 @@ class FilteredTagSubcategory(Endpoint):
73
73
  )
74
74
  if limit_match:
75
75
  print("The 'limit' parameter must be >= 0 and <= 5000.")
76
- if not tag_category_match and not ordering_match and not limit_match:
76
+ token_match = re.search(
77
+ r"token",
78
+ error_message,
79
+ )
80
+ if token_match:
81
+ raise ve
82
+ if (
83
+ not tag_category_match
84
+ and not ordering_match
85
+ and not limit_match
86
+ and not token_match
87
+ ):
77
88
  print(
78
89
  "An error occurred, but no specific pattern was matched in the 422 HTTP error message."
79
90
  )
@@ -20,7 +20,7 @@ from datalake_scripts.scripts import (
20
20
 
21
21
  class Cli:
22
22
  CLI_NAME = "ocd-dtl"
23
- VERSION = "2.7.2"
23
+ VERSION = "2.7.4"
24
24
 
25
25
  def __init__(self):
26
26
  parser = argparse.ArgumentParser(
@@ -33,14 +33,26 @@ class BaseEngine:
33
33
  Json = Union[dict, list] # json like object that can be a dict or root level array
34
34
 
35
35
  def __init__(
36
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
36
+ self,
37
+ endpoint_config: dict,
38
+ environment: str,
39
+ token_manager: TokenManager,
40
+ proxies: dict = None,
41
+ verify: bool = True,
37
42
  ):
38
43
  self.endpoint_config = endpoint_config
39
44
  self.environment = environment
40
- self.requests_ssl_verify = suppress_insecure_request_warns(environment)
45
+ self.requests_ssl_verify = (
46
+ verify
47
+ if suppress_insecure_request_warns(environment)
48
+ else suppress_insecure_request_warns(environment)
49
+ )
41
50
  self.url = self._build_url(endpoint_config, environment)
42
51
  self.token_manager = token_manager
43
- self.endpoint = Endpoint(endpoint_config, environment, token_manager)
52
+ self.proxies = proxies
53
+ self.endpoint = Endpoint(
54
+ endpoint_config, environment, token_manager, proxies, verify
55
+ )
44
56
 
45
57
  def datalake_requests(
46
58
  self, url: str, method: str, headers: dict, post_body: dict = None
@@ -50,7 +62,8 @@ class BaseEngine:
50
62
  """
51
63
  try:
52
64
  response = self.endpoint.datalake_requests(url, method, headers, post_body)
53
- except ValueError:
65
+ except ValueError as ve:
66
+ logger.error(f"Error while attempting to request datalake : {ve}")
54
67
  logger.error("Request failed: Will return nothing for this request")
55
68
  return {}
56
69
  if (
@@ -47,11 +47,15 @@ class HandleBulkTaskMixin(BaseEngine):
47
47
 
48
48
  json_response = None
49
49
  while not json_response:
50
- headers = {"Authorization": self.token_manager.access_token}
50
+ headers = {
51
+ "Authorization": self.token_manager.access_token
52
+ or self.token_manager.longterm_token
53
+ }
51
54
  response = requests.get(
52
55
  url=retrieve_bulk_result_url,
53
56
  headers=headers,
54
57
  verify=self.requests_ssl_verify,
58
+ proxies=self.proxies,
55
59
  )
56
60
  if response.status_code == 200:
57
61
  potential_json_response = response.json()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.7.2
3
+ Version: 2.7.4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
40
40
  $ pip3 install datalake-scripts
41
41
  ```
42
42
  ### Using as a library
43
- see [the following link](tutorial.md)
43
+ ```python
44
+ from datalake import Datalake, AtomType, Output
45
+
46
+ dtl = Datalake(username='username', password='password')
47
+ dtl.Threats.lookup(
48
+ atom_value='mayoclinic.org',
49
+ atom_type=AtomType.DOMAIN,
50
+ hashkey_only=False,
51
+ output=Output.JSON
52
+ )
53
+ ```
54
+
55
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
44
56
 
45
57
  ### Using as a CLI
46
58
 
@@ -57,10 +69,31 @@ You can also use a script directly by using the following command: `<script_name
57
69
  ## Environment variables
58
70
 
59
71
  #### Authentication
60
- In case you don't want to enter credential for each commands and you are on a secured terminal, set those variables:
72
+
73
+ There are two methods of authentication:
74
+ - The first one is the use of the username and password. Every request to the API, will then use fresh tokens periodically created with these credentials.
75
+ - The second one is the use of a long term token. You can create long term token through the GUI, it can have more restricted permissions than your account. You can create several long term tokens for one account.
76
+
77
+ In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
78
+ * `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
79
+ Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
80
+
81
+ or
82
+
61
83
  * `OCD_DTL_USERNAME` email address used to login on Datalake API/GUI.
62
84
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
63
- > They are independent and one can be used without the other if you wish.
85
+ > These last two are independent and one can be used without the other if you wish.
86
+
87
+ #### Using a Proxy
88
+
89
+ You can set up following environment variables :
90
+
91
+ * `HTTP_PROXY`
92
+ * `HTTPS_PROXY`
93
+
94
+ We use the format accepted by the requests python library.
95
+ See its documenation for other possible kinds of proxy to set up.
96
+
64
97
 
65
98
  #### Throttling
66
99
  For throttling the request, those two environment variable can be used:
@@ -49,3 +49,9 @@ def datalake():
49
49
 
50
50
  responses.add(responses.POST, url, json=auth_response, status=200)
51
51
  return Datalake(username="username", password="password")
52
+
53
+
54
+ @pytest.fixture
55
+ @responses.activate
56
+ def datalake_longterm_token():
57
+ return Datalake(longterm_token="longterm_token1234")