datalake-scripts 3.0.0rc6__tar.gz → 3.0.0rc7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/PKG-INFO +3 -2
  2. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/README.md +2 -1
  3. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/bulk_search.py +8 -0
  4. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/endpoint.py +47 -26
  5. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sightings.py +1 -1
  6. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/cli.py +1 -1
  7. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/PKG-INFO +3 -2
  8. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_sightings.py +1 -1
  9. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/__init__.py +0 -0
  10. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/__init__.py +0 -0
  11. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/atom.py +0 -0
  12. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/atom_type.py +0 -0
  13. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/bulk_search_task.py +0 -0
  14. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/config.py +0 -0
  15. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/logger.py +0 -0
  16. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/output.py +0 -0
  17. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/throttler.py +0 -0
  18. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/token_manager.py +0 -0
  19. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/utils.py +0 -0
  20. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/common/warn.py +0 -0
  21. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/config/endpoints.json +0 -0
  22. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/datalake.py +0 -0
  23. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/__init__.py +0 -0
  24. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/advanced_search.py +0 -0
  25. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/comments.py +0 -0
  26. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/filtered_threat_entity.py +0 -0
  27. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/my_account.py +0 -0
  28. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sources.py +0 -0
  29. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/tags.py +0 -0
  30. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/endpoints/threats.py +0 -0
  31. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/__init__.py +0 -0
  32. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/search_watch.py +0 -0
  33. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/__init__.py +0 -0
  34. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/__init__.py +0 -0
  35. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/base_script.py +0 -0
  36. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/__init__.py +0 -0
  37. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  38. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/utils.py +0 -0
  39. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/__init__.py +0 -0
  40. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_comments.py +0 -0
  41. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_tags.py +0 -0
  42. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_threats.py +0 -0
  43. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/advanced_search.py +0 -0
  44. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  45. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/edit_score.py +0 -0
  46. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_atom_values.py +0 -0
  47. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
  48. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_my_user_info.py +0 -0
  49. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_query_hash.py +0 -0
  50. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats.py +0 -0
  51. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  52. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/lookup_threats.py +0 -0
  53. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/search_watch.py +0 -0
  54. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/SOURCES.txt +0 -0
  55. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  56. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/entry_points.txt +0 -0
  57. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/requires.txt +0 -0
  58. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/top_level.txt +0 -0
  59. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/pyproject.toml +0 -0
  60. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/setup.cfg +0 -0
  61. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/setup.py +0 -0
  62. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/__init__.py +0 -0
  63. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/common/__init__.py +0 -0
  64. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/common/fixture.py +0 -0
  65. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/common/test_output.py +0 -0
  66. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/common/test_utils.py +0 -0
  67. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/conftest.py +0 -0
  68. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/__init__.py +0 -0
  69. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_advanced_search.py +0 -0
  70. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_atom_type.py +0 -0
  71. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_auth_via_tokens.py +0 -0
  72. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_bulk_search.py +0 -0
  73. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_filtered_threat_entity.py +0 -0
  74. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_my_account.py +0 -0
  75. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_search_watch.py +0 -0
  76. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_tags.py +0 -0
  77. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/lib/test_threats.py +0 -0
  78. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/scripts/__init__.py +0 -0
  79. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/scripts/test_add_new_threats.py +0 -0
  80. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/scripts/test_base_scripts.py +0 -0
  81. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/scripts/test_cli.py +0 -0
  82. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.0rc7}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc6
3
+ Version: 3.0.0rc7
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
125
125
  See its documenation for other possible kinds of proxy to set up.
126
126
 
127
127
 
128
- #### Throttling
128
+ #### Throttling and retries
129
129
  For throttling the requests, those two environment variables can be used:
130
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
131
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
133
133
 
134
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
135
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
136
137
 
137
138
  ### Contributing
138
139
 
@@ -95,7 +95,7 @@ We use the format accepted by the requests python library.
95
95
  See its documenation for other possible kinds of proxy to set up.
96
96
 
97
97
 
98
- #### Throttling
98
+ #### Throttling and retries
99
99
  For throttling the requests, those two environment variables can be used:
100
100
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
101
101
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -103,6 +103,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
103
103
 
104
104
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
105
105
 
106
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
106
107
 
107
108
  ### Contributing
108
109
 
@@ -15,6 +15,8 @@ class BulkSearch(Endpoint):
15
15
  query_body: dict = None,
16
16
  query_hash: str = None,
17
17
  query_fields: list = None,
18
+ indicators_only: bool = None,
19
+ indicators_and_threat_entities_only: bool = None,
18
20
  ) -> BulkSearchTask:
19
21
  """Creates a bulk search task"""
20
22
  if not query_body and not query_hash:
@@ -27,6 +29,12 @@ class BulkSearch(Endpoint):
27
29
  body["query_hash"] = query_hash
28
30
  if for_stix_export:
29
31
  body["for_stix_export"] = for_stix_export
32
+ if indicators_only:
33
+ body["indicators_only"] = indicators_only
34
+ if indicators_and_threat_entities_only:
35
+ body["indicators_and_threat_entities_only"] = (
36
+ indicators_and_threat_entities_only
37
+ )
30
38
 
31
39
  url = self._build_url_for_endpoint("bulk-search")
32
40
  response = self.datalake_requests(
@@ -9,6 +9,7 @@ from urllib.parse import urljoin
9
9
 
10
10
  import requests
11
11
  from requests import Response
12
+ from requests.adapters import HTTPAdapter, Retry
12
13
 
13
14
  from datalake.common.output import Output
14
15
  from datalake.common.throttler import throttle
@@ -17,12 +18,11 @@ from datalake.common.utils import get_error_message
17
18
 
18
19
  OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
19
20
  OCD_DTL_REQUESTS_PER_QUOTA_TIME = int(os.getenv("OCD_DTL_REQUESTS_PER_QUOTA_TIME", 5))
21
+ OCD_DTL_MAX_RETRIES = int(os.getenv("OCD_DTL_MAX_RETRIES", 3))
20
22
 
21
23
 
22
24
  class Endpoint:
23
25
 
24
- SET_MAX_RETRY = 3
25
-
26
26
  def __init__(
27
27
  self,
28
28
  logger,
@@ -40,9 +40,25 @@ class Endpoint:
40
40
  self.environment = environment
41
41
  self.terminal_size = self._get_terminal_size()
42
42
  self.token_manager = token_manager
43
- self.SET_MAX_RETRY = 3
44
43
  self.proxies = proxies
45
44
  self.verify = verify
45
+ self.session = requests.Session()
46
+
47
+ # Configure HTTP retry policy
48
+ retry_policy = Retry(
49
+ total=OCD_DTL_MAX_RETRIES, # Number of retries
50
+ backoff_factor=1, # How much time between retries (exponential)
51
+ raise_on_status=False, # Raise an error when the number of retries is exhausted
52
+ status_forcelist=[
53
+ 429,
54
+ 500,
55
+ 502,
56
+ 503,
57
+ 504,
58
+ ], # Retry for those HTTP status codes
59
+ )
60
+ self.session.mount("http://", HTTPAdapter(max_retries=retry_policy))
61
+ self.session.mount("https://", HTTPAdapter(max_retries=retry_policy))
46
62
 
47
63
  def _get_terminal_size(self) -> int:
48
64
  """Return the terminal size for pretty print"""
@@ -71,15 +87,17 @@ class Endpoint:
71
87
  """
72
88
  Use it to request the API
73
89
  """
74
- tries_left = self.SET_MAX_RETRY
75
90
 
76
- while tries_left > 0:
91
+ self.logger.debug(self._pretty_debug_request(url, method, post_body, headers))
92
+
93
+ response = None
94
+ retry_auth_count = OCD_DTL_MAX_RETRIES
95
+ if OCD_DTL_MAX_RETRIES < 1:
96
+ retry_auth_count = 1
97
+ while retry_auth_count >= 0:
77
98
  headers["Authorization"] = (
78
99
  self.token_manager.access_token or self.token_manager.longterm_token
79
100
  )
80
- self.logger.debug(
81
- self._pretty_debug_request(url, method, post_body, headers)
82
- )
83
101
 
84
102
  response = self._send_request(
85
103
  self,
@@ -100,20 +118,23 @@ class Endpoint:
100
118
  "Missing authorization header or Token Error. Updating token"
101
119
  )
102
120
  self.token_manager.process_auth_error(response.json())
103
- elif response.status_code == 422:
104
- json_resp = response.json()
105
- try:
106
- error_msg = get_error_message(json_resp)
107
- except ValueError:
108
- error_msg = response.text
109
- raise ValueError(f"422 HTTP code: {error_msg}")
110
- elif response.status_code < 200 or response.status_code > 299:
111
- self.logger.error(
112
- f"API returned non 2xx response code : {response.status_code}\n{response.text}\n Retrying"
113
- )
114
121
  else:
115
- return response
116
- tries_left -= 1
122
+ break
123
+ retry_auth_count -= 1
124
+
125
+ if response.status_code == 422:
126
+ json_resp = response.json()
127
+ try:
128
+ error_msg = get_error_message(json_resp)
129
+ except ValueError:
130
+ error_msg = response.text
131
+ raise ValueError(f"422 HTTP code: {error_msg}")
132
+ elif response.status_code < 200 or response.status_code > 299:
133
+ self.logger.error(
134
+ f"API returned non 2xx response code : {response.status_code}\n{response.text}"
135
+ )
136
+ else:
137
+ return response
117
138
  self.logger.error("Request failed")
118
139
  raise ValueError(f"{response.status_code}: {response.text.strip()}")
119
140
 
@@ -152,15 +173,15 @@ class Endpoint:
152
173
  common_kwargs["proxies"] = proxies
153
174
 
154
175
  if method == "get":
155
- api_response = requests.get(**common_kwargs)
176
+ api_response = self.session.get(**common_kwargs)
156
177
  elif method == "post":
157
- api_response = requests.post(**common_kwargs, data=json.dumps(data))
178
+ api_response = self.session.post(**common_kwargs, data=json.dumps(data))
158
179
  elif method == "delete":
159
- api_response = requests.delete(**common_kwargs, data=json.dumps(data))
180
+ api_response = self.session.delete(**common_kwargs, data=json.dumps(data))
160
181
  elif method == "patch":
161
- api_response = requests.patch(**common_kwargs, data=json.dumps(data))
182
+ api_response = self.session.patch(**common_kwargs, data=json.dumps(data))
162
183
  elif method == "put":
163
- api_response = requests.put(**common_kwargs, data=json.dumps(data))
184
+ api_response = self.session.put(**common_kwargs, data=json.dumps(data))
164
185
  else:
165
186
  self.logger.debug(
166
187
  "ERROR : Wrong requests, please only do [get, post, put, patch, delete] method"
@@ -132,7 +132,7 @@ class Sightings(Endpoint):
132
132
  ):
133
133
  raise ValueError(
134
134
  'For POSITIVE and NEGATIVE sightings "threat_types" field is required and has to be '
135
- "an instance of the Visibility class"
135
+ "an instance of the ThreatType class."
136
136
  )
137
137
  elif threat_types:
138
138
  raise ValueError(
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
21
21
 
22
22
  class Cli:
23
23
  CLI_NAME = "ocd-dtl"
24
- VERSION = "3.0.0rc6"
24
+ VERSION = "3.0.0rc7"
25
25
 
26
26
  def __init__(self):
27
27
  parser = argparse.ArgumentParser(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc6
3
+ Version: 3.0.0rc7
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
125
125
  See its documenation for other possible kinds of proxy to set up.
126
126
 
127
127
 
128
- #### Throttling
128
+ #### Throttling and retries
129
129
  For throttling the requests, those two environment variables can be used:
130
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
131
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
133
133
 
134
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
135
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
136
137
 
137
138
  ### Contributing
138
139
 
@@ -393,7 +393,7 @@ def test_submit_sightings_no_threat_types(datalake):
393
393
  assert (
394
394
  str(err.value)
395
395
  == 'For POSITIVE and NEGATIVE sightings "threat_types" field is required and'
396
- " has to be an instance of the Visibility class"
396
+ " has to be an instance of the ThreatType class."
397
397
  )
398
398
 
399
399