datalake-scripts 3.0.0rc6__tar.gz → 3.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/PKG-INFO +3 -2
  2. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/README.md +2 -1
  3. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/bulk_search_task.py +8 -1
  4. datalake_scripts-3.0.1/datalake/common/user_agent.py +26 -0
  5. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/bulk_search.py +23 -0
  6. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/endpoint.py +49 -26
  7. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/sightings.py +1 -1
  8. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/cli.py +4 -2
  9. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/PKG-INFO +3 -2
  10. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/SOURCES.txt +4 -0
  11. datalake_scripts-3.0.1/tests/common/test_user_agent.py +36 -0
  12. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_bulk_search.py +137 -0
  13. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_sightings.py +1 -1
  14. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/__init__.py +0 -0
  15. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/__init__.py +0 -0
  16. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/atom.py +0 -0
  17. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/atom_type.py +0 -0
  18. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/config.py +0 -0
  19. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/logger.py +0 -0
  20. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/output.py +0 -0
  21. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/throttler.py +0 -0
  22. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/token_manager.py +0 -0
  23. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/utils.py +0 -0
  24. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/warn.py +0 -0
  25. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/config/endpoints.json +0 -0
  26. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/datalake.py +0 -0
  27. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/__init__.py +0 -0
  28. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/advanced_search.py +0 -0
  29. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/comments.py +0 -0
  30. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/filtered_threat_entity.py +0 -0
  31. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/my_account.py +0 -0
  32. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/sources.py +0 -0
  33. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/tags.py +0 -0
  34. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/threats.py +0 -0
  35. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/miscellaneous/__init__.py +0 -0
  36. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/miscellaneous/search_watch.py +0 -0
  37. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/__init__.py +0 -0
  38. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/common/__init__.py +0 -0
  39. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/common/base_script.py +0 -0
  40. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/__init__.py +0 -0
  41. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  42. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/utils.py +0 -0
  43. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/__init__.py +0 -0
  44. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_comments.py +0 -0
  45. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_tags.py +0 -0
  46. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_threats.py +0 -0
  47. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/advanced_search.py +0 -0
  48. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  49. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/edit_score.py +0 -0
  50. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_atom_values.py +0 -0
  51. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
  52. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_my_user_info.py +0 -0
  53. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_query_hash.py +0 -0
  54. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats.py +0 -0
  55. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  56. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/lookup_threats.py +0 -0
  57. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/search_watch.py +0 -0
  58. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  59. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/entry_points.txt +0 -0
  60. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/requires.txt +0 -0
  61. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/top_level.txt +0 -0
  62. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/pyproject.toml +0 -0
  63. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/setup.cfg +0 -0
  64. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/setup.py +0 -0
  65. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/__init__.py +0 -0
  66. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/__init__.py +0 -0
  67. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/fixture.py +0 -0
  68. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/test_output.py +0 -0
  69. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/test_utils.py +0 -0
  70. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/conftest.py +0 -0
  71. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/__init__.py +0 -0
  72. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_advanced_search.py +0 -0
  73. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_atom_type.py +0 -0
  74. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_auth_via_tokens.py +0 -0
  75. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_filtered_threat_entity.py +0 -0
  76. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_my_account.py +0 -0
  77. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_search_watch.py +0 -0
  78. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_tags.py +0 -0
  79. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_threats.py +0 -0
  80. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/__init__.py +0 -0
  81. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_add_new_threats.py +0 -0
  82. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_base_scripts.py +0 -0
  83. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_cli.py +0 -0
  84. {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc6
3
+ Version: 3.0.1
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
125
125
  See its documenation for other possible kinds of proxy to set up.
126
126
 
127
127
 
128
- #### Throttling
128
+ #### Throttling and retries
129
129
  For throttling the requests, those two environment variables can be used:
130
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
131
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
133
133
 
134
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
135
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
136
137
 
137
138
  ### Contributing
138
139
 
@@ -95,7 +95,7 @@ We use the format accepted by the requests python library.
95
95
  See its documenation for other possible kinds of proxy to set up.
96
96
 
97
97
 
98
- #### Throttling
98
+ #### Throttling and retries
99
99
  For throttling the requests, those two environment variables can be used:
100
100
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
101
101
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -103,6 +103,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
103
103
 
104
104
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
105
105
 
106
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
106
107
 
107
108
  ### Contributing
108
109
 
@@ -108,8 +108,11 @@ class BulkSearchTask:
108
108
  raise BulkSearchFailedError(self.state)
109
109
  time_passed = datetime.datetime.now(datetime.timezone.utc) - start
110
110
  if time_passed.total_seconds() > timeout:
111
+ try:
112
+ self.cancel()
113
+ except Exception:
114
+ pass
111
115
  raise TimeoutError()
112
-
113
116
  await asyncio.sleep(self.REQUEST_INTERVAL)
114
117
  self.update()
115
118
  return self.download(output=output, stream=stream)
@@ -134,3 +137,7 @@ class BulkSearchTask:
134
137
  """Query the API to refresh the tasks attributes"""
135
138
  updated_bs = self._endpoint.get_task(self.uuid)
136
139
  self.__dict__.update(updated_bs.__dict__) # Avoid to return a new object
140
+
141
+ def cancel(self):
142
+ """Cancel this bulk search task server-side."""
143
+ self._endpoint.cancel(self.uuid)
@@ -0,0 +1,26 @@
1
+ """User-Agent string builder for outbound Endpoint HTTP calls."""
2
+
3
+ import os
4
+
5
+ # this weird code block is to maintain python 3.7 compatibility
6
+ try:
7
+ from importlib.metadata import PackageNotFoundError, version
8
+ except ImportError: # Python < 3.8
9
+ from importlib_metadata import PackageNotFoundError, version
10
+
11
+ USER_AGENT_INTEGRATION_ENV = "OCD_DTL_USER_AGENT_INTEGRATION"
12
+
13
+
14
+ def _get_version() -> str:
15
+ try:
16
+ return version("datalake_scripts")
17
+ except PackageNotFoundError:
18
+ return "unknown"
19
+
20
+
21
+ def build_user_agent() -> str:
22
+ base = f"datalake-python-sdk/{_get_version()}"
23
+ integration = (os.getenv(USER_AGENT_INTEGRATION_ENV) or "").strip()
24
+ if integration:
25
+ return f"{integration} {base}"
26
+ return base
@@ -15,6 +15,8 @@ class BulkSearch(Endpoint):
15
15
  query_body: dict = None,
16
16
  query_hash: str = None,
17
17
  query_fields: list = None,
18
+ indicators_only: bool = None,
19
+ indicators_and_threat_entities_only: bool = None,
18
20
  ) -> BulkSearchTask:
19
21
  """Creates a bulk search task"""
20
22
  if not query_body and not query_hash:
@@ -27,6 +29,12 @@ class BulkSearch(Endpoint):
27
29
  body["query_hash"] = query_hash
28
30
  if for_stix_export:
29
31
  body["for_stix_export"] = for_stix_export
32
+ if indicators_only:
33
+ body["indicators_only"] = indicators_only
34
+ if indicators_and_threat_entities_only:
35
+ body["indicators_and_threat_entities_only"] = (
36
+ indicators_and_threat_entities_only
37
+ )
30
38
 
31
39
  url = self._build_url_for_endpoint("bulk-search")
32
40
  response = self.datalake_requests(
@@ -46,6 +54,21 @@ class BulkSearch(Endpoint):
46
54
  bs_as_json = results[0]
47
55
  return BulkSearchTask(endpoint=self, **bs_as_json)
48
56
 
57
+ def cancel(self, task_uuid):
58
+ """
59
+ Cancel the bulk search task with the given uuid.
60
+
61
+ Raises BulkSearchNotFound if the task does not exist.
62
+ """
63
+ url = self._build_url_for_endpoint("bulk-search-task")
64
+ url = url.format(task_uuid=task_uuid)
65
+ try:
66
+ self.datalake_requests(url, "delete", headers=self._get_headers())
67
+ except ValueError as e:
68
+ if str(e).startswith("404"):
69
+ raise BulkSearchNotFound from e
70
+ raise
71
+
49
72
  @output_supported(
50
73
  {
51
74
  Output.JSON,
@@ -9,20 +9,21 @@ from urllib.parse import urljoin
9
9
 
10
10
  import requests
11
11
  from requests import Response
12
+ from requests.adapters import HTTPAdapter, Retry
12
13
 
13
14
  from datalake.common.output import Output
14
15
  from datalake.common.throttler import throttle
15
16
  from datalake.common.token_manager import TokenManager
17
+ from datalake.common.user_agent import build_user_agent
16
18
  from datalake.common.utils import get_error_message
17
19
 
18
20
  OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
19
21
  OCD_DTL_REQUESTS_PER_QUOTA_TIME = int(os.getenv("OCD_DTL_REQUESTS_PER_QUOTA_TIME", 5))
22
+ OCD_DTL_MAX_RETRIES = int(os.getenv("OCD_DTL_MAX_RETRIES", 3))
20
23
 
21
24
 
22
25
  class Endpoint:
23
26
 
24
- SET_MAX_RETRY = 3
25
-
26
27
  def __init__(
27
28
  self,
28
29
  logger,
@@ -40,9 +41,26 @@ class Endpoint:
40
41
  self.environment = environment
41
42
  self.terminal_size = self._get_terminal_size()
42
43
  self.token_manager = token_manager
43
- self.SET_MAX_RETRY = 3
44
44
  self.proxies = proxies
45
45
  self.verify = verify
46
+ self.session = requests.Session()
47
+ self.session.headers.update({"User-Agent": build_user_agent()})
48
+
49
+ # Configure HTTP retry policy
50
+ retry_policy = Retry(
51
+ total=OCD_DTL_MAX_RETRIES, # Number of retries
52
+ backoff_factor=1, # How much time between retries (exponential)
53
+ raise_on_status=False, # Raise an error when the number of retries is exhausted
54
+ status_forcelist=[
55
+ 429,
56
+ 500,
57
+ 502,
58
+ 503,
59
+ 504,
60
+ ], # Retry for those HTTP status codes
61
+ )
62
+ self.session.mount("http://", HTTPAdapter(max_retries=retry_policy))
63
+ self.session.mount("https://", HTTPAdapter(max_retries=retry_policy))
46
64
 
47
65
  def _get_terminal_size(self) -> int:
48
66
  """Return the terminal size for pretty print"""
@@ -71,15 +89,17 @@ class Endpoint:
71
89
  """
72
90
  Use it to request the API
73
91
  """
74
- tries_left = self.SET_MAX_RETRY
75
92
 
76
- while tries_left > 0:
93
+ self.logger.debug(self._pretty_debug_request(url, method, post_body, headers))
94
+
95
+ response = None
96
+ retry_auth_count = OCD_DTL_MAX_RETRIES
97
+ if OCD_DTL_MAX_RETRIES < 1:
98
+ retry_auth_count = 1
99
+ while retry_auth_count >= 0:
77
100
  headers["Authorization"] = (
78
101
  self.token_manager.access_token or self.token_manager.longterm_token
79
102
  )
80
- self.logger.debug(
81
- self._pretty_debug_request(url, method, post_body, headers)
82
- )
83
103
 
84
104
  response = self._send_request(
85
105
  self,
@@ -100,20 +120,23 @@ class Endpoint:
100
120
  "Missing authorization header or Token Error. Updating token"
101
121
  )
102
122
  self.token_manager.process_auth_error(response.json())
103
- elif response.status_code == 422:
104
- json_resp = response.json()
105
- try:
106
- error_msg = get_error_message(json_resp)
107
- except ValueError:
108
- error_msg = response.text
109
- raise ValueError(f"422 HTTP code: {error_msg}")
110
- elif response.status_code < 200 or response.status_code > 299:
111
- self.logger.error(
112
- f"API returned non 2xx response code : {response.status_code}\n{response.text}\n Retrying"
113
- )
114
123
  else:
115
- return response
116
- tries_left -= 1
124
+ break
125
+ retry_auth_count -= 1
126
+
127
+ if response.status_code == 422:
128
+ json_resp = response.json()
129
+ try:
130
+ error_msg = get_error_message(json_resp)
131
+ except ValueError:
132
+ error_msg = response.text
133
+ raise ValueError(f"422 HTTP code: {error_msg}")
134
+ elif response.status_code < 200 or response.status_code > 299:
135
+ self.logger.error(
136
+ f"API returned non 2xx response code : {response.status_code}\n{response.text}"
137
+ )
138
+ else:
139
+ return response
117
140
  self.logger.error("Request failed")
118
141
  raise ValueError(f"{response.status_code}: {response.text.strip()}")
119
142
 
@@ -152,15 +175,15 @@ class Endpoint:
152
175
  common_kwargs["proxies"] = proxies
153
176
 
154
177
  if method == "get":
155
- api_response = requests.get(**common_kwargs)
178
+ api_response = self.session.get(**common_kwargs)
156
179
  elif method == "post":
157
- api_response = requests.post(**common_kwargs, data=json.dumps(data))
180
+ api_response = self.session.post(**common_kwargs, data=json.dumps(data))
158
181
  elif method == "delete":
159
- api_response = requests.delete(**common_kwargs, data=json.dumps(data))
182
+ api_response = self.session.delete(**common_kwargs, data=json.dumps(data))
160
183
  elif method == "patch":
161
- api_response = requests.patch(**common_kwargs, data=json.dumps(data))
184
+ api_response = self.session.patch(**common_kwargs, data=json.dumps(data))
162
185
  elif method == "put":
163
- api_response = requests.put(**common_kwargs, data=json.dumps(data))
186
+ api_response = self.session.put(**common_kwargs, data=json.dumps(data))
164
187
  else:
165
188
  self.logger.debug(
166
189
  "ERROR : Wrong requests, please only do [get, post, put, patch, delete] method"
@@ -132,7 +132,7 @@ class Sightings(Endpoint):
132
132
  ):
133
133
  raise ValueError(
134
134
  'For POSITIVE and NEGATIVE sightings "threat_types" field is required and has to be '
135
- "an instance of the Visibility class"
135
+ "an instance of the ThreatType class."
136
136
  )
137
137
  elif threat_types:
138
138
  raise ValueError(
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env python3
2
2
  import argparse
3
- import sys
3
+ import sys, os
4
4
  from datalake_scripts.scripts import (
5
5
  add_threats,
6
6
  get_threats,
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
21
21
 
22
22
  class Cli:
23
23
  CLI_NAME = "ocd-dtl"
24
- VERSION = "3.0.0rc6"
24
+ VERSION = "3.0.1"
25
25
 
26
26
  def __init__(self):
27
27
  parser = argparse.ArgumentParser(
@@ -74,6 +74,8 @@ class Cli:
74
74
  parser.print_help()
75
75
  exit(1)
76
76
 
77
+ os.environ["OCD_DTL_USER_AGENT_INTEGRATION"] = f"{self.CLI_NAME}/{self.VERSION}"
78
+
77
79
  # Call the subcommand method
78
80
  args.func(command_arguments)
79
81
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc6
3
+ Version: 3.0.1
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
125
125
  See its documenation for other possible kinds of proxy to set up.
126
126
 
127
127
 
128
- #### Throttling
128
+ #### Throttling and retries
129
129
  For throttling the requests, those two environment variables can be used:
130
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
131
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
133
133
 
134
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
135
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
136
137
 
137
138
  ### Contributing
138
139
 
@@ -13,6 +13,7 @@ setup.py
13
13
  ./datalake/common/output.py
14
14
  ./datalake/common/throttler.py
15
15
  ./datalake/common/token_manager.py
16
+ ./datalake/common/user_agent.py
16
17
  ./datalake/common/utils.py
17
18
  ./datalake/common/warn.py
18
19
  ./datalake/config/endpoints.json
@@ -56,6 +57,7 @@ setup.py
56
57
  ./tests/common/__init__.py
57
58
  ./tests/common/fixture.py
58
59
  ./tests/common/test_output.py
60
+ ./tests/common/test_user_agent.py
59
61
  ./tests/common/test_utils.py
60
62
  ./tests/lib/__init__.py
61
63
  ./tests/lib/test_advanced_search.py
@@ -84,6 +86,7 @@ datalake/common/logger.py
84
86
  datalake/common/output.py
85
87
  datalake/common/throttler.py
86
88
  datalake/common/token_manager.py
89
+ datalake/common/user_agent.py
87
90
  datalake/common/utils.py
88
91
  datalake/common/warn.py
89
92
  datalake/config/endpoints.json
@@ -133,6 +136,7 @@ tests/conftest.py
133
136
  tests/common/__init__.py
134
137
  tests/common/fixture.py
135
138
  tests/common/test_output.py
139
+ tests/common/test_user_agent.py
136
140
  tests/common/test_utils.py
137
141
  tests/lib/__init__.py
138
142
  tests/lib/test_advanced_search.py
@@ -0,0 +1,36 @@
1
+ import pytest
2
+
3
+ from datalake.common import user_agent as user_agent_mod
4
+ from datalake.common.user_agent import (
5
+ USER_AGENT_INTEGRATION_ENV,
6
+ build_user_agent,
7
+ )
8
+
9
+
10
+ @pytest.fixture
11
+ def fixed_version(monkeypatch):
12
+ monkeypatch.setattr(user_agent_mod, "_get_version", lambda: "9.9.9")
13
+ return "9.9.9"
14
+
15
+
16
+ def test_build_user_agent_default(monkeypatch, fixed_version):
17
+ monkeypatch.delenv(USER_AGENT_INTEGRATION_ENV, raising=False)
18
+ assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
19
+
20
+
21
+ def test_build_user_agent_with_integration(monkeypatch, fixed_version):
22
+ monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, "my-soar/2.1")
23
+ assert build_user_agent() == f"my-soar/2.1 datalake-python-sdk/{fixed_version}"
24
+
25
+
26
+ @pytest.mark.parametrize("value", ["", " "])
27
+ def test_build_user_agent_ignores_empty_or_whitespace(
28
+ monkeypatch, fixed_version, value
29
+ ):
30
+ monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, value)
31
+ assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
32
+
33
+
34
+ def test_build_user_agent_strips_surrounding_whitespace(monkeypatch, fixed_version):
35
+ monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, " foo/1.0 ")
36
+ assert build_user_agent() == f"foo/1.0 datalake-python-sdk/{fixed_version}"
@@ -418,11 +418,26 @@ async def test_bulk_search_task_download_async_timeout(
418
418
  response_context.add(
419
419
  responses.POST, bs_status_url, json=bs_update_json, status=200
420
420
  )
421
+ bs_cancel_url = (
422
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
423
+ + TestData.TEST_CONFIG["api_version"]
424
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
425
+ "{task_uuid}", bulk_search_task.uuid
426
+ )
427
+ )
428
+ response_context.add(responses.DELETE, bs_cancel_url, status=204)
421
429
 
422
430
  task = bulk_search_task.download_async(timeout=0.2)
423
431
  with pytest.raises(TimeoutError):
424
432
  await task
425
433
 
434
+ cancel_calls = [
435
+ c
436
+ for c in response_context.calls
437
+ if c.request.method == "DELETE" and c.request.url == bs_cancel_url
438
+ ]
439
+ assert len(cancel_calls) == 1, "cancel() must be called on timeout"
440
+
426
441
 
427
442
  @responses.activate
428
443
  def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask):
@@ -437,10 +452,25 @@ def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask
437
452
  + TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
438
453
  )
439
454
  responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
455
+ bs_cancel_url = (
456
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
457
+ + TestData.TEST_CONFIG["api_version"]
458
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
459
+ "{task_uuid}", bulk_search_task.uuid
460
+ )
461
+ )
462
+ responses.add(responses.DELETE, bs_cancel_url, status=204)
440
463
 
441
464
  with pytest.raises(TimeoutError):
442
465
  bulk_search_task.download_sync(timeout=0.2)
443
466
 
467
+ cancel_calls = [
468
+ c
469
+ for c in responses.calls
470
+ if c.request.method == "DELETE" and c.request.url == bs_cancel_url
471
+ ]
472
+ assert len(cancel_calls) == 1, "cancel() must be called on timeout"
473
+
444
474
 
445
475
  @responses.activate
446
476
  def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask):
@@ -453,7 +483,114 @@ def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask)
453
483
  + TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
454
484
  )
455
485
  responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
486
+ bs_cancel_url = (
487
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
488
+ + TestData.TEST_CONFIG["api_version"]
489
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
490
+ "{task_uuid}", bulk_search_task.uuid
491
+ )
492
+ )
493
+ responses.add(responses.DELETE, bs_cancel_url, status=204)
456
494
 
457
495
  with pytest.raises(BulkSearchFailedError) as err:
458
496
  bulk_search_task.download_sync()
459
497
  assert err.value.failed_state == BulkSearchTaskState.CANCELLED
498
+
499
+ cancel_calls = [
500
+ c
501
+ for c in responses.calls
502
+ if c.request.method == "DELETE" and c.request.url == bs_cancel_url
503
+ ]
504
+ assert len(cancel_calls) == 0, "cancel() must not be called on failed state"
505
+
506
+
507
+ @pytest.mark.parametrize(
508
+ "failed_state",
509
+ [
510
+ "CANCELLED",
511
+ "FAILED_TIMEOUT",
512
+ "FAILED_ERROR",
513
+ "FAILED_QUOTA_EXCEEDED",
514
+ "FAILED_TOO_MANY_RESULTS",
515
+ ],
516
+ )
517
+ @responses.activate
518
+ def test_bulk_search_task_download_sync_cancel_on_any_failed_state(
519
+ bulk_search_task: BulkSearchTask, failed_state
520
+ ):
521
+ bulk_search_task.state = BulkSearchTaskState.IN_PROGRESS
522
+ bs_update_json = copy.deepcopy(bs_status_json)
523
+ bs_update_json["results"][0]["state"] = failed_state
524
+ bs_status_url = (
525
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
526
+ + TestData.TEST_CONFIG["api_version"]
527
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
528
+ )
529
+ responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
530
+ bs_cancel_url = (
531
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
532
+ + TestData.TEST_CONFIG["api_version"]
533
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
534
+ "{task_uuid}", bulk_search_task.uuid
535
+ )
536
+ )
537
+ responses.add(responses.DELETE, bs_cancel_url, status=204)
538
+
539
+ with pytest.raises(BulkSearchFailedError) as err:
540
+ bulk_search_task.download_sync()
541
+ assert err.value.failed_state == BulkSearchTaskState[failed_state]
542
+
543
+ cancel_calls = [
544
+ c
545
+ for c in responses.calls
546
+ if c.request.method == "DELETE" and c.request.url == bs_cancel_url
547
+ ]
548
+ assert (
549
+ len(cancel_calls) == 0
550
+ ), f"cancel() must not be called for state {failed_state}"
551
+
552
+
553
+ @responses.activate
554
+ def test_bulk_search_task_cancel(bulk_search_task: BulkSearchTask):
555
+ task_uuid = bulk_search_task.uuid
556
+ bs_cancel_url = (
557
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
558
+ + TestData.TEST_CONFIG["api_version"]
559
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
560
+ "{task_uuid}", task_uuid
561
+ )
562
+ )
563
+ responses.add(responses.DELETE, bs_cancel_url, status=204)
564
+
565
+ assert bulk_search_task.cancel() is None
566
+
567
+
568
+ @responses.activate
569
+ def test_bulk_search_task_cancel_not_found(bulk_search_task: BulkSearchTask):
570
+ task_uuid = bulk_search_task.uuid
571
+ bs_cancel_url = (
572
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
573
+ + TestData.TEST_CONFIG["api_version"]
574
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
575
+ "{task_uuid}", task_uuid
576
+ )
577
+ )
578
+ responses.add(responses.DELETE, bs_cancel_url, json={}, status=404)
579
+
580
+ with pytest.raises(BulkSearchNotFound):
581
+ bulk_search_task.cancel()
582
+
583
+
584
+ @responses.activate
585
+ def test_bulk_search_endpoint_cancel(datalake: Datalake):
586
+ task_uuid = "d9c00380-2784-4386-9bc3-aff35cfeeb41"
587
+ bs_cancel_url = (
588
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
589
+ + TestData.TEST_CONFIG["api_version"]
590
+ + TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
591
+ "{task_uuid}", task_uuid
592
+ )
593
+ )
594
+ responses.add(responses.DELETE, bs_cancel_url, status=204)
595
+
596
+ assert datalake.BulkSearch.cancel(task_uuid) is None
@@ -393,7 +393,7 @@ def test_submit_sightings_no_threat_types(datalake):
393
393
  assert (
394
394
  str(err.value)
395
395
  == 'For POSITIVE and NEGATIVE sightings "threat_types" field is required and'
396
- " has to be an instance of the Visibility class"
396
+ " has to be an instance of the ThreatType class."
397
397
  )
398
398
 
399
399