datalake-scripts 3.0.0rc4__tar.gz → 3.0.0rc7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/PKG-INFO +16 -6
  2. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/README.md +2 -1
  3. datalake_scripts-3.0.0rc7/datalake/config/endpoints.json +35 -0
  4. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/bulk_search.py +8 -0
  5. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/endpoint.py +47 -26
  6. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sightings.py +36 -3
  7. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/cli.py +1 -1
  8. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/PKG-INFO +17 -7
  9. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/SOURCES.txt +74 -1
  10. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/entry_points.txt +0 -1
  11. datalake_scripts-3.0.0rc7/pyproject.toml +3 -0
  12. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/setup.cfg +3 -5
  13. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/setup.py +17 -6
  14. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_sightings.py +241 -2
  15. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/__init__.py +0 -0
  16. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/__init__.py +0 -0
  17. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/atom.py +0 -0
  18. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/atom_type.py +0 -0
  19. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/bulk_search_task.py +0 -0
  20. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/config.py +0 -0
  21. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/logger.py +0 -0
  22. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/output.py +0 -0
  23. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/throttler.py +0 -0
  24. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/token_manager.py +0 -0
  25. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/utils.py +0 -0
  26. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/warn.py +0 -0
  27. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/datalake.py +0 -0
  28. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/__init__.py +0 -0
  29. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/advanced_search.py +0 -0
  30. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/comments.py +0 -0
  31. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/filtered_threat_entity.py +0 -0
  32. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/my_account.py +0 -0
  33. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sources.py +0 -0
  34. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/tags.py +0 -0
  35. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/threats.py +0 -0
  36. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/__init__.py +0 -0
  37. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/search_watch.py +0 -0
  38. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/__init__.py +0 -0
  39. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/__init__.py +0 -0
  40. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/base_script.py +0 -0
  41. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/__init__.py +0 -0
  42. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  43. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/utils.py +0 -0
  44. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/__init__.py +0 -0
  45. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_comments.py +0 -0
  46. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_tags.py +0 -0
  47. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_threats.py +0 -0
  48. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/advanced_search.py +0 -0
  49. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  50. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/edit_score.py +0 -0
  51. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_atom_values.py +0 -0
  52. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
  53. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_my_user_info.py +0 -0
  54. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_query_hash.py +0 -0
  55. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats.py +0 -0
  56. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  57. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/lookup_threats.py +0 -0
  58. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/search_watch.py +0 -0
  59. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  60. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/requires.txt +0 -0
  61. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/top_level.txt +0 -0
  62. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/__init__.py +0 -0
  63. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/__init__.py +0 -0
  64. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/fixture.py +0 -0
  65. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/test_output.py +0 -0
  66. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/test_utils.py +0 -0
  67. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/conftest.py +0 -0
  68. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/__init__.py +0 -0
  69. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_advanced_search.py +0 -0
  70. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_atom_type.py +0 -0
  71. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_auth_via_tokens.py +0 -0
  72. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_bulk_search.py +0 -0
  73. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_filtered_threat_entity.py +0 -0
  74. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_my_account.py +0 -0
  75. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_search_watch.py +0 -0
  76. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_tags.py +0 -0
  77. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_threats.py +0 -0
  78. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/__init__.py +0 -0
  79. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_add_new_threats.py +0 -0
  80. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_base_scripts.py +0 -0
  81. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_cli.py +0 -0
  82. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc4
3
+ Version: 3.0.0rc7
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -115,7 +125,7 @@ We use the format accepted by the requests python library.
115
125
  See its documenation for other possible kinds of proxy to set up.
116
126
 
117
127
 
118
- #### Throttling
128
+ #### Throttling and retries
119
129
  For throttling the requests, those two environment variables can be used:
120
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
121
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -123,8 +133,8 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
123
133
 
124
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
125
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
126
137
 
127
138
  ### Contributing
128
139
 
129
140
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -95,7 +95,7 @@ We use the format accepted by the requests python library.
95
95
  See its documenation for other possible kinds of proxy to set up.
96
96
 
97
97
 
98
- #### Throttling
98
+ #### Throttling and retries
99
99
  For throttling the requests, those two environment variables can be used:
100
100
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
101
101
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -103,6 +103,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
103
103
 
104
104
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
105
105
 
106
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
106
107
 
107
108
  ### Contributing
108
109
 
@@ -0,0 +1,35 @@
1
+ {
2
+ "main": {
3
+ "preprod": "https://ti2.extranet.mrti-center.com/api/",
4
+ "prod": "https://datalake.cert.orangecyberdefense.com/api/"
5
+ },
6
+ "endpoints": {
7
+ "token": "auth/token/",
8
+ "refresh-token": "auth/refresh-token/",
9
+ "advanced-queries-threats": "mrti/advanced-queries/threats/",
10
+ "advanced-queries-threats-query-hash": "mrti/advanced-queries/threats/{query_hash}/",
11
+ "atom-values": "mrti/atom-values/",
12
+ "bulk-search": "mrti/bulk-search/",
13
+ "bulk-search-tasks": "mrti/bulk-search/tasks/",
14
+ "bulk-search-task": "mrti/bulk-search/task/{task_uuid}/",
15
+ "bulk-manual-threats": "mrti/bulk-manual-threats/",
16
+ "bulk-manual-threats-task": "mrti/bulk-manual-threats/task/{task_uuid}/",
17
+ "sources": "mrti/sources/",
18
+ "threat-entity-filtered":"mrti/threat-entity/filtered/",
19
+ "threats-manual": "mrti/threats-manual/",
20
+ "threats-bulk-lookup": "mrti/threats/bulk-lookup/",
21
+ "threats": "mrti/threats/{hashkey}/",
22
+ "threats-lookup": "mrti/threats/lookup/",
23
+ "threats-comments": "mrti/threats/{hashkey}/comments/",
24
+ "threats-tags": "mrti/threats/{hashkey}/tags/",
25
+ "threats-atom-values-extract": "mrti/threats/atom-values-extract/",
26
+ "threats-bulk-scoring-edits": "mrti/threats/bulk-scoring-edits/",
27
+ "threats-sighting": "mrti/threats/sighting/",
28
+ "threats-bulk-sighting": "mrti/threats/bulk-sighting/",
29
+ "threats-sighting-file": "mrti/threats/sighting-file/{hashkey}/",
30
+ "threats-sighting-filtered": "mrti/threats/sighting/filtered/",
31
+ "users-me": "users/me/"
32
+ },
33
+ "threats-manual-bulk-size": 100,
34
+ "api_version": "v3/"
35
+ }
@@ -15,6 +15,8 @@ class BulkSearch(Endpoint):
15
15
  query_body: dict = None,
16
16
  query_hash: str = None,
17
17
  query_fields: list = None,
18
+ indicators_only: bool = None,
19
+ indicators_and_threat_entities_only: bool = None,
18
20
  ) -> BulkSearchTask:
19
21
  """Creates a bulk search task"""
20
22
  if not query_body and not query_hash:
@@ -27,6 +29,12 @@ class BulkSearch(Endpoint):
27
29
  body["query_hash"] = query_hash
28
30
  if for_stix_export:
29
31
  body["for_stix_export"] = for_stix_export
32
+ if indicators_only:
33
+ body["indicators_only"] = indicators_only
34
+ if indicators_and_threat_entities_only:
35
+ body["indicators_and_threat_entities_only"] = (
36
+ indicators_and_threat_entities_only
37
+ )
30
38
 
31
39
  url = self._build_url_for_endpoint("bulk-search")
32
40
  response = self.datalake_requests(
@@ -9,6 +9,7 @@ from urllib.parse import urljoin
9
9
 
10
10
  import requests
11
11
  from requests import Response
12
+ from requests.adapters import HTTPAdapter, Retry
12
13
 
13
14
  from datalake.common.output import Output
14
15
  from datalake.common.throttler import throttle
@@ -17,12 +18,11 @@ from datalake.common.utils import get_error_message
17
18
 
18
19
  OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
19
20
  OCD_DTL_REQUESTS_PER_QUOTA_TIME = int(os.getenv("OCD_DTL_REQUESTS_PER_QUOTA_TIME", 5))
21
+ OCD_DTL_MAX_RETRIES = int(os.getenv("OCD_DTL_MAX_RETRIES", 3))
20
22
 
21
23
 
22
24
  class Endpoint:
23
25
 
24
- SET_MAX_RETRY = 3
25
-
26
26
  def __init__(
27
27
  self,
28
28
  logger,
@@ -40,9 +40,25 @@ class Endpoint:
40
40
  self.environment = environment
41
41
  self.terminal_size = self._get_terminal_size()
42
42
  self.token_manager = token_manager
43
- self.SET_MAX_RETRY = 3
44
43
  self.proxies = proxies
45
44
  self.verify = verify
45
+ self.session = requests.Session()
46
+
47
+ # Configure HTTP retry policy
48
+ retry_policy = Retry(
49
+ total=OCD_DTL_MAX_RETRIES, # Number of retries
50
+ backoff_factor=1, # How much time between retries (exponential)
51
+ raise_on_status=False, # Raise an error when the number of retries is exhausted
52
+ status_forcelist=[
53
+ 429,
54
+ 500,
55
+ 502,
56
+ 503,
57
+ 504,
58
+ ], # Retry for those HTTP status codes
59
+ )
60
+ self.session.mount("http://", HTTPAdapter(max_retries=retry_policy))
61
+ self.session.mount("https://", HTTPAdapter(max_retries=retry_policy))
46
62
 
47
63
  def _get_terminal_size(self) -> int:
48
64
  """Return the terminal size for pretty print"""
@@ -71,15 +87,17 @@ class Endpoint:
71
87
  """
72
88
  Use it to request the API
73
89
  """
74
- tries_left = self.SET_MAX_RETRY
75
90
 
76
- while tries_left > 0:
91
+ self.logger.debug(self._pretty_debug_request(url, method, post_body, headers))
92
+
93
+ response = None
94
+ retry_auth_count = OCD_DTL_MAX_RETRIES
95
+ if OCD_DTL_MAX_RETRIES < 1:
96
+ retry_auth_count = 1
97
+ while retry_auth_count >= 0:
77
98
  headers["Authorization"] = (
78
99
  self.token_manager.access_token or self.token_manager.longterm_token
79
100
  )
80
- self.logger.debug(
81
- self._pretty_debug_request(url, method, post_body, headers)
82
- )
83
101
 
84
102
  response = self._send_request(
85
103
  self,
@@ -100,20 +118,23 @@ class Endpoint:
100
118
  "Missing authorization header or Token Error. Updating token"
101
119
  )
102
120
  self.token_manager.process_auth_error(response.json())
103
- elif response.status_code == 422:
104
- json_resp = response.json()
105
- try:
106
- error_msg = get_error_message(json_resp)
107
- except ValueError:
108
- error_msg = response.text
109
- raise ValueError(f"422 HTTP code: {error_msg}")
110
- elif response.status_code < 200 or response.status_code > 299:
111
- self.logger.error(
112
- f"API returned non 2xx response code : {response.status_code}\n{response.text}\n Retrying"
113
- )
114
121
  else:
115
- return response
116
- tries_left -= 1
122
+ break
123
+ retry_auth_count -= 1
124
+
125
+ if response.status_code == 422:
126
+ json_resp = response.json()
127
+ try:
128
+ error_msg = get_error_message(json_resp)
129
+ except ValueError:
130
+ error_msg = response.text
131
+ raise ValueError(f"422 HTTP code: {error_msg}")
132
+ elif response.status_code < 200 or response.status_code > 299:
133
+ self.logger.error(
134
+ f"API returned non 2xx response code : {response.status_code}\n{response.text}"
135
+ )
136
+ else:
137
+ return response
117
138
  self.logger.error("Request failed")
118
139
  raise ValueError(f"{response.status_code}: {response.text.strip()}")
119
140
 
@@ -152,15 +173,15 @@ class Endpoint:
152
173
  common_kwargs["proxies"] = proxies
153
174
 
154
175
  if method == "get":
155
- api_response = requests.get(**common_kwargs)
176
+ api_response = self.session.get(**common_kwargs)
156
177
  elif method == "post":
157
- api_response = requests.post(**common_kwargs, data=json.dumps(data))
178
+ api_response = self.session.post(**common_kwargs, data=json.dumps(data))
158
179
  elif method == "delete":
159
- api_response = requests.delete(**common_kwargs, data=json.dumps(data))
180
+ api_response = self.session.delete(**common_kwargs, data=json.dumps(data))
160
181
  elif method == "patch":
161
- api_response = requests.patch(**common_kwargs, data=json.dumps(data))
182
+ api_response = self.session.patch(**common_kwargs, data=json.dumps(data))
162
183
  elif method == "put":
163
- api_response = requests.put(**common_kwargs, data=json.dumps(data))
184
+ api_response = self.session.put(**common_kwargs, data=json.dumps(data))
164
185
  else:
165
186
  self.logger.debug(
166
187
  "ERROR : Wrong requests, please only do [get, post, put, patch, delete] method"
@@ -46,9 +46,9 @@ class Sightings(Endpoint):
46
46
  impersonate_id: int = None,
47
47
  ):
48
48
  """
49
- Submit a list of sightings.
49
+ Submit a list of atoms as 1 sighting.
50
50
  Either threat "hashkeys" or list of atom objects in "atoms" is required.
51
- Possible sightings type: POSITIVE, NEGATIVE, NEUTRAL.
51
+ Possible sighting's type: POSITIVE, NEGATIVE, NEUTRAL.
52
52
  For POSITIVE and NEGATIVE sightings, "threat_types"
53
53
  field is required.
54
54
  End date timestamp should always be in the past.
@@ -73,6 +73,39 @@ class Sightings(Endpoint):
73
73
  res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
74
74
  return res
75
75
 
76
+ def bulk_submit_sightings(
77
+ self,
78
+ sightings: List[dict] = [],
79
+ ):
80
+ """
81
+ Submit a list of Sightings.
82
+ Input must be a list of valid dict.
83
+ Each dict is a sighting.
84
+ """
85
+ payload = {}
86
+ list_sightings_for_url_call = []
87
+ for sighting in sightings:
88
+ payload_item = self._prepare_sightings_payload(
89
+ sighting.get("atoms"),
90
+ sighting.get("hashkeys"),
91
+ sighting.get("start_timestamp"),
92
+ sighting.get("end_timestamp"),
93
+ sighting.get("sighting_type"),
94
+ sighting.get("description_visibility"),
95
+ sighting.get("count"),
96
+ sighting.get("threat_types"),
97
+ sighting.get("tags"),
98
+ sighting.get("description"),
99
+ sighting.get("relation"),
100
+ sighting.get("editable"),
101
+ sighting.get("impersonate_id"),
102
+ )
103
+ list_sightings_for_url_call.append(payload_item)
104
+ payload["data"] = list_sightings_for_url_call
105
+ url = self._build_url_for_endpoint("threats-bulk-sighting")
106
+ res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
107
+ return res
108
+
76
109
  @staticmethod
77
110
  def _check_sightings_payload_parameters(
78
111
  atoms,
@@ -99,7 +132,7 @@ class Sightings(Endpoint):
99
132
  ):
100
133
  raise ValueError(
101
134
  'For POSITIVE and NEGATIVE sightings "threat_types" field is required and has to be '
102
- "an instance of the Visibility class"
135
+ "an instance of the ThreatType class."
103
136
  )
104
137
  elif threat_types:
105
138
  raise ValueError(
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
21
21
 
22
22
  class Cli:
23
23
  CLI_NAME = "ocd-dtl"
24
- VERSION = "3.0.0rc4"
24
+ VERSION = "3.0.0rc7"
25
25
 
26
26
  def __init__(self):
27
27
  parser = argparse.ArgumentParser(
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
2
- Name: datalake-scripts
3
- Version: 3.0.0rc4
1
+ Metadata-Version: 2.4
2
+ Name: datalake_scripts
3
+ Version: 3.0.0rc7
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -115,7 +125,7 @@ We use the format accepted by the requests python library.
115
125
  See its documenation for other possible kinds of proxy to set up.
116
126
 
117
127
 
118
- #### Throttling
128
+ #### Throttling and retries
119
129
  For throttling the requests, those two environment variables can be used:
120
130
  * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
121
131
  * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
@@ -123,8 +133,8 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
123
133
 
124
134
  > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
125
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
126
137
 
127
138
  ### Contributing
128
139
 
129
140
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -1,4 +1,5 @@
1
1
  README.md
2
+ pyproject.toml
2
3
  setup.cfg
3
4
  setup.py
4
5
  ./datalake/__init__.py
@@ -14,6 +15,7 @@ setup.py
14
15
  ./datalake/common/token_manager.py
15
16
  ./datalake/common/utils.py
16
17
  ./datalake/common/warn.py
18
+ ./datalake/config/endpoints.json
17
19
  ./datalake/endpoints/__init__.py
18
20
  ./datalake/endpoints/advanced_search.py
19
21
  ./datalake/endpoints/bulk_search.py
@@ -71,9 +73,80 @@ setup.py
71
73
  ./tests/scripts/test_base_scripts.py
72
74
  ./tests/scripts/test_cli.py
73
75
  ./tests/scripts/test_csv_builder.py
76
+ datalake/__init__.py
77
+ datalake/datalake.py
78
+ datalake/common/__init__.py
79
+ datalake/common/atom.py
80
+ datalake/common/atom_type.py
81
+ datalake/common/bulk_search_task.py
82
+ datalake/common/config.py
83
+ datalake/common/logger.py
84
+ datalake/common/output.py
85
+ datalake/common/throttler.py
86
+ datalake/common/token_manager.py
87
+ datalake/common/utils.py
88
+ datalake/common/warn.py
89
+ datalake/config/endpoints.json
90
+ datalake/endpoints/__init__.py
91
+ datalake/endpoints/advanced_search.py
92
+ datalake/endpoints/bulk_search.py
93
+ datalake/endpoints/comments.py
94
+ datalake/endpoints/endpoint.py
95
+ datalake/endpoints/filtered_threat_entity.py
96
+ datalake/endpoints/my_account.py
97
+ datalake/endpoints/sightings.py
98
+ datalake/endpoints/sources.py
99
+ datalake/endpoints/tags.py
100
+ datalake/endpoints/threats.py
101
+ datalake/miscellaneous/__init__.py
102
+ datalake/miscellaneous/search_watch.py
103
+ datalake_scripts/__init__.py
104
+ datalake_scripts/cli.py
74
105
  datalake_scripts.egg-info/PKG-INFO
75
106
  datalake_scripts.egg-info/SOURCES.txt
76
107
  datalake_scripts.egg-info/dependency_links.txt
77
108
  datalake_scripts.egg-info/entry_points.txt
78
109
  datalake_scripts.egg-info/requires.txt
79
- datalake_scripts.egg-info/top_level.txt
110
+ datalake_scripts.egg-info/top_level.txt
111
+ datalake_scripts/common/__init__.py
112
+ datalake_scripts/common/base_script.py
113
+ datalake_scripts/helper_scripts/__init__.py
114
+ datalake_scripts/helper_scripts/output_builder.py
115
+ datalake_scripts/helper_scripts/utils.py
116
+ datalake_scripts/scripts/__init__.py
117
+ datalake_scripts/scripts/add_comments.py
118
+ datalake_scripts/scripts/add_tags.py
119
+ datalake_scripts/scripts/add_threats.py
120
+ datalake_scripts/scripts/advanced_search.py
121
+ datalake_scripts/scripts/bulk_lookup_threats.py
122
+ datalake_scripts/scripts/edit_score.py
123
+ datalake_scripts/scripts/get_atom_values.py
124
+ datalake_scripts/scripts/get_filtered_threat_entity.py
125
+ datalake_scripts/scripts/get_my_user_info.py
126
+ datalake_scripts/scripts/get_query_hash.py
127
+ datalake_scripts/scripts/get_threats.py
128
+ datalake_scripts/scripts/get_threats_from_query_hash.py
129
+ datalake_scripts/scripts/lookup_threats.py
130
+ datalake_scripts/scripts/search_watch.py
131
+ tests/__init__.py
132
+ tests/conftest.py
133
+ tests/common/__init__.py
134
+ tests/common/fixture.py
135
+ tests/common/test_output.py
136
+ tests/common/test_utils.py
137
+ tests/lib/__init__.py
138
+ tests/lib/test_advanced_search.py
139
+ tests/lib/test_atom_type.py
140
+ tests/lib/test_auth_via_tokens.py
141
+ tests/lib/test_bulk_search.py
142
+ tests/lib/test_filtered_threat_entity.py
143
+ tests/lib/test_my_account.py
144
+ tests/lib/test_search_watch.py
145
+ tests/lib/test_sightings.py
146
+ tests/lib/test_tags.py
147
+ tests/lib/test_threats.py
148
+ tests/scripts/__init__.py
149
+ tests/scripts/test_add_new_threats.py
150
+ tests/scripts/test_base_scripts.py
151
+ tests/scripts/test_cli.py
152
+ tests/scripts/test_csv_builder.py
@@ -1,3 +1,2 @@
1
1
  [console_scripts]
2
2
  ocd-dtl = datalake_scripts.cli:main
3
-
@@ -0,0 +1,3 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
@@ -2,12 +2,10 @@
2
2
  upgrade = true
3
3
 
4
4
  [egg_info]
5
- tag-date = false
6
- tag-build =
7
- tag_svn_revision = false
8
- tag_build =
9
5
  tag_date = 0
6
+ tag_build =
7
+ tag_svn_revision = false
10
8
 
11
9
  [metadata]
12
- description-file = README.md
10
+ description_file = README.md
13
11
 
@@ -2,20 +2,31 @@ import os
2
2
 
3
3
  from setuptools import find_packages, setup
4
4
 
5
- from datalake_scripts.cli import Cli
6
-
7
5
  here = os.path.abspath(os.path.dirname(__file__))
8
6
 
9
7
  with open("README.md", "r") as fh:
10
- long_description = fh.read()
8
+ LONG_DESCRIPTION = fh.read()
9
+
10
+
11
+ def read_version_or_cli_name(option: str):
12
+ with open("datalake_scripts/cli.py") as f:
13
+ for line in f:
14
+ if option == "version" and line.strip().startswith("VERSION"):
15
+ return line.split("=")[1].strip().strip('"')
16
+ if option == "cli_name" and line.strip().startswith("CLI_NAME"):
17
+ return line.split("=")[1].strip().strip('"')
18
+
19
+
20
+ CLI_NAME = read_version_or_cli_name("cli_name")
21
+ VERSION = read_version_or_cli_name("version")
11
22
 
12
23
  setup(
13
24
  name="datalake_scripts",
14
- version=Cli.VERSION,
25
+ version=VERSION,
15
26
  author="OCD",
16
27
  author_email="cert-contact.ocd@orange.com",
17
28
  description="A collection of scripts to easily use the API of OCD Datalake",
18
- long_description=long_description,
29
+ long_description=LONG_DESCRIPTION,
19
30
  long_description_content_type="text/markdown",
20
31
  url="https://github.com/cert-orangecyberdefense/datalake/",
21
32
  package_dir={
@@ -25,7 +36,7 @@ setup(
25
36
  install_requires=["requests", "halo", "prettytable"],
26
37
  test_suite="nose.collector",
27
38
  entry_points={
28
- "console_scripts": (f"{Cli.CLI_NAME} = datalake_scripts.cli:main",),
39
+ "console_scripts": (f"{CLI_NAME} = datalake_scripts.cli:main",),
29
40
  },
30
41
  classifiers=[
31
42
  "Programming Language :: Python :: 3.7",
@@ -1,16 +1,19 @@
1
1
  from unittest.mock import patch
2
2
  import pytest
3
3
  import responses
4
- from datetime import datetime
4
+ from datetime import datetime, timedelta, timezone
5
5
 
6
6
  from responses import matchers
7
7
  import os
8
+ import ast
8
9
 
9
10
 
10
11
  from tests.common.fixture import TestData, datalake # noqa needed fixture import
11
12
  from datalake import (
12
13
  IpAtom,
13
14
  FileAtom,
15
+ EmailAtom,
16
+ UrlAtom,
14
17
  Hashes,
15
18
  Jarm,
16
19
  IpService,
@@ -390,7 +393,7 @@ def test_submit_sightings_no_threat_types(datalake):
390
393
  assert (
391
394
  str(err.value)
392
395
  == 'For POSITIVE and NEGATIVE sightings "threat_types" field is required and'
393
- " has to be an instance of the Visibility class"
396
+ " has to be an instance of the ThreatType class."
394
397
  )
395
398
 
396
399
 
@@ -448,6 +451,242 @@ def test_submit_sightings_bad_atom(datalake):
448
451
  assert str(err.value) == '"atoms" needs to be a list of Atom subclasses.'
449
452
 
450
453
 
454
+ @responses.activate
455
+ def test_bulk_submit_sightings_success(datalake):
456
+ url = (
457
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
458
+ + TestData.TEST_CONFIG["api_version"]
459
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
460
+ )
461
+
462
+ f1 = FileAtom(
463
+ hashes=Hashes(
464
+ md5="d41d8cd98f00b204e9800998ecf8427e",
465
+ sha1="da39a3ee5e6b4b0d3255bfef95601890afd80709",
466
+ sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
467
+ )
468
+ )
469
+ ip1 = IpAtom("52.48.79.33")
470
+ em1 = EmailAtom("hacker@hacker.fr")
471
+ url1 = UrlAtom("http://notfishing.com")
472
+
473
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
474
+
475
+ ## Prepare threat types for this sighting
476
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
477
+
478
+ ## Prepare start and end timestamps for this sighting
479
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
480
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
481
+ input_sighting = {
482
+ "atoms": [ip1, f1, em1, url1],
483
+ "start_timestamp": start,
484
+ "end_timestamp": end,
485
+ "sighting_type": SightingType.POSITIVE,
486
+ "description_visibility": Visibility.PUBLIC,
487
+ "count": 1,
488
+ "threat_types": threat_types,
489
+ "tags": ["some_tag"],
490
+ "description": "some_description",
491
+ "editable": True,
492
+ }
493
+ expected_request = {
494
+ "data": [
495
+ {
496
+ "count": 1,
497
+ "description": "some_description",
498
+ "description_visibility": "PUBLIC",
499
+ "editable": True,
500
+ "email_list": [{"email": "hacker@hacker.fr"}],
501
+ "end_timestamp": "2025-12-16T10:00:00Z",
502
+ "file_list": [
503
+ {
504
+ "hashes": {
505
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
506
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
507
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
508
+ }
509
+ }
510
+ ],
511
+ "ip_list": [{"ip_address": "52.48.79.33"}],
512
+ "start_timestamp": "2025-12-16T09:00:00Z",
513
+ "tags": ["some_tag"],
514
+ "threat_types": ["phishing", "scam"],
515
+ "type": "positive",
516
+ "url_list": [{"url": "http://notfishing.com"}],
517
+ }
518
+ ]
519
+ }
520
+ expected_res = {
521
+ "count": 1,
522
+ "results": [
523
+ {
524
+ "count": 1,
525
+ "description": "some_description",
526
+ "description_visibility": "PUBLIC",
527
+ "editable": True,
528
+ "end_timestamp": "2025-12-16T10:00:00Z",
529
+ "relation_type": "sighting",
530
+ "reliability": 100,
531
+ "sighting_version": "3.0.0",
532
+ "sightings": {
533
+ "email_list": [{"email": "hacker@hacker.fr"}],
534
+ "file_list": [
535
+ {
536
+ "hashes": {
537
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
538
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
539
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
540
+ "sha512": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
541
+ }
542
+ }
543
+ ],
544
+ "ip_list": [{"ip_address": "52.48.79.33"}],
545
+ "url_list": [{"url": "http://notfishing.com"}],
546
+ },
547
+ "source_context": {
548
+ "source_id": "org: Orange Cyberdefense",
549
+ "source_uses": ["sightings"],
550
+ },
551
+ "start_timestamp": "2025-12-16T09:00:00Z",
552
+ "tags": [{"categories": [], "name": "some_tag"}],
553
+ "threat_types": ["phishing", "scam"],
554
+ "timestamp_created": "2025-12-16T10:00:00Z",
555
+ "type": "positive",
556
+ "uid": "e14dd426-ed41-436c-9e0e-0d785e750428",
557
+ }
558
+ ],
559
+ }
560
+
561
+ responses.post(
562
+ url=url,
563
+ json=expected_res,
564
+ status=200,
565
+ match=[matchers.json_params_matcher(expected_request)],
566
+ )
567
+
568
+ with patch.dict(
569
+ os.environ,
570
+ {
571
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
572
+ },
573
+ clear=True,
574
+ ):
575
+ res = datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
576
+
577
+ assert res == expected_res
578
+
579
+
580
+ def test_bulk_submit_sightings_payload_error(datalake):
581
+ with pytest.raises(ValueError) as err:
582
+ datalake.Sightings.bulk_submit_sightings(
583
+ sightings=[
584
+ {
585
+ "atoms": [FileAtom(Hashes(md5="d41d8cd98f00b204e9800998ecf8427e"))],
586
+ "start_timestamp": start,
587
+ "end_timestamp": end,
588
+ "sighting_type": SightingType.NEUTRAL,
589
+ "description_visibility": Visibility.PUBLIC,
590
+ "count": 3,
591
+ "threat_types": threat_types,
592
+ "tags": ["some_tag_bis"],
593
+ "description": "some_description_bis",
594
+ "editable": False,
595
+ }
596
+ ]
597
+ )
598
+ assert (
599
+ str(err.value) == """For NEUTRAL sightings, "threat_types" can't be passed."""
600
+ )
601
+
602
+
603
+ @responses.activate
604
+ def test_bulk_submit_sightings_failure(datalake):
605
+ url = (
606
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
607
+ + TestData.TEST_CONFIG["api_version"]
608
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
609
+ )
610
+
611
+ hashkeys = [
612
+ "d41d8cd98f00b204e9800998ecf8427e",
613
+ "a7b25b324871a7695aa2cc5d09681dda",
614
+ "1ed07771327e850255b09b042ad00e3d",
615
+ "bf1f33c3a56e1dfda6a2f4f3d3e4361a",
616
+ ]
617
+
618
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
619
+
620
+ ## Prepare threat types for this sighting
621
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
622
+
623
+ ## Prepare start and end timestamps for this sighting
624
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
625
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
626
+ input_sighting = {
627
+ "hashkeys": hashkeys,
628
+ "start_timestamp": start,
629
+ "end_timestamp": end,
630
+ "sighting_type": SightingType.POSITIVE,
631
+ "description_visibility": Visibility.PUBLIC,
632
+ "count": 1,
633
+ "threat_types": threat_types,
634
+ "tags": ["some_tag"],
635
+ "description": "some_description",
636
+ "editable": True,
637
+ }
638
+ expected_request = {
639
+ "data": [
640
+ {
641
+ "count": 1,
642
+ "description": "some_description",
643
+ "description_visibility": "PUBLIC",
644
+ "editable": True,
645
+ "end_timestamp": "2025-12-16T10:00:00Z",
646
+ "hashkeys": hashkeys,
647
+ "start_timestamp": "2025-12-16T09:00:00Z",
648
+ "tags": ["some_tag"],
649
+ "threat_types": ["phishing", "scam"],
650
+ "type": "positive",
651
+ }
652
+ ]
653
+ }
654
+ expected_res = {
655
+ "data": {
656
+ "0": {
657
+ "hashkeys": [
658
+ "Some of the provided hashkeys has not been found: "
659
+ "['d41d8cd98f00b204e9800998ecf8427e']"
660
+ ]
661
+ }
662
+ }
663
+ }
664
+
665
+ responses.post(
666
+ url=url,
667
+ json=expected_res,
668
+ status=422,
669
+ match=[matchers.json_params_matcher(expected_request)],
670
+ )
671
+
672
+ with patch.dict(
673
+ os.environ,
674
+ {
675
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
676
+ },
677
+ clear=True,
678
+ ):
679
+ with pytest.raises(ValueError) as exc:
680
+ datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
681
+
682
+ prefix = "422 HTTP code: "
683
+ msg = str(exc.value)
684
+ assert msg.startswith(prefix)
685
+
686
+ payload = ast.literal_eval(msg[len(prefix) :])
687
+ assert payload == expected_res
688
+
689
+
451
690
  def test_sightings_filtered_bad_ordering(datalake):
452
691
  with pytest.raises(ValueError) as err:
453
692
  datalake.Sightings.sightings_filtered(ordering="bad_ordering")