datalake-scripts 3.0.0rc6__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/PKG-INFO +3 -2
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/README.md +2 -1
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/bulk_search_task.py +8 -1
- datalake_scripts-3.0.1/datalake/common/user_agent.py +26 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/bulk_search.py +23 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/endpoint.py +49 -26
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/sightings.py +1 -1
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/cli.py +4 -2
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/PKG-INFO +3 -2
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/SOURCES.txt +4 -0
- datalake_scripts-3.0.1/tests/common/test_user_agent.py +36 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_bulk_search.py +137 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_sightings.py +1 -1
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/atom.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/atom_type.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/config.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/logger.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/output.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/throttler.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/token_manager.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/utils.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/common/warn.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/config/endpoints.json +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/datalake.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/comments.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/my_account.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/sources.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/tags.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/miscellaneous/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/common/base_script.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_my_user_info.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/entry_points.txt +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/pyproject.toml +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/setup.cfg +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/setup.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/fixture.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/test_output.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/common/test_utils.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/conftest.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_atom_type.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_my_account.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_search_watch.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_tags.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_base_scripts.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_cli.py +0 -0
- {datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datalake_scripts
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
|
|
|
125
125
|
See its documenation for other possible kinds of proxy to set up.
|
|
126
126
|
|
|
127
127
|
|
|
128
|
-
#### Throttling
|
|
128
|
+
#### Throttling and retries
|
|
129
129
|
For throttling the requests, those two environment variables can be used:
|
|
130
130
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
131
131
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
133
133
|
|
|
134
134
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
135
135
|
|
|
136
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
136
137
|
|
|
137
138
|
### Contributing
|
|
138
139
|
|
|
@@ -95,7 +95,7 @@ We use the format accepted by the requests python library.
|
|
|
95
95
|
See its documenation for other possible kinds of proxy to set up.
|
|
96
96
|
|
|
97
97
|
|
|
98
|
-
#### Throttling
|
|
98
|
+
#### Throttling and retries
|
|
99
99
|
For throttling the requests, those two environment variables can be used:
|
|
100
100
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
101
101
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -103,6 +103,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
103
103
|
|
|
104
104
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
105
105
|
|
|
106
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
106
107
|
|
|
107
108
|
### Contributing
|
|
108
109
|
|
|
@@ -108,8 +108,11 @@ class BulkSearchTask:
|
|
|
108
108
|
raise BulkSearchFailedError(self.state)
|
|
109
109
|
time_passed = datetime.datetime.now(datetime.timezone.utc) - start
|
|
110
110
|
if time_passed.total_seconds() > timeout:
|
|
111
|
+
try:
|
|
112
|
+
self.cancel()
|
|
113
|
+
except Exception:
|
|
114
|
+
pass
|
|
111
115
|
raise TimeoutError()
|
|
112
|
-
|
|
113
116
|
await asyncio.sleep(self.REQUEST_INTERVAL)
|
|
114
117
|
self.update()
|
|
115
118
|
return self.download(output=output, stream=stream)
|
|
@@ -134,3 +137,7 @@ class BulkSearchTask:
|
|
|
134
137
|
"""Query the API to refresh the tasks attributes"""
|
|
135
138
|
updated_bs = self._endpoint.get_task(self.uuid)
|
|
136
139
|
self.__dict__.update(updated_bs.__dict__) # Avoid to return a new object
|
|
140
|
+
|
|
141
|
+
def cancel(self):
|
|
142
|
+
"""Cancel this bulk search task server-side."""
|
|
143
|
+
self._endpoint.cancel(self.uuid)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""User-Agent string builder for outbound Endpoint HTTP calls."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
# this weird code block is to maintain python 3.7 compatibility
|
|
6
|
+
try:
|
|
7
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
8
|
+
except ImportError: # Python < 3.8
|
|
9
|
+
from importlib_metadata import PackageNotFoundError, version
|
|
10
|
+
|
|
11
|
+
USER_AGENT_INTEGRATION_ENV = "OCD_DTL_USER_AGENT_INTEGRATION"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_version() -> str:
|
|
15
|
+
try:
|
|
16
|
+
return version("datalake_scripts")
|
|
17
|
+
except PackageNotFoundError:
|
|
18
|
+
return "unknown"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_user_agent() -> str:
|
|
22
|
+
base = f"datalake-python-sdk/{_get_version()}"
|
|
23
|
+
integration = (os.getenv(USER_AGENT_INTEGRATION_ENV) or "").strip()
|
|
24
|
+
if integration:
|
|
25
|
+
return f"{integration} {base}"
|
|
26
|
+
return base
|
|
@@ -15,6 +15,8 @@ class BulkSearch(Endpoint):
|
|
|
15
15
|
query_body: dict = None,
|
|
16
16
|
query_hash: str = None,
|
|
17
17
|
query_fields: list = None,
|
|
18
|
+
indicators_only: bool = None,
|
|
19
|
+
indicators_and_threat_entities_only: bool = None,
|
|
18
20
|
) -> BulkSearchTask:
|
|
19
21
|
"""Creates a bulk search task"""
|
|
20
22
|
if not query_body and not query_hash:
|
|
@@ -27,6 +29,12 @@ class BulkSearch(Endpoint):
|
|
|
27
29
|
body["query_hash"] = query_hash
|
|
28
30
|
if for_stix_export:
|
|
29
31
|
body["for_stix_export"] = for_stix_export
|
|
32
|
+
if indicators_only:
|
|
33
|
+
body["indicators_only"] = indicators_only
|
|
34
|
+
if indicators_and_threat_entities_only:
|
|
35
|
+
body["indicators_and_threat_entities_only"] = (
|
|
36
|
+
indicators_and_threat_entities_only
|
|
37
|
+
)
|
|
30
38
|
|
|
31
39
|
url = self._build_url_for_endpoint("bulk-search")
|
|
32
40
|
response = self.datalake_requests(
|
|
@@ -46,6 +54,21 @@ class BulkSearch(Endpoint):
|
|
|
46
54
|
bs_as_json = results[0]
|
|
47
55
|
return BulkSearchTask(endpoint=self, **bs_as_json)
|
|
48
56
|
|
|
57
|
+
def cancel(self, task_uuid):
|
|
58
|
+
"""
|
|
59
|
+
Cancel the bulk search task with the given uuid.
|
|
60
|
+
|
|
61
|
+
Raises BulkSearchNotFound if the task does not exist.
|
|
62
|
+
"""
|
|
63
|
+
url = self._build_url_for_endpoint("bulk-search-task")
|
|
64
|
+
url = url.format(task_uuid=task_uuid)
|
|
65
|
+
try:
|
|
66
|
+
self.datalake_requests(url, "delete", headers=self._get_headers())
|
|
67
|
+
except ValueError as e:
|
|
68
|
+
if str(e).startswith("404"):
|
|
69
|
+
raise BulkSearchNotFound from e
|
|
70
|
+
raise
|
|
71
|
+
|
|
49
72
|
@output_supported(
|
|
50
73
|
{
|
|
51
74
|
Output.JSON,
|
|
@@ -9,20 +9,21 @@ from urllib.parse import urljoin
|
|
|
9
9
|
|
|
10
10
|
import requests
|
|
11
11
|
from requests import Response
|
|
12
|
+
from requests.adapters import HTTPAdapter, Retry
|
|
12
13
|
|
|
13
14
|
from datalake.common.output import Output
|
|
14
15
|
from datalake.common.throttler import throttle
|
|
15
16
|
from datalake.common.token_manager import TokenManager
|
|
17
|
+
from datalake.common.user_agent import build_user_agent
|
|
16
18
|
from datalake.common.utils import get_error_message
|
|
17
19
|
|
|
18
20
|
OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
|
|
19
21
|
OCD_DTL_REQUESTS_PER_QUOTA_TIME = int(os.getenv("OCD_DTL_REQUESTS_PER_QUOTA_TIME", 5))
|
|
22
|
+
OCD_DTL_MAX_RETRIES = int(os.getenv("OCD_DTL_MAX_RETRIES", 3))
|
|
20
23
|
|
|
21
24
|
|
|
22
25
|
class Endpoint:
|
|
23
26
|
|
|
24
|
-
SET_MAX_RETRY = 3
|
|
25
|
-
|
|
26
27
|
def __init__(
|
|
27
28
|
self,
|
|
28
29
|
logger,
|
|
@@ -40,9 +41,26 @@ class Endpoint:
|
|
|
40
41
|
self.environment = environment
|
|
41
42
|
self.terminal_size = self._get_terminal_size()
|
|
42
43
|
self.token_manager = token_manager
|
|
43
|
-
self.SET_MAX_RETRY = 3
|
|
44
44
|
self.proxies = proxies
|
|
45
45
|
self.verify = verify
|
|
46
|
+
self.session = requests.Session()
|
|
47
|
+
self.session.headers.update({"User-Agent": build_user_agent()})
|
|
48
|
+
|
|
49
|
+
# Configure HTTP retry policy
|
|
50
|
+
retry_policy = Retry(
|
|
51
|
+
total=OCD_DTL_MAX_RETRIES, # Number of retries
|
|
52
|
+
backoff_factor=1, # How much time between retries (exponential)
|
|
53
|
+
raise_on_status=False, # Raise an error when the number of retries is exhausted
|
|
54
|
+
status_forcelist=[
|
|
55
|
+
429,
|
|
56
|
+
500,
|
|
57
|
+
502,
|
|
58
|
+
503,
|
|
59
|
+
504,
|
|
60
|
+
], # Retry for those HTTP status codes
|
|
61
|
+
)
|
|
62
|
+
self.session.mount("http://", HTTPAdapter(max_retries=retry_policy))
|
|
63
|
+
self.session.mount("https://", HTTPAdapter(max_retries=retry_policy))
|
|
46
64
|
|
|
47
65
|
def _get_terminal_size(self) -> int:
|
|
48
66
|
"""Return the terminal size for pretty print"""
|
|
@@ -71,15 +89,17 @@ class Endpoint:
|
|
|
71
89
|
"""
|
|
72
90
|
Use it to request the API
|
|
73
91
|
"""
|
|
74
|
-
tries_left = self.SET_MAX_RETRY
|
|
75
92
|
|
|
76
|
-
|
|
93
|
+
self.logger.debug(self._pretty_debug_request(url, method, post_body, headers))
|
|
94
|
+
|
|
95
|
+
response = None
|
|
96
|
+
retry_auth_count = OCD_DTL_MAX_RETRIES
|
|
97
|
+
if OCD_DTL_MAX_RETRIES < 1:
|
|
98
|
+
retry_auth_count = 1
|
|
99
|
+
while retry_auth_count >= 0:
|
|
77
100
|
headers["Authorization"] = (
|
|
78
101
|
self.token_manager.access_token or self.token_manager.longterm_token
|
|
79
102
|
)
|
|
80
|
-
self.logger.debug(
|
|
81
|
-
self._pretty_debug_request(url, method, post_body, headers)
|
|
82
|
-
)
|
|
83
103
|
|
|
84
104
|
response = self._send_request(
|
|
85
105
|
self,
|
|
@@ -100,20 +120,23 @@ class Endpoint:
|
|
|
100
120
|
"Missing authorization header or Token Error. Updating token"
|
|
101
121
|
)
|
|
102
122
|
self.token_manager.process_auth_error(response.json())
|
|
103
|
-
elif response.status_code == 422:
|
|
104
|
-
json_resp = response.json()
|
|
105
|
-
try:
|
|
106
|
-
error_msg = get_error_message(json_resp)
|
|
107
|
-
except ValueError:
|
|
108
|
-
error_msg = response.text
|
|
109
|
-
raise ValueError(f"422 HTTP code: {error_msg}")
|
|
110
|
-
elif response.status_code < 200 or response.status_code > 299:
|
|
111
|
-
self.logger.error(
|
|
112
|
-
f"API returned non 2xx response code : {response.status_code}\n{response.text}\n Retrying"
|
|
113
|
-
)
|
|
114
123
|
else:
|
|
115
|
-
|
|
116
|
-
|
|
124
|
+
break
|
|
125
|
+
retry_auth_count -= 1
|
|
126
|
+
|
|
127
|
+
if response.status_code == 422:
|
|
128
|
+
json_resp = response.json()
|
|
129
|
+
try:
|
|
130
|
+
error_msg = get_error_message(json_resp)
|
|
131
|
+
except ValueError:
|
|
132
|
+
error_msg = response.text
|
|
133
|
+
raise ValueError(f"422 HTTP code: {error_msg}")
|
|
134
|
+
elif response.status_code < 200 or response.status_code > 299:
|
|
135
|
+
self.logger.error(
|
|
136
|
+
f"API returned non 2xx response code : {response.status_code}\n{response.text}"
|
|
137
|
+
)
|
|
138
|
+
else:
|
|
139
|
+
return response
|
|
117
140
|
self.logger.error("Request failed")
|
|
118
141
|
raise ValueError(f"{response.status_code}: {response.text.strip()}")
|
|
119
142
|
|
|
@@ -152,15 +175,15 @@ class Endpoint:
|
|
|
152
175
|
common_kwargs["proxies"] = proxies
|
|
153
176
|
|
|
154
177
|
if method == "get":
|
|
155
|
-
api_response =
|
|
178
|
+
api_response = self.session.get(**common_kwargs)
|
|
156
179
|
elif method == "post":
|
|
157
|
-
api_response =
|
|
180
|
+
api_response = self.session.post(**common_kwargs, data=json.dumps(data))
|
|
158
181
|
elif method == "delete":
|
|
159
|
-
api_response =
|
|
182
|
+
api_response = self.session.delete(**common_kwargs, data=json.dumps(data))
|
|
160
183
|
elif method == "patch":
|
|
161
|
-
api_response =
|
|
184
|
+
api_response = self.session.patch(**common_kwargs, data=json.dumps(data))
|
|
162
185
|
elif method == "put":
|
|
163
|
-
api_response =
|
|
186
|
+
api_response = self.session.put(**common_kwargs, data=json.dumps(data))
|
|
164
187
|
else:
|
|
165
188
|
self.logger.debug(
|
|
166
189
|
"ERROR : Wrong requests, please only do [get, post, put, patch, delete] method"
|
|
@@ -132,7 +132,7 @@ class Sightings(Endpoint):
|
|
|
132
132
|
):
|
|
133
133
|
raise ValueError(
|
|
134
134
|
'For POSITIVE and NEGATIVE sightings "threat_types" field is required and has to be '
|
|
135
|
-
"an instance of the
|
|
135
|
+
"an instance of the ThreatType class."
|
|
136
136
|
)
|
|
137
137
|
elif threat_types:
|
|
138
138
|
raise ValueError(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
|
-
import sys
|
|
3
|
+
import sys, os
|
|
4
4
|
from datalake_scripts.scripts import (
|
|
5
5
|
add_threats,
|
|
6
6
|
get_threats,
|
|
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
|
|
|
21
21
|
|
|
22
22
|
class Cli:
|
|
23
23
|
CLI_NAME = "ocd-dtl"
|
|
24
|
-
VERSION = "3.0.
|
|
24
|
+
VERSION = "3.0.1"
|
|
25
25
|
|
|
26
26
|
def __init__(self):
|
|
27
27
|
parser = argparse.ArgumentParser(
|
|
@@ -74,6 +74,8 @@ class Cli:
|
|
|
74
74
|
parser.print_help()
|
|
75
75
|
exit(1)
|
|
76
76
|
|
|
77
|
+
os.environ["OCD_DTL_USER_AGENT_INTEGRATION"] = f"{self.CLI_NAME}/{self.VERSION}"
|
|
78
|
+
|
|
77
79
|
# Call the subcommand method
|
|
78
80
|
args.func(command_arguments)
|
|
79
81
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datalake_scripts
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -125,7 +125,7 @@ We use the format accepted by the requests python library.
|
|
|
125
125
|
See its documenation for other possible kinds of proxy to set up.
|
|
126
126
|
|
|
127
127
|
|
|
128
|
-
#### Throttling
|
|
128
|
+
#### Throttling and retries
|
|
129
129
|
For throttling the requests, those two environment variables can be used:
|
|
130
130
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
131
131
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -133,6 +133,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
133
133
|
|
|
134
134
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
135
135
|
|
|
136
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
136
137
|
|
|
137
138
|
### Contributing
|
|
138
139
|
|
|
@@ -13,6 +13,7 @@ setup.py
|
|
|
13
13
|
./datalake/common/output.py
|
|
14
14
|
./datalake/common/throttler.py
|
|
15
15
|
./datalake/common/token_manager.py
|
|
16
|
+
./datalake/common/user_agent.py
|
|
16
17
|
./datalake/common/utils.py
|
|
17
18
|
./datalake/common/warn.py
|
|
18
19
|
./datalake/config/endpoints.json
|
|
@@ -56,6 +57,7 @@ setup.py
|
|
|
56
57
|
./tests/common/__init__.py
|
|
57
58
|
./tests/common/fixture.py
|
|
58
59
|
./tests/common/test_output.py
|
|
60
|
+
./tests/common/test_user_agent.py
|
|
59
61
|
./tests/common/test_utils.py
|
|
60
62
|
./tests/lib/__init__.py
|
|
61
63
|
./tests/lib/test_advanced_search.py
|
|
@@ -84,6 +86,7 @@ datalake/common/logger.py
|
|
|
84
86
|
datalake/common/output.py
|
|
85
87
|
datalake/common/throttler.py
|
|
86
88
|
datalake/common/token_manager.py
|
|
89
|
+
datalake/common/user_agent.py
|
|
87
90
|
datalake/common/utils.py
|
|
88
91
|
datalake/common/warn.py
|
|
89
92
|
datalake/config/endpoints.json
|
|
@@ -133,6 +136,7 @@ tests/conftest.py
|
|
|
133
136
|
tests/common/__init__.py
|
|
134
137
|
tests/common/fixture.py
|
|
135
138
|
tests/common/test_output.py
|
|
139
|
+
tests/common/test_user_agent.py
|
|
136
140
|
tests/common/test_utils.py
|
|
137
141
|
tests/lib/__init__.py
|
|
138
142
|
tests/lib/test_advanced_search.py
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from datalake.common import user_agent as user_agent_mod
|
|
4
|
+
from datalake.common.user_agent import (
|
|
5
|
+
USER_AGENT_INTEGRATION_ENV,
|
|
6
|
+
build_user_agent,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture
|
|
11
|
+
def fixed_version(monkeypatch):
|
|
12
|
+
monkeypatch.setattr(user_agent_mod, "_get_version", lambda: "9.9.9")
|
|
13
|
+
return "9.9.9"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_build_user_agent_default(monkeypatch, fixed_version):
|
|
17
|
+
monkeypatch.delenv(USER_AGENT_INTEGRATION_ENV, raising=False)
|
|
18
|
+
assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_build_user_agent_with_integration(monkeypatch, fixed_version):
|
|
22
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, "my-soar/2.1")
|
|
23
|
+
assert build_user_agent() == f"my-soar/2.1 datalake-python-sdk/{fixed_version}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.mark.parametrize("value", ["", " "])
|
|
27
|
+
def test_build_user_agent_ignores_empty_or_whitespace(
|
|
28
|
+
monkeypatch, fixed_version, value
|
|
29
|
+
):
|
|
30
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, value)
|
|
31
|
+
assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_build_user_agent_strips_surrounding_whitespace(monkeypatch, fixed_version):
|
|
35
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, " foo/1.0 ")
|
|
36
|
+
assert build_user_agent() == f"foo/1.0 datalake-python-sdk/{fixed_version}"
|
|
@@ -418,11 +418,26 @@ async def test_bulk_search_task_download_async_timeout(
|
|
|
418
418
|
response_context.add(
|
|
419
419
|
responses.POST, bs_status_url, json=bs_update_json, status=200
|
|
420
420
|
)
|
|
421
|
+
bs_cancel_url = (
|
|
422
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
423
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
424
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
425
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
426
|
+
)
|
|
427
|
+
)
|
|
428
|
+
response_context.add(responses.DELETE, bs_cancel_url, status=204)
|
|
421
429
|
|
|
422
430
|
task = bulk_search_task.download_async(timeout=0.2)
|
|
423
431
|
with pytest.raises(TimeoutError):
|
|
424
432
|
await task
|
|
425
433
|
|
|
434
|
+
cancel_calls = [
|
|
435
|
+
c
|
|
436
|
+
for c in response_context.calls
|
|
437
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
438
|
+
]
|
|
439
|
+
assert len(cancel_calls) == 1, "cancel() must be called on timeout"
|
|
440
|
+
|
|
426
441
|
|
|
427
442
|
@responses.activate
|
|
428
443
|
def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask):
|
|
@@ -437,10 +452,25 @@ def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask
|
|
|
437
452
|
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
438
453
|
)
|
|
439
454
|
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
455
|
+
bs_cancel_url = (
|
|
456
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
457
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
458
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
459
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
460
|
+
)
|
|
461
|
+
)
|
|
462
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
440
463
|
|
|
441
464
|
with pytest.raises(TimeoutError):
|
|
442
465
|
bulk_search_task.download_sync(timeout=0.2)
|
|
443
466
|
|
|
467
|
+
cancel_calls = [
|
|
468
|
+
c
|
|
469
|
+
for c in responses.calls
|
|
470
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
471
|
+
]
|
|
472
|
+
assert len(cancel_calls) == 1, "cancel() must be called on timeout"
|
|
473
|
+
|
|
444
474
|
|
|
445
475
|
@responses.activate
|
|
446
476
|
def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask):
|
|
@@ -453,7 +483,114 @@ def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask)
|
|
|
453
483
|
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
454
484
|
)
|
|
455
485
|
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
486
|
+
bs_cancel_url = (
|
|
487
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
488
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
489
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
490
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
491
|
+
)
|
|
492
|
+
)
|
|
493
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
456
494
|
|
|
457
495
|
with pytest.raises(BulkSearchFailedError) as err:
|
|
458
496
|
bulk_search_task.download_sync()
|
|
459
497
|
assert err.value.failed_state == BulkSearchTaskState.CANCELLED
|
|
498
|
+
|
|
499
|
+
cancel_calls = [
|
|
500
|
+
c
|
|
501
|
+
for c in responses.calls
|
|
502
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
503
|
+
]
|
|
504
|
+
assert len(cancel_calls) == 0, "cancel() must not be called on failed state"
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@pytest.mark.parametrize(
|
|
508
|
+
"failed_state",
|
|
509
|
+
[
|
|
510
|
+
"CANCELLED",
|
|
511
|
+
"FAILED_TIMEOUT",
|
|
512
|
+
"FAILED_ERROR",
|
|
513
|
+
"FAILED_QUOTA_EXCEEDED",
|
|
514
|
+
"FAILED_TOO_MANY_RESULTS",
|
|
515
|
+
],
|
|
516
|
+
)
|
|
517
|
+
@responses.activate
|
|
518
|
+
def test_bulk_search_task_download_sync_cancel_on_any_failed_state(
|
|
519
|
+
bulk_search_task: BulkSearchTask, failed_state
|
|
520
|
+
):
|
|
521
|
+
bulk_search_task.state = BulkSearchTaskState.IN_PROGRESS
|
|
522
|
+
bs_update_json = copy.deepcopy(bs_status_json)
|
|
523
|
+
bs_update_json["results"][0]["state"] = failed_state
|
|
524
|
+
bs_status_url = (
|
|
525
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
526
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
527
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
528
|
+
)
|
|
529
|
+
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
530
|
+
bs_cancel_url = (
|
|
531
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
532
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
533
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
534
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
535
|
+
)
|
|
536
|
+
)
|
|
537
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
538
|
+
|
|
539
|
+
with pytest.raises(BulkSearchFailedError) as err:
|
|
540
|
+
bulk_search_task.download_sync()
|
|
541
|
+
assert err.value.failed_state == BulkSearchTaskState[failed_state]
|
|
542
|
+
|
|
543
|
+
cancel_calls = [
|
|
544
|
+
c
|
|
545
|
+
for c in responses.calls
|
|
546
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
547
|
+
]
|
|
548
|
+
assert (
|
|
549
|
+
len(cancel_calls) == 0
|
|
550
|
+
), f"cancel() must not be called for state {failed_state}"
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
@responses.activate
|
|
554
|
+
def test_bulk_search_task_cancel(bulk_search_task: BulkSearchTask):
|
|
555
|
+
task_uuid = bulk_search_task.uuid
|
|
556
|
+
bs_cancel_url = (
|
|
557
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
558
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
559
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
560
|
+
"{task_uuid}", task_uuid
|
|
561
|
+
)
|
|
562
|
+
)
|
|
563
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
564
|
+
|
|
565
|
+
assert bulk_search_task.cancel() is None
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
@responses.activate
|
|
569
|
+
def test_bulk_search_task_cancel_not_found(bulk_search_task: BulkSearchTask):
|
|
570
|
+
task_uuid = bulk_search_task.uuid
|
|
571
|
+
bs_cancel_url = (
|
|
572
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
573
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
574
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
575
|
+
"{task_uuid}", task_uuid
|
|
576
|
+
)
|
|
577
|
+
)
|
|
578
|
+
responses.add(responses.DELETE, bs_cancel_url, json={}, status=404)
|
|
579
|
+
|
|
580
|
+
with pytest.raises(BulkSearchNotFound):
|
|
581
|
+
bulk_search_task.cancel()
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@responses.activate
|
|
585
|
+
def test_bulk_search_endpoint_cancel(datalake: Datalake):
|
|
586
|
+
task_uuid = "d9c00380-2784-4386-9bc3-aff35cfeeb41"
|
|
587
|
+
bs_cancel_url = (
|
|
588
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
589
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
590
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
591
|
+
"{task_uuid}", task_uuid
|
|
592
|
+
)
|
|
593
|
+
)
|
|
594
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
595
|
+
|
|
596
|
+
assert datalake.BulkSearch.cancel(task_uuid) is None
|
|
@@ -393,7 +393,7 @@ def test_submit_sightings_no_threat_types(datalake):
|
|
|
393
393
|
assert (
|
|
394
394
|
str(err.value)
|
|
395
395
|
== 'For POSITIVE and NEGATIVE sightings "threat_types" field is required and'
|
|
396
|
-
" has to be an instance of the
|
|
396
|
+
" has to be an instance of the ThreatType class."
|
|
397
397
|
)
|
|
398
398
|
|
|
399
399
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake/endpoints/filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/utils.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_comments.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/bulk_lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_my_user_info.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts/scripts/search_watch.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc6 → datalake_scripts-3.0.1}/tests/lib/test_filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|