datalake-scripts 3.0.0rc4__tar.gz → 3.0.0rc7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/PKG-INFO +16 -6
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/README.md +2 -1
- datalake_scripts-3.0.0rc7/datalake/config/endpoints.json +35 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/bulk_search.py +8 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/endpoint.py +47 -26
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sightings.py +36 -3
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/cli.py +1 -1
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/PKG-INFO +17 -7
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/SOURCES.txt +74 -1
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/entry_points.txt +0 -1
- datalake_scripts-3.0.0rc7/pyproject.toml +3 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/setup.cfg +3 -5
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/setup.py +17 -6
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_sightings.py +241 -2
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/atom.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/atom_type.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/bulk_search_task.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/config.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/logger.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/output.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/throttler.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/token_manager.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/utils.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/common/warn.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/datalake.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/comments.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/my_account.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/sources.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/tags.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/base_script.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_my_user_info.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/fixture.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/test_output.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/common/test_utils.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/conftest.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_atom_type.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_bulk_search.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_my_account.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_search_watch.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_tags.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_base_scripts.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_cli.py +0 -0
- {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: datalake_scripts
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.0rc7
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
7
7
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Platform: UNKNOWN
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.7
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
|
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Requires-Python: >=3.6
|
|
19
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: halo
|
|
20
|
+
Requires-Dist: prettytable
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: requires-dist
|
|
28
|
+
Dynamic: requires-python
|
|
29
|
+
Dynamic: summary
|
|
20
30
|
|
|
21
31
|
|
|
22
32
|
____ _ _ _ ____ _ _
|
|
@@ -115,7 +125,7 @@ We use the format accepted by the requests python library.
|
|
|
115
125
|
See its documenation for other possible kinds of proxy to set up.
|
|
116
126
|
|
|
117
127
|
|
|
118
|
-
#### Throttling
|
|
128
|
+
#### Throttling and retries
|
|
119
129
|
For throttling the requests, those two environment variables can be used:
|
|
120
130
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
121
131
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -123,8 +133,8 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
123
133
|
|
|
124
134
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
125
135
|
|
|
136
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
126
137
|
|
|
127
138
|
### Contributing
|
|
128
139
|
|
|
129
140
|
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
130
|
-
|
|
@@ -95,7 +95,7 @@ We use the format accepted by the requests python library.
|
|
|
95
95
|
See its documenation for other possible kinds of proxy to set up.
|
|
96
96
|
|
|
97
97
|
|
|
98
|
-
#### Throttling
|
|
98
|
+
#### Throttling and retries
|
|
99
99
|
For throttling the requests, those two environment variables can be used:
|
|
100
100
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
101
101
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -103,6 +103,7 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
103
103
|
|
|
104
104
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
105
105
|
|
|
106
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
106
107
|
|
|
107
108
|
### Contributing
|
|
108
109
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"main": {
|
|
3
|
+
"preprod": "https://ti2.extranet.mrti-center.com/api/",
|
|
4
|
+
"prod": "https://datalake.cert.orangecyberdefense.com/api/"
|
|
5
|
+
},
|
|
6
|
+
"endpoints": {
|
|
7
|
+
"token": "auth/token/",
|
|
8
|
+
"refresh-token": "auth/refresh-token/",
|
|
9
|
+
"advanced-queries-threats": "mrti/advanced-queries/threats/",
|
|
10
|
+
"advanced-queries-threats-query-hash": "mrti/advanced-queries/threats/{query_hash}/",
|
|
11
|
+
"atom-values": "mrti/atom-values/",
|
|
12
|
+
"bulk-search": "mrti/bulk-search/",
|
|
13
|
+
"bulk-search-tasks": "mrti/bulk-search/tasks/",
|
|
14
|
+
"bulk-search-task": "mrti/bulk-search/task/{task_uuid}/",
|
|
15
|
+
"bulk-manual-threats": "mrti/bulk-manual-threats/",
|
|
16
|
+
"bulk-manual-threats-task": "mrti/bulk-manual-threats/task/{task_uuid}/",
|
|
17
|
+
"sources": "mrti/sources/",
|
|
18
|
+
"threat-entity-filtered":"mrti/threat-entity/filtered/",
|
|
19
|
+
"threats-manual": "mrti/threats-manual/",
|
|
20
|
+
"threats-bulk-lookup": "mrti/threats/bulk-lookup/",
|
|
21
|
+
"threats": "mrti/threats/{hashkey}/",
|
|
22
|
+
"threats-lookup": "mrti/threats/lookup/",
|
|
23
|
+
"threats-comments": "mrti/threats/{hashkey}/comments/",
|
|
24
|
+
"threats-tags": "mrti/threats/{hashkey}/tags/",
|
|
25
|
+
"threats-atom-values-extract": "mrti/threats/atom-values-extract/",
|
|
26
|
+
"threats-bulk-scoring-edits": "mrti/threats/bulk-scoring-edits/",
|
|
27
|
+
"threats-sighting": "mrti/threats/sighting/",
|
|
28
|
+
"threats-bulk-sighting": "mrti/threats/bulk-sighting/",
|
|
29
|
+
"threats-sighting-file": "mrti/threats/sighting-file/{hashkey}/",
|
|
30
|
+
"threats-sighting-filtered": "mrti/threats/sighting/filtered/",
|
|
31
|
+
"users-me": "users/me/"
|
|
32
|
+
},
|
|
33
|
+
"threats-manual-bulk-size": 100,
|
|
34
|
+
"api_version": "v3/"
|
|
35
|
+
}
|
|
@@ -15,6 +15,8 @@ class BulkSearch(Endpoint):
|
|
|
15
15
|
query_body: dict = None,
|
|
16
16
|
query_hash: str = None,
|
|
17
17
|
query_fields: list = None,
|
|
18
|
+
indicators_only: bool = None,
|
|
19
|
+
indicators_and_threat_entities_only: bool = None,
|
|
18
20
|
) -> BulkSearchTask:
|
|
19
21
|
"""Creates a bulk search task"""
|
|
20
22
|
if not query_body and not query_hash:
|
|
@@ -27,6 +29,12 @@ class BulkSearch(Endpoint):
|
|
|
27
29
|
body["query_hash"] = query_hash
|
|
28
30
|
if for_stix_export:
|
|
29
31
|
body["for_stix_export"] = for_stix_export
|
|
32
|
+
if indicators_only:
|
|
33
|
+
body["indicators_only"] = indicators_only
|
|
34
|
+
if indicators_and_threat_entities_only:
|
|
35
|
+
body["indicators_and_threat_entities_only"] = (
|
|
36
|
+
indicators_and_threat_entities_only
|
|
37
|
+
)
|
|
30
38
|
|
|
31
39
|
url = self._build_url_for_endpoint("bulk-search")
|
|
32
40
|
response = self.datalake_requests(
|
|
@@ -9,6 +9,7 @@ from urllib.parse import urljoin
|
|
|
9
9
|
|
|
10
10
|
import requests
|
|
11
11
|
from requests import Response
|
|
12
|
+
from requests.adapters import HTTPAdapter, Retry
|
|
12
13
|
|
|
13
14
|
from datalake.common.output import Output
|
|
14
15
|
from datalake.common.throttler import throttle
|
|
@@ -17,12 +18,11 @@ from datalake.common.utils import get_error_message
|
|
|
17
18
|
|
|
18
19
|
OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
|
|
19
20
|
OCD_DTL_REQUESTS_PER_QUOTA_TIME = int(os.getenv("OCD_DTL_REQUESTS_PER_QUOTA_TIME", 5))
|
|
21
|
+
OCD_DTL_MAX_RETRIES = int(os.getenv("OCD_DTL_MAX_RETRIES", 3))
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
class Endpoint:
|
|
23
25
|
|
|
24
|
-
SET_MAX_RETRY = 3
|
|
25
|
-
|
|
26
26
|
def __init__(
|
|
27
27
|
self,
|
|
28
28
|
logger,
|
|
@@ -40,9 +40,25 @@ class Endpoint:
|
|
|
40
40
|
self.environment = environment
|
|
41
41
|
self.terminal_size = self._get_terminal_size()
|
|
42
42
|
self.token_manager = token_manager
|
|
43
|
-
self.SET_MAX_RETRY = 3
|
|
44
43
|
self.proxies = proxies
|
|
45
44
|
self.verify = verify
|
|
45
|
+
self.session = requests.Session()
|
|
46
|
+
|
|
47
|
+
# Configure HTTP retry policy
|
|
48
|
+
retry_policy = Retry(
|
|
49
|
+
total=OCD_DTL_MAX_RETRIES, # Number of retries
|
|
50
|
+
backoff_factor=1, # How much time between retries (exponential)
|
|
51
|
+
raise_on_status=False, # Raise an error when the number of retries is exhausted
|
|
52
|
+
status_forcelist=[
|
|
53
|
+
429,
|
|
54
|
+
500,
|
|
55
|
+
502,
|
|
56
|
+
503,
|
|
57
|
+
504,
|
|
58
|
+
], # Retry for those HTTP status codes
|
|
59
|
+
)
|
|
60
|
+
self.session.mount("http://", HTTPAdapter(max_retries=retry_policy))
|
|
61
|
+
self.session.mount("https://", HTTPAdapter(max_retries=retry_policy))
|
|
46
62
|
|
|
47
63
|
def _get_terminal_size(self) -> int:
|
|
48
64
|
"""Return the terminal size for pretty print"""
|
|
@@ -71,15 +87,17 @@ class Endpoint:
|
|
|
71
87
|
"""
|
|
72
88
|
Use it to request the API
|
|
73
89
|
"""
|
|
74
|
-
tries_left = self.SET_MAX_RETRY
|
|
75
90
|
|
|
76
|
-
|
|
91
|
+
self.logger.debug(self._pretty_debug_request(url, method, post_body, headers))
|
|
92
|
+
|
|
93
|
+
response = None
|
|
94
|
+
retry_auth_count = OCD_DTL_MAX_RETRIES
|
|
95
|
+
if OCD_DTL_MAX_RETRIES < 1:
|
|
96
|
+
retry_auth_count = 1
|
|
97
|
+
while retry_auth_count >= 0:
|
|
77
98
|
headers["Authorization"] = (
|
|
78
99
|
self.token_manager.access_token or self.token_manager.longterm_token
|
|
79
100
|
)
|
|
80
|
-
self.logger.debug(
|
|
81
|
-
self._pretty_debug_request(url, method, post_body, headers)
|
|
82
|
-
)
|
|
83
101
|
|
|
84
102
|
response = self._send_request(
|
|
85
103
|
self,
|
|
@@ -100,20 +118,23 @@ class Endpoint:
|
|
|
100
118
|
"Missing authorization header or Token Error. Updating token"
|
|
101
119
|
)
|
|
102
120
|
self.token_manager.process_auth_error(response.json())
|
|
103
|
-
elif response.status_code == 422:
|
|
104
|
-
json_resp = response.json()
|
|
105
|
-
try:
|
|
106
|
-
error_msg = get_error_message(json_resp)
|
|
107
|
-
except ValueError:
|
|
108
|
-
error_msg = response.text
|
|
109
|
-
raise ValueError(f"422 HTTP code: {error_msg}")
|
|
110
|
-
elif response.status_code < 200 or response.status_code > 299:
|
|
111
|
-
self.logger.error(
|
|
112
|
-
f"API returned non 2xx response code : {response.status_code}\n{response.text}\n Retrying"
|
|
113
|
-
)
|
|
114
121
|
else:
|
|
115
|
-
|
|
116
|
-
|
|
122
|
+
break
|
|
123
|
+
retry_auth_count -= 1
|
|
124
|
+
|
|
125
|
+
if response.status_code == 422:
|
|
126
|
+
json_resp = response.json()
|
|
127
|
+
try:
|
|
128
|
+
error_msg = get_error_message(json_resp)
|
|
129
|
+
except ValueError:
|
|
130
|
+
error_msg = response.text
|
|
131
|
+
raise ValueError(f"422 HTTP code: {error_msg}")
|
|
132
|
+
elif response.status_code < 200 or response.status_code > 299:
|
|
133
|
+
self.logger.error(
|
|
134
|
+
f"API returned non 2xx response code : {response.status_code}\n{response.text}"
|
|
135
|
+
)
|
|
136
|
+
else:
|
|
137
|
+
return response
|
|
117
138
|
self.logger.error("Request failed")
|
|
118
139
|
raise ValueError(f"{response.status_code}: {response.text.strip()}")
|
|
119
140
|
|
|
@@ -152,15 +173,15 @@ class Endpoint:
|
|
|
152
173
|
common_kwargs["proxies"] = proxies
|
|
153
174
|
|
|
154
175
|
if method == "get":
|
|
155
|
-
api_response =
|
|
176
|
+
api_response = self.session.get(**common_kwargs)
|
|
156
177
|
elif method == "post":
|
|
157
|
-
api_response =
|
|
178
|
+
api_response = self.session.post(**common_kwargs, data=json.dumps(data))
|
|
158
179
|
elif method == "delete":
|
|
159
|
-
api_response =
|
|
180
|
+
api_response = self.session.delete(**common_kwargs, data=json.dumps(data))
|
|
160
181
|
elif method == "patch":
|
|
161
|
-
api_response =
|
|
182
|
+
api_response = self.session.patch(**common_kwargs, data=json.dumps(data))
|
|
162
183
|
elif method == "put":
|
|
163
|
-
api_response =
|
|
184
|
+
api_response = self.session.put(**common_kwargs, data=json.dumps(data))
|
|
164
185
|
else:
|
|
165
186
|
self.logger.debug(
|
|
166
187
|
"ERROR : Wrong requests, please only do [get, post, put, patch, delete] method"
|
|
@@ -46,9 +46,9 @@ class Sightings(Endpoint):
|
|
|
46
46
|
impersonate_id: int = None,
|
|
47
47
|
):
|
|
48
48
|
"""
|
|
49
|
-
Submit a list of
|
|
49
|
+
Submit a list of atoms as 1 sighting.
|
|
50
50
|
Either threat "hashkeys" or list of atom objects in "atoms" is required.
|
|
51
|
-
Possible
|
|
51
|
+
Possible sighting's type: POSITIVE, NEGATIVE, NEUTRAL.
|
|
52
52
|
For POSITIVE and NEGATIVE sightings, "threat_types"
|
|
53
53
|
field is required.
|
|
54
54
|
End date timestamp should always be in the past.
|
|
@@ -73,6 +73,39 @@ class Sightings(Endpoint):
|
|
|
73
73
|
res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
|
|
74
74
|
return res
|
|
75
75
|
|
|
76
|
+
def bulk_submit_sightings(
|
|
77
|
+
self,
|
|
78
|
+
sightings: List[dict] = [],
|
|
79
|
+
):
|
|
80
|
+
"""
|
|
81
|
+
Submit a list of Sightings.
|
|
82
|
+
Input must be a list of valid dict.
|
|
83
|
+
Each dict is a sighting.
|
|
84
|
+
"""
|
|
85
|
+
payload = {}
|
|
86
|
+
list_sightings_for_url_call = []
|
|
87
|
+
for sighting in sightings:
|
|
88
|
+
payload_item = self._prepare_sightings_payload(
|
|
89
|
+
sighting.get("atoms"),
|
|
90
|
+
sighting.get("hashkeys"),
|
|
91
|
+
sighting.get("start_timestamp"),
|
|
92
|
+
sighting.get("end_timestamp"),
|
|
93
|
+
sighting.get("sighting_type"),
|
|
94
|
+
sighting.get("description_visibility"),
|
|
95
|
+
sighting.get("count"),
|
|
96
|
+
sighting.get("threat_types"),
|
|
97
|
+
sighting.get("tags"),
|
|
98
|
+
sighting.get("description"),
|
|
99
|
+
sighting.get("relation"),
|
|
100
|
+
sighting.get("editable"),
|
|
101
|
+
sighting.get("impersonate_id"),
|
|
102
|
+
)
|
|
103
|
+
list_sightings_for_url_call.append(payload_item)
|
|
104
|
+
payload["data"] = list_sightings_for_url_call
|
|
105
|
+
url = self._build_url_for_endpoint("threats-bulk-sighting")
|
|
106
|
+
res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
|
|
107
|
+
return res
|
|
108
|
+
|
|
76
109
|
@staticmethod
|
|
77
110
|
def _check_sightings_payload_parameters(
|
|
78
111
|
atoms,
|
|
@@ -99,7 +132,7 @@ class Sightings(Endpoint):
|
|
|
99
132
|
):
|
|
100
133
|
raise ValueError(
|
|
101
134
|
'For POSITIVE and NEGATIVE sightings "threat_types" field is required and has to be '
|
|
102
|
-
"an instance of the
|
|
135
|
+
"an instance of the ThreatType class."
|
|
103
136
|
)
|
|
104
137
|
elif threat_types:
|
|
105
138
|
raise ValueError(
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version: 3.0.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datalake_scripts
|
|
3
|
+
Version: 3.0.0rc7
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
7
7
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Platform: UNKNOWN
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.7
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
|
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Requires-Python: >=3.6
|
|
19
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: halo
|
|
20
|
+
Requires-Dist: prettytable
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: requires-dist
|
|
28
|
+
Dynamic: requires-python
|
|
29
|
+
Dynamic: summary
|
|
20
30
|
|
|
21
31
|
|
|
22
32
|
____ _ _ _ ____ _ _
|
|
@@ -115,7 +125,7 @@ We use the format accepted by the requests python library.
|
|
|
115
125
|
See its documenation for other possible kinds of proxy to set up.
|
|
116
126
|
|
|
117
127
|
|
|
118
|
-
#### Throttling
|
|
128
|
+
#### Throttling and retries
|
|
119
129
|
For throttling the requests, those two environment variables can be used:
|
|
120
130
|
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
121
131
|
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
@@ -123,8 +133,8 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
123
133
|
|
|
124
134
|
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
125
135
|
|
|
136
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
126
137
|
|
|
127
138
|
### Contributing
|
|
128
139
|
|
|
129
140
|
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
130
|
-
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/SOURCES.txt
RENAMED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
README.md
|
|
2
|
+
pyproject.toml
|
|
2
3
|
setup.cfg
|
|
3
4
|
setup.py
|
|
4
5
|
./datalake/__init__.py
|
|
@@ -14,6 +15,7 @@ setup.py
|
|
|
14
15
|
./datalake/common/token_manager.py
|
|
15
16
|
./datalake/common/utils.py
|
|
16
17
|
./datalake/common/warn.py
|
|
18
|
+
./datalake/config/endpoints.json
|
|
17
19
|
./datalake/endpoints/__init__.py
|
|
18
20
|
./datalake/endpoints/advanced_search.py
|
|
19
21
|
./datalake/endpoints/bulk_search.py
|
|
@@ -71,9 +73,80 @@ setup.py
|
|
|
71
73
|
./tests/scripts/test_base_scripts.py
|
|
72
74
|
./tests/scripts/test_cli.py
|
|
73
75
|
./tests/scripts/test_csv_builder.py
|
|
76
|
+
datalake/__init__.py
|
|
77
|
+
datalake/datalake.py
|
|
78
|
+
datalake/common/__init__.py
|
|
79
|
+
datalake/common/atom.py
|
|
80
|
+
datalake/common/atom_type.py
|
|
81
|
+
datalake/common/bulk_search_task.py
|
|
82
|
+
datalake/common/config.py
|
|
83
|
+
datalake/common/logger.py
|
|
84
|
+
datalake/common/output.py
|
|
85
|
+
datalake/common/throttler.py
|
|
86
|
+
datalake/common/token_manager.py
|
|
87
|
+
datalake/common/utils.py
|
|
88
|
+
datalake/common/warn.py
|
|
89
|
+
datalake/config/endpoints.json
|
|
90
|
+
datalake/endpoints/__init__.py
|
|
91
|
+
datalake/endpoints/advanced_search.py
|
|
92
|
+
datalake/endpoints/bulk_search.py
|
|
93
|
+
datalake/endpoints/comments.py
|
|
94
|
+
datalake/endpoints/endpoint.py
|
|
95
|
+
datalake/endpoints/filtered_threat_entity.py
|
|
96
|
+
datalake/endpoints/my_account.py
|
|
97
|
+
datalake/endpoints/sightings.py
|
|
98
|
+
datalake/endpoints/sources.py
|
|
99
|
+
datalake/endpoints/tags.py
|
|
100
|
+
datalake/endpoints/threats.py
|
|
101
|
+
datalake/miscellaneous/__init__.py
|
|
102
|
+
datalake/miscellaneous/search_watch.py
|
|
103
|
+
datalake_scripts/__init__.py
|
|
104
|
+
datalake_scripts/cli.py
|
|
74
105
|
datalake_scripts.egg-info/PKG-INFO
|
|
75
106
|
datalake_scripts.egg-info/SOURCES.txt
|
|
76
107
|
datalake_scripts.egg-info/dependency_links.txt
|
|
77
108
|
datalake_scripts.egg-info/entry_points.txt
|
|
78
109
|
datalake_scripts.egg-info/requires.txt
|
|
79
|
-
datalake_scripts.egg-info/top_level.txt
|
|
110
|
+
datalake_scripts.egg-info/top_level.txt
|
|
111
|
+
datalake_scripts/common/__init__.py
|
|
112
|
+
datalake_scripts/common/base_script.py
|
|
113
|
+
datalake_scripts/helper_scripts/__init__.py
|
|
114
|
+
datalake_scripts/helper_scripts/output_builder.py
|
|
115
|
+
datalake_scripts/helper_scripts/utils.py
|
|
116
|
+
datalake_scripts/scripts/__init__.py
|
|
117
|
+
datalake_scripts/scripts/add_comments.py
|
|
118
|
+
datalake_scripts/scripts/add_tags.py
|
|
119
|
+
datalake_scripts/scripts/add_threats.py
|
|
120
|
+
datalake_scripts/scripts/advanced_search.py
|
|
121
|
+
datalake_scripts/scripts/bulk_lookup_threats.py
|
|
122
|
+
datalake_scripts/scripts/edit_score.py
|
|
123
|
+
datalake_scripts/scripts/get_atom_values.py
|
|
124
|
+
datalake_scripts/scripts/get_filtered_threat_entity.py
|
|
125
|
+
datalake_scripts/scripts/get_my_user_info.py
|
|
126
|
+
datalake_scripts/scripts/get_query_hash.py
|
|
127
|
+
datalake_scripts/scripts/get_threats.py
|
|
128
|
+
datalake_scripts/scripts/get_threats_from_query_hash.py
|
|
129
|
+
datalake_scripts/scripts/lookup_threats.py
|
|
130
|
+
datalake_scripts/scripts/search_watch.py
|
|
131
|
+
tests/__init__.py
|
|
132
|
+
tests/conftest.py
|
|
133
|
+
tests/common/__init__.py
|
|
134
|
+
tests/common/fixture.py
|
|
135
|
+
tests/common/test_output.py
|
|
136
|
+
tests/common/test_utils.py
|
|
137
|
+
tests/lib/__init__.py
|
|
138
|
+
tests/lib/test_advanced_search.py
|
|
139
|
+
tests/lib/test_atom_type.py
|
|
140
|
+
tests/lib/test_auth_via_tokens.py
|
|
141
|
+
tests/lib/test_bulk_search.py
|
|
142
|
+
tests/lib/test_filtered_threat_entity.py
|
|
143
|
+
tests/lib/test_my_account.py
|
|
144
|
+
tests/lib/test_search_watch.py
|
|
145
|
+
tests/lib/test_sightings.py
|
|
146
|
+
tests/lib/test_tags.py
|
|
147
|
+
tests/lib/test_threats.py
|
|
148
|
+
tests/scripts/__init__.py
|
|
149
|
+
tests/scripts/test_add_new_threats.py
|
|
150
|
+
tests/scripts/test_base_scripts.py
|
|
151
|
+
tests/scripts/test_cli.py
|
|
152
|
+
tests/scripts/test_csv_builder.py
|
|
@@ -2,20 +2,31 @@ import os
|
|
|
2
2
|
|
|
3
3
|
from setuptools import find_packages, setup
|
|
4
4
|
|
|
5
|
-
from datalake_scripts.cli import Cli
|
|
6
|
-
|
|
7
5
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
8
6
|
|
|
9
7
|
with open("README.md", "r") as fh:
|
|
10
|
-
|
|
8
|
+
LONG_DESCRIPTION = fh.read()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def read_version_or_cli_name(option: str):
|
|
12
|
+
with open("datalake_scripts/cli.py") as f:
|
|
13
|
+
for line in f:
|
|
14
|
+
if option == "version" and line.strip().startswith("VERSION"):
|
|
15
|
+
return line.split("=")[1].strip().strip('"')
|
|
16
|
+
if option == "cli_name" and line.strip().startswith("CLI_NAME"):
|
|
17
|
+
return line.split("=")[1].strip().strip('"')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
CLI_NAME = read_version_or_cli_name("cli_name")
|
|
21
|
+
VERSION = read_version_or_cli_name("version")
|
|
11
22
|
|
|
12
23
|
setup(
|
|
13
24
|
name="datalake_scripts",
|
|
14
|
-
version=
|
|
25
|
+
version=VERSION,
|
|
15
26
|
author="OCD",
|
|
16
27
|
author_email="cert-contact.ocd@orange.com",
|
|
17
28
|
description="A collection of scripts to easily use the API of OCD Datalake",
|
|
18
|
-
long_description=
|
|
29
|
+
long_description=LONG_DESCRIPTION,
|
|
19
30
|
long_description_content_type="text/markdown",
|
|
20
31
|
url="https://github.com/cert-orangecyberdefense/datalake/",
|
|
21
32
|
package_dir={
|
|
@@ -25,7 +36,7 @@ setup(
|
|
|
25
36
|
install_requires=["requests", "halo", "prettytable"],
|
|
26
37
|
test_suite="nose.collector",
|
|
27
38
|
entry_points={
|
|
28
|
-
"console_scripts": (f"{
|
|
39
|
+
"console_scripts": (f"{CLI_NAME} = datalake_scripts.cli:main",),
|
|
29
40
|
},
|
|
30
41
|
classifiers=[
|
|
31
42
|
"Programming Language :: Python :: 3.7",
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
from unittest.mock import patch
|
|
2
2
|
import pytest
|
|
3
3
|
import responses
|
|
4
|
-
from datetime import datetime
|
|
4
|
+
from datetime import datetime, timedelta, timezone
|
|
5
5
|
|
|
6
6
|
from responses import matchers
|
|
7
7
|
import os
|
|
8
|
+
import ast
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
from tests.common.fixture import TestData, datalake # noqa needed fixture import
|
|
11
12
|
from datalake import (
|
|
12
13
|
IpAtom,
|
|
13
14
|
FileAtom,
|
|
15
|
+
EmailAtom,
|
|
16
|
+
UrlAtom,
|
|
14
17
|
Hashes,
|
|
15
18
|
Jarm,
|
|
16
19
|
IpService,
|
|
@@ -390,7 +393,7 @@ def test_submit_sightings_no_threat_types(datalake):
|
|
|
390
393
|
assert (
|
|
391
394
|
str(err.value)
|
|
392
395
|
== 'For POSITIVE and NEGATIVE sightings "threat_types" field is required and'
|
|
393
|
-
" has to be an instance of the
|
|
396
|
+
" has to be an instance of the ThreatType class."
|
|
394
397
|
)
|
|
395
398
|
|
|
396
399
|
|
|
@@ -448,6 +451,242 @@ def test_submit_sightings_bad_atom(datalake):
|
|
|
448
451
|
assert str(err.value) == '"atoms" needs to be a list of Atom subclasses.'
|
|
449
452
|
|
|
450
453
|
|
|
454
|
+
@responses.activate
|
|
455
|
+
def test_bulk_submit_sightings_success(datalake):
|
|
456
|
+
url = (
|
|
457
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
458
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
459
|
+
+ TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
f1 = FileAtom(
|
|
463
|
+
hashes=Hashes(
|
|
464
|
+
md5="d41d8cd98f00b204e9800998ecf8427e",
|
|
465
|
+
sha1="da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
466
|
+
sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
467
|
+
)
|
|
468
|
+
)
|
|
469
|
+
ip1 = IpAtom("52.48.79.33")
|
|
470
|
+
em1 = EmailAtom("hacker@hacker.fr")
|
|
471
|
+
url1 = UrlAtom("http://notfishing.com")
|
|
472
|
+
|
|
473
|
+
# You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
|
|
474
|
+
|
|
475
|
+
## Prepare threat types for this sighting
|
|
476
|
+
threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
|
|
477
|
+
|
|
478
|
+
## Prepare start and end timestamps for this sighting
|
|
479
|
+
start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
|
|
480
|
+
end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
|
|
481
|
+
input_sighting = {
|
|
482
|
+
"atoms": [ip1, f1, em1, url1],
|
|
483
|
+
"start_timestamp": start,
|
|
484
|
+
"end_timestamp": end,
|
|
485
|
+
"sighting_type": SightingType.POSITIVE,
|
|
486
|
+
"description_visibility": Visibility.PUBLIC,
|
|
487
|
+
"count": 1,
|
|
488
|
+
"threat_types": threat_types,
|
|
489
|
+
"tags": ["some_tag"],
|
|
490
|
+
"description": "some_description",
|
|
491
|
+
"editable": True,
|
|
492
|
+
}
|
|
493
|
+
expected_request = {
|
|
494
|
+
"data": [
|
|
495
|
+
{
|
|
496
|
+
"count": 1,
|
|
497
|
+
"description": "some_description",
|
|
498
|
+
"description_visibility": "PUBLIC",
|
|
499
|
+
"editable": True,
|
|
500
|
+
"email_list": [{"email": "hacker@hacker.fr"}],
|
|
501
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
502
|
+
"file_list": [
|
|
503
|
+
{
|
|
504
|
+
"hashes": {
|
|
505
|
+
"md5": "d41d8cd98f00b204e9800998ecf8427e",
|
|
506
|
+
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
507
|
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
],
|
|
511
|
+
"ip_list": [{"ip_address": "52.48.79.33"}],
|
|
512
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
513
|
+
"tags": ["some_tag"],
|
|
514
|
+
"threat_types": ["phishing", "scam"],
|
|
515
|
+
"type": "positive",
|
|
516
|
+
"url_list": [{"url": "http://notfishing.com"}],
|
|
517
|
+
}
|
|
518
|
+
]
|
|
519
|
+
}
|
|
520
|
+
expected_res = {
|
|
521
|
+
"count": 1,
|
|
522
|
+
"results": [
|
|
523
|
+
{
|
|
524
|
+
"count": 1,
|
|
525
|
+
"description": "some_description",
|
|
526
|
+
"description_visibility": "PUBLIC",
|
|
527
|
+
"editable": True,
|
|
528
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
529
|
+
"relation_type": "sighting",
|
|
530
|
+
"reliability": 100,
|
|
531
|
+
"sighting_version": "3.0.0",
|
|
532
|
+
"sightings": {
|
|
533
|
+
"email_list": [{"email": "hacker@hacker.fr"}],
|
|
534
|
+
"file_list": [
|
|
535
|
+
{
|
|
536
|
+
"hashes": {
|
|
537
|
+
"md5": "d41d8cd98f00b204e9800998ecf8427e",
|
|
538
|
+
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
539
|
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
540
|
+
"sha512": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
],
|
|
544
|
+
"ip_list": [{"ip_address": "52.48.79.33"}],
|
|
545
|
+
"url_list": [{"url": "http://notfishing.com"}],
|
|
546
|
+
},
|
|
547
|
+
"source_context": {
|
|
548
|
+
"source_id": "org: Orange Cyberdefense",
|
|
549
|
+
"source_uses": ["sightings"],
|
|
550
|
+
},
|
|
551
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
552
|
+
"tags": [{"categories": [], "name": "some_tag"}],
|
|
553
|
+
"threat_types": ["phishing", "scam"],
|
|
554
|
+
"timestamp_created": "2025-12-16T10:00:00Z",
|
|
555
|
+
"type": "positive",
|
|
556
|
+
"uid": "e14dd426-ed41-436c-9e0e-0d785e750428",
|
|
557
|
+
}
|
|
558
|
+
],
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
responses.post(
|
|
562
|
+
url=url,
|
|
563
|
+
json=expected_res,
|
|
564
|
+
status=200,
|
|
565
|
+
match=[matchers.json_params_matcher(expected_request)],
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
with patch.dict(
|
|
569
|
+
os.environ,
|
|
570
|
+
{
|
|
571
|
+
"IGNORE_SIGHTING_BUILDER_WARNING": "1",
|
|
572
|
+
},
|
|
573
|
+
clear=True,
|
|
574
|
+
):
|
|
575
|
+
res = datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
|
|
576
|
+
|
|
577
|
+
assert res == expected_res
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def test_bulk_submit_sightings_payload_error(datalake):
|
|
581
|
+
with pytest.raises(ValueError) as err:
|
|
582
|
+
datalake.Sightings.bulk_submit_sightings(
|
|
583
|
+
sightings=[
|
|
584
|
+
{
|
|
585
|
+
"atoms": [FileAtom(Hashes(md5="d41d8cd98f00b204e9800998ecf8427e"))],
|
|
586
|
+
"start_timestamp": start,
|
|
587
|
+
"end_timestamp": end,
|
|
588
|
+
"sighting_type": SightingType.NEUTRAL,
|
|
589
|
+
"description_visibility": Visibility.PUBLIC,
|
|
590
|
+
"count": 3,
|
|
591
|
+
"threat_types": threat_types,
|
|
592
|
+
"tags": ["some_tag_bis"],
|
|
593
|
+
"description": "some_description_bis",
|
|
594
|
+
"editable": False,
|
|
595
|
+
}
|
|
596
|
+
]
|
|
597
|
+
)
|
|
598
|
+
assert (
|
|
599
|
+
str(err.value) == """For NEUTRAL sightings, "threat_types" can't be passed."""
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
@responses.activate
|
|
604
|
+
def test_bulk_submit_sightings_failure(datalake):
|
|
605
|
+
url = (
|
|
606
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
607
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
608
|
+
+ TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
hashkeys = [
|
|
612
|
+
"d41d8cd98f00b204e9800998ecf8427e",
|
|
613
|
+
"a7b25b324871a7695aa2cc5d09681dda",
|
|
614
|
+
"1ed07771327e850255b09b042ad00e3d",
|
|
615
|
+
"bf1f33c3a56e1dfda6a2f4f3d3e4361a",
|
|
616
|
+
]
|
|
617
|
+
|
|
618
|
+
# You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
|
|
619
|
+
|
|
620
|
+
## Prepare threat types for this sighting
|
|
621
|
+
threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
|
|
622
|
+
|
|
623
|
+
## Prepare start and end timestamps for this sighting
|
|
624
|
+
start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
|
|
625
|
+
end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
|
|
626
|
+
input_sighting = {
|
|
627
|
+
"hashkeys": hashkeys,
|
|
628
|
+
"start_timestamp": start,
|
|
629
|
+
"end_timestamp": end,
|
|
630
|
+
"sighting_type": SightingType.POSITIVE,
|
|
631
|
+
"description_visibility": Visibility.PUBLIC,
|
|
632
|
+
"count": 1,
|
|
633
|
+
"threat_types": threat_types,
|
|
634
|
+
"tags": ["some_tag"],
|
|
635
|
+
"description": "some_description",
|
|
636
|
+
"editable": True,
|
|
637
|
+
}
|
|
638
|
+
expected_request = {
|
|
639
|
+
"data": [
|
|
640
|
+
{
|
|
641
|
+
"count": 1,
|
|
642
|
+
"description": "some_description",
|
|
643
|
+
"description_visibility": "PUBLIC",
|
|
644
|
+
"editable": True,
|
|
645
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
646
|
+
"hashkeys": hashkeys,
|
|
647
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
648
|
+
"tags": ["some_tag"],
|
|
649
|
+
"threat_types": ["phishing", "scam"],
|
|
650
|
+
"type": "positive",
|
|
651
|
+
}
|
|
652
|
+
]
|
|
653
|
+
}
|
|
654
|
+
expected_res = {
|
|
655
|
+
"data": {
|
|
656
|
+
"0": {
|
|
657
|
+
"hashkeys": [
|
|
658
|
+
"Some of the provided hashkeys has not been found: "
|
|
659
|
+
"['d41d8cd98f00b204e9800998ecf8427e']"
|
|
660
|
+
]
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
responses.post(
|
|
666
|
+
url=url,
|
|
667
|
+
json=expected_res,
|
|
668
|
+
status=422,
|
|
669
|
+
match=[matchers.json_params_matcher(expected_request)],
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
with patch.dict(
|
|
673
|
+
os.environ,
|
|
674
|
+
{
|
|
675
|
+
"IGNORE_SIGHTING_BUILDER_WARNING": "1",
|
|
676
|
+
},
|
|
677
|
+
clear=True,
|
|
678
|
+
):
|
|
679
|
+
with pytest.raises(ValueError) as exc:
|
|
680
|
+
datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
|
|
681
|
+
|
|
682
|
+
prefix = "422 HTTP code: "
|
|
683
|
+
msg = str(exc.value)
|
|
684
|
+
assert msg.startswith(prefix)
|
|
685
|
+
|
|
686
|
+
payload = ast.literal_eval(msg[len(prefix) :])
|
|
687
|
+
assert payload == expected_res
|
|
688
|
+
|
|
689
|
+
|
|
451
690
|
def test_sightings_filtered_bad_ordering(datalake):
|
|
452
691
|
with pytest.raises(ValueError) as err:
|
|
453
692
|
datalake.Sightings.sightings_filtered(ordering="bad_ordering")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/advanced_search.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/endpoints/filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake/miscellaneous/search_watch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/common/base_script.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/helper_scripts/utils.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/__init__.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_comments.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_tags.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/add_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/edit_score.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_my_user_info.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/get_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts/scripts/search_watch.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/requires.txt
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/datalake_scripts.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/lib/test_filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc7}/tests/scripts/test_add_new_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|