datalake-scripts 3.0.0rc3__tar.gz → 3.0.0rc6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/PKG-INFO +14 -5
- datalake_scripts-3.0.0rc6/datalake/config/endpoints.json +35 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/datalake.py +9 -0
- datalake_scripts-3.0.0rc6/datalake/endpoints/my_account.py +17 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sightings.py +35 -2
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/cli.py +11 -3
- datalake_scripts-3.0.0rc6/datalake_scripts/scripts/get_my_user_info.py +32 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/PKG-INFO +15 -6
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/SOURCES.txt +77 -1
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/entry_points.txt +0 -1
- datalake_scripts-3.0.0rc6/pyproject.toml +3 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/setup.cfg +3 -5
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/setup.py +17 -6
- datalake_scripts-3.0.0rc6/tests/lib/test_my_account.py +51 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_sightings.py +240 -1
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_cli.py +2 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/README.md +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/atom.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/atom_type.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/bulk_search_task.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/config.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/logger.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/output.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/throttler.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/token_manager.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/utils.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/warn.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/bulk_search.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/comments.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/endpoint.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sources.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/tags.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/base_script.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/fixture.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/test_output.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/test_utils.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/conftest.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_atom_type.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_bulk_search.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_search_watch.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_tags.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_base_scripts.py +0 -0
- {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: datalake_scripts
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.0rc6
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
7
7
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Platform: UNKNOWN
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.7
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
|
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Requires-Python: >=3.6
|
|
19
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: halo
|
|
20
|
+
Requires-Dist: prettytable
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: requires-dist
|
|
28
|
+
Dynamic: requires-python
|
|
29
|
+
Dynamic: summary
|
|
20
30
|
|
|
21
31
|
|
|
22
32
|
____ _ _ _ ____ _ _
|
|
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
127
137
|
### Contributing
|
|
128
138
|
|
|
129
139
|
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
130
|
-
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"main": {
|
|
3
|
+
"preprod": "https://ti2.extranet.mrti-center.com/api/",
|
|
4
|
+
"prod": "https://datalake.cert.orangecyberdefense.com/api/"
|
|
5
|
+
},
|
|
6
|
+
"endpoints": {
|
|
7
|
+
"token": "auth/token/",
|
|
8
|
+
"refresh-token": "auth/refresh-token/",
|
|
9
|
+
"advanced-queries-threats": "mrti/advanced-queries/threats/",
|
|
10
|
+
"advanced-queries-threats-query-hash": "mrti/advanced-queries/threats/{query_hash}/",
|
|
11
|
+
"atom-values": "mrti/atom-values/",
|
|
12
|
+
"bulk-search": "mrti/bulk-search/",
|
|
13
|
+
"bulk-search-tasks": "mrti/bulk-search/tasks/",
|
|
14
|
+
"bulk-search-task": "mrti/bulk-search/task/{task_uuid}/",
|
|
15
|
+
"bulk-manual-threats": "mrti/bulk-manual-threats/",
|
|
16
|
+
"bulk-manual-threats-task": "mrti/bulk-manual-threats/task/{task_uuid}/",
|
|
17
|
+
"sources": "mrti/sources/",
|
|
18
|
+
"threat-entity-filtered":"mrti/threat-entity/filtered/",
|
|
19
|
+
"threats-manual": "mrti/threats-manual/",
|
|
20
|
+
"threats-bulk-lookup": "mrti/threats/bulk-lookup/",
|
|
21
|
+
"threats": "mrti/threats/{hashkey}/",
|
|
22
|
+
"threats-lookup": "mrti/threats/lookup/",
|
|
23
|
+
"threats-comments": "mrti/threats/{hashkey}/comments/",
|
|
24
|
+
"threats-tags": "mrti/threats/{hashkey}/tags/",
|
|
25
|
+
"threats-atom-values-extract": "mrti/threats/atom-values-extract/",
|
|
26
|
+
"threats-bulk-scoring-edits": "mrti/threats/bulk-scoring-edits/",
|
|
27
|
+
"threats-sighting": "mrti/threats/sighting/",
|
|
28
|
+
"threats-bulk-sighting": "mrti/threats/bulk-sighting/",
|
|
29
|
+
"threats-sighting-file": "mrti/threats/sighting-file/{hashkey}/",
|
|
30
|
+
"threats-sighting-filtered": "mrti/threats/sighting/filtered/",
|
|
31
|
+
"users-me": "users/me/"
|
|
32
|
+
},
|
|
33
|
+
"threats-manual-bulk-size": 100,
|
|
34
|
+
"api_version": "v3/"
|
|
35
|
+
}
|
|
@@ -11,6 +11,7 @@ from datalake.endpoints.advanced_search import AdvancedSearch
|
|
|
11
11
|
from datalake.endpoints.sightings import Sightings
|
|
12
12
|
from datalake.endpoints.sources import Sources
|
|
13
13
|
from datalake.endpoints.filtered_threat_entity import FilteredThreatEntity
|
|
14
|
+
from datalake.endpoints.my_account import MyAccount
|
|
14
15
|
from datalake.miscellaneous.search_watch import SearchWatch
|
|
15
16
|
|
|
16
17
|
|
|
@@ -58,6 +59,14 @@ class Datalake:
|
|
|
58
59
|
raise
|
|
59
60
|
|
|
60
61
|
# Endpoints
|
|
62
|
+
self.MyAccount = MyAccount(
|
|
63
|
+
self.logger,
|
|
64
|
+
endpoint_config,
|
|
65
|
+
env,
|
|
66
|
+
token_manager,
|
|
67
|
+
proxies=proxies,
|
|
68
|
+
verify=verify,
|
|
69
|
+
)
|
|
61
70
|
self.AdvancedSearch = AdvancedSearch(
|
|
62
71
|
self.logger,
|
|
63
72
|
endpoint_config,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from typing import List, Tuple
|
|
2
|
+
|
|
3
|
+
from datalake.endpoints.endpoint import Endpoint
|
|
4
|
+
from datalake.common.output import parse_response
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class MyAccount(Endpoint):
|
|
8
|
+
|
|
9
|
+
def me(self):
|
|
10
|
+
"""
|
|
11
|
+
Gets details of the currently logged in user.
|
|
12
|
+
"""
|
|
13
|
+
url = self._build_url_for_endpoint("users-me")
|
|
14
|
+
response = parse_response(
|
|
15
|
+
self.datalake_requests(url, "get", self._get_headers())
|
|
16
|
+
)
|
|
17
|
+
return response
|
|
@@ -46,9 +46,9 @@ class Sightings(Endpoint):
|
|
|
46
46
|
impersonate_id: int = None,
|
|
47
47
|
):
|
|
48
48
|
"""
|
|
49
|
-
Submit a list of
|
|
49
|
+
Submit a list of atoms as 1 sighting.
|
|
50
50
|
Either threat "hashkeys" or list of atom objects in "atoms" is required.
|
|
51
|
-
Possible
|
|
51
|
+
Possible sighting's type: POSITIVE, NEGATIVE, NEUTRAL.
|
|
52
52
|
For POSITIVE and NEGATIVE sightings, "threat_types"
|
|
53
53
|
field is required.
|
|
54
54
|
End date timestamp should always be in the past.
|
|
@@ -73,6 +73,39 @@ class Sightings(Endpoint):
|
|
|
73
73
|
res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
|
|
74
74
|
return res
|
|
75
75
|
|
|
76
|
+
def bulk_submit_sightings(
|
|
77
|
+
self,
|
|
78
|
+
sightings: List[dict] = [],
|
|
79
|
+
):
|
|
80
|
+
"""
|
|
81
|
+
Submit a list of Sightings.
|
|
82
|
+
Input must be a list of valid dict.
|
|
83
|
+
Each dict is a sighting.
|
|
84
|
+
"""
|
|
85
|
+
payload = {}
|
|
86
|
+
list_sightings_for_url_call = []
|
|
87
|
+
for sighting in sightings:
|
|
88
|
+
payload_item = self._prepare_sightings_payload(
|
|
89
|
+
sighting.get("atoms"),
|
|
90
|
+
sighting.get("hashkeys"),
|
|
91
|
+
sighting.get("start_timestamp"),
|
|
92
|
+
sighting.get("end_timestamp"),
|
|
93
|
+
sighting.get("sighting_type"),
|
|
94
|
+
sighting.get("description_visibility"),
|
|
95
|
+
sighting.get("count"),
|
|
96
|
+
sighting.get("threat_types"),
|
|
97
|
+
sighting.get("tags"),
|
|
98
|
+
sighting.get("description"),
|
|
99
|
+
sighting.get("relation"),
|
|
100
|
+
sighting.get("editable"),
|
|
101
|
+
sighting.get("impersonate_id"),
|
|
102
|
+
)
|
|
103
|
+
list_sightings_for_url_call.append(payload_item)
|
|
104
|
+
payload["data"] = list_sightings_for_url_call
|
|
105
|
+
url = self._build_url_for_endpoint("threats-bulk-sighting")
|
|
106
|
+
res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
|
|
107
|
+
return res
|
|
108
|
+
|
|
76
109
|
@staticmethod
|
|
77
110
|
def _check_sightings_payload_parameters(
|
|
78
111
|
atoms,
|
|
@@ -15,12 +15,13 @@ from datalake_scripts.scripts import (
|
|
|
15
15
|
get_atom_values,
|
|
16
16
|
get_filtered_threat_entity,
|
|
17
17
|
search_watch,
|
|
18
|
+
get_my_user_info,
|
|
18
19
|
)
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
class Cli:
|
|
22
23
|
CLI_NAME = "ocd-dtl"
|
|
23
|
-
VERSION = "3.0.
|
|
24
|
+
VERSION = "3.0.0rc6"
|
|
24
25
|
|
|
25
26
|
def __init__(self):
|
|
26
27
|
parser = argparse.ArgumentParser(
|
|
@@ -54,8 +55,15 @@ class Cli:
|
|
|
54
55
|
get_filtered_threat_entity.main,
|
|
55
56
|
)
|
|
56
57
|
self._add_command_subparser(subparsers, "search_watch", search_watch.main)
|
|
58
|
+
self._add_command_subparser(
|
|
59
|
+
subparsers, "get_my_user_info", get_my_user_info.main
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# args = parser.parse_args(sys.argv[1:2])
|
|
63
|
+
command_called = sys.argv[1:2]
|
|
64
|
+
command_arguments = sys.argv[2:]
|
|
57
65
|
|
|
58
|
-
args = parser.parse_args(
|
|
66
|
+
args = parser.parse_args(command_called)
|
|
59
67
|
|
|
60
68
|
if args.version:
|
|
61
69
|
print(self.VERSION)
|
|
@@ -67,7 +75,7 @@ class Cli:
|
|
|
67
75
|
exit(1)
|
|
68
76
|
|
|
69
77
|
# Call the subcommand method
|
|
70
|
-
args.func(
|
|
78
|
+
args.func(command_arguments)
|
|
71
79
|
|
|
72
80
|
def _add_command_subparser(self, subparsers, name, method):
|
|
73
81
|
command_parser = subparsers.add_parser(name)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from datalake import Datalake
|
|
4
|
+
from datalake_scripts.common.base_script import BaseScripts
|
|
5
|
+
from datalake_scripts.helper_scripts.utils import save_output
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_my_user_info_func(dtl: Datalake):
|
|
9
|
+
response = dtl.MyAccount.me()
|
|
10
|
+
return response
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(override_args=None):
|
|
14
|
+
"""Method to start the script"""
|
|
15
|
+
# Load initial args
|
|
16
|
+
parser = BaseScripts.start("Gets details of the currently logged in user.")
|
|
17
|
+
|
|
18
|
+
args = parser.parse_args(override_args or [])
|
|
19
|
+
|
|
20
|
+
dtl = Datalake(env=args.env, log_level=args.loglevel)
|
|
21
|
+
response = get_my_user_info_func(dtl)
|
|
22
|
+
|
|
23
|
+
if args.output:
|
|
24
|
+
save_output(args.output, response)
|
|
25
|
+
dtl.logger.debug(f"Results saved in {args.output}\n")
|
|
26
|
+
else:
|
|
27
|
+
print(response)
|
|
28
|
+
dtl.logger.debug(f"END: get_my_user_info.py")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
sys.exit(main())
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version: 3.0.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datalake_scripts
|
|
3
|
+
Version: 3.0.0rc6
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
7
7
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Platform: UNKNOWN
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.7
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
|
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Requires-Python: >=3.6
|
|
19
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: halo
|
|
20
|
+
Requires-Dist: prettytable
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: requires-dist
|
|
28
|
+
Dynamic: requires-python
|
|
29
|
+
Dynamic: summary
|
|
20
30
|
|
|
21
31
|
|
|
22
32
|
____ _ _ _ ____ _ _
|
|
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
|
|
|
127
137
|
### Contributing
|
|
128
138
|
|
|
129
139
|
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
130
|
-
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/SOURCES.txt
RENAMED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
README.md
|
|
2
|
+
pyproject.toml
|
|
2
3
|
setup.cfg
|
|
3
4
|
setup.py
|
|
4
5
|
./datalake/__init__.py
|
|
@@ -14,12 +15,14 @@ setup.py
|
|
|
14
15
|
./datalake/common/token_manager.py
|
|
15
16
|
./datalake/common/utils.py
|
|
16
17
|
./datalake/common/warn.py
|
|
18
|
+
./datalake/config/endpoints.json
|
|
17
19
|
./datalake/endpoints/__init__.py
|
|
18
20
|
./datalake/endpoints/advanced_search.py
|
|
19
21
|
./datalake/endpoints/bulk_search.py
|
|
20
22
|
./datalake/endpoints/comments.py
|
|
21
23
|
./datalake/endpoints/endpoint.py
|
|
22
24
|
./datalake/endpoints/filtered_threat_entity.py
|
|
25
|
+
./datalake/endpoints/my_account.py
|
|
23
26
|
./datalake/endpoints/sightings.py
|
|
24
27
|
./datalake/endpoints/sources.py
|
|
25
28
|
./datalake/endpoints/tags.py
|
|
@@ -42,6 +45,7 @@ setup.py
|
|
|
42
45
|
./datalake_scripts/scripts/edit_score.py
|
|
43
46
|
./datalake_scripts/scripts/get_atom_values.py
|
|
44
47
|
./datalake_scripts/scripts/get_filtered_threat_entity.py
|
|
48
|
+
./datalake_scripts/scripts/get_my_user_info.py
|
|
45
49
|
./datalake_scripts/scripts/get_query_hash.py
|
|
46
50
|
./datalake_scripts/scripts/get_threats.py
|
|
47
51
|
./datalake_scripts/scripts/get_threats_from_query_hash.py
|
|
@@ -59,6 +63,7 @@ setup.py
|
|
|
59
63
|
./tests/lib/test_auth_via_tokens.py
|
|
60
64
|
./tests/lib/test_bulk_search.py
|
|
61
65
|
./tests/lib/test_filtered_threat_entity.py
|
|
66
|
+
./tests/lib/test_my_account.py
|
|
62
67
|
./tests/lib/test_search_watch.py
|
|
63
68
|
./tests/lib/test_sightings.py
|
|
64
69
|
./tests/lib/test_tags.py
|
|
@@ -68,9 +73,80 @@ setup.py
|
|
|
68
73
|
./tests/scripts/test_base_scripts.py
|
|
69
74
|
./tests/scripts/test_cli.py
|
|
70
75
|
./tests/scripts/test_csv_builder.py
|
|
76
|
+
datalake/__init__.py
|
|
77
|
+
datalake/datalake.py
|
|
78
|
+
datalake/common/__init__.py
|
|
79
|
+
datalake/common/atom.py
|
|
80
|
+
datalake/common/atom_type.py
|
|
81
|
+
datalake/common/bulk_search_task.py
|
|
82
|
+
datalake/common/config.py
|
|
83
|
+
datalake/common/logger.py
|
|
84
|
+
datalake/common/output.py
|
|
85
|
+
datalake/common/throttler.py
|
|
86
|
+
datalake/common/token_manager.py
|
|
87
|
+
datalake/common/utils.py
|
|
88
|
+
datalake/common/warn.py
|
|
89
|
+
datalake/config/endpoints.json
|
|
90
|
+
datalake/endpoints/__init__.py
|
|
91
|
+
datalake/endpoints/advanced_search.py
|
|
92
|
+
datalake/endpoints/bulk_search.py
|
|
93
|
+
datalake/endpoints/comments.py
|
|
94
|
+
datalake/endpoints/endpoint.py
|
|
95
|
+
datalake/endpoints/filtered_threat_entity.py
|
|
96
|
+
datalake/endpoints/my_account.py
|
|
97
|
+
datalake/endpoints/sightings.py
|
|
98
|
+
datalake/endpoints/sources.py
|
|
99
|
+
datalake/endpoints/tags.py
|
|
100
|
+
datalake/endpoints/threats.py
|
|
101
|
+
datalake/miscellaneous/__init__.py
|
|
102
|
+
datalake/miscellaneous/search_watch.py
|
|
103
|
+
datalake_scripts/__init__.py
|
|
104
|
+
datalake_scripts/cli.py
|
|
71
105
|
datalake_scripts.egg-info/PKG-INFO
|
|
72
106
|
datalake_scripts.egg-info/SOURCES.txt
|
|
73
107
|
datalake_scripts.egg-info/dependency_links.txt
|
|
74
108
|
datalake_scripts.egg-info/entry_points.txt
|
|
75
109
|
datalake_scripts.egg-info/requires.txt
|
|
76
|
-
datalake_scripts.egg-info/top_level.txt
|
|
110
|
+
datalake_scripts.egg-info/top_level.txt
|
|
111
|
+
datalake_scripts/common/__init__.py
|
|
112
|
+
datalake_scripts/common/base_script.py
|
|
113
|
+
datalake_scripts/helper_scripts/__init__.py
|
|
114
|
+
datalake_scripts/helper_scripts/output_builder.py
|
|
115
|
+
datalake_scripts/helper_scripts/utils.py
|
|
116
|
+
datalake_scripts/scripts/__init__.py
|
|
117
|
+
datalake_scripts/scripts/add_comments.py
|
|
118
|
+
datalake_scripts/scripts/add_tags.py
|
|
119
|
+
datalake_scripts/scripts/add_threats.py
|
|
120
|
+
datalake_scripts/scripts/advanced_search.py
|
|
121
|
+
datalake_scripts/scripts/bulk_lookup_threats.py
|
|
122
|
+
datalake_scripts/scripts/edit_score.py
|
|
123
|
+
datalake_scripts/scripts/get_atom_values.py
|
|
124
|
+
datalake_scripts/scripts/get_filtered_threat_entity.py
|
|
125
|
+
datalake_scripts/scripts/get_my_user_info.py
|
|
126
|
+
datalake_scripts/scripts/get_query_hash.py
|
|
127
|
+
datalake_scripts/scripts/get_threats.py
|
|
128
|
+
datalake_scripts/scripts/get_threats_from_query_hash.py
|
|
129
|
+
datalake_scripts/scripts/lookup_threats.py
|
|
130
|
+
datalake_scripts/scripts/search_watch.py
|
|
131
|
+
tests/__init__.py
|
|
132
|
+
tests/conftest.py
|
|
133
|
+
tests/common/__init__.py
|
|
134
|
+
tests/common/fixture.py
|
|
135
|
+
tests/common/test_output.py
|
|
136
|
+
tests/common/test_utils.py
|
|
137
|
+
tests/lib/__init__.py
|
|
138
|
+
tests/lib/test_advanced_search.py
|
|
139
|
+
tests/lib/test_atom_type.py
|
|
140
|
+
tests/lib/test_auth_via_tokens.py
|
|
141
|
+
tests/lib/test_bulk_search.py
|
|
142
|
+
tests/lib/test_filtered_threat_entity.py
|
|
143
|
+
tests/lib/test_my_account.py
|
|
144
|
+
tests/lib/test_search_watch.py
|
|
145
|
+
tests/lib/test_sightings.py
|
|
146
|
+
tests/lib/test_tags.py
|
|
147
|
+
tests/lib/test_threats.py
|
|
148
|
+
tests/scripts/__init__.py
|
|
149
|
+
tests/scripts/test_add_new_threats.py
|
|
150
|
+
tests/scripts/test_base_scripts.py
|
|
151
|
+
tests/scripts/test_cli.py
|
|
152
|
+
tests/scripts/test_csv_builder.py
|
|
@@ -2,20 +2,31 @@ import os
|
|
|
2
2
|
|
|
3
3
|
from setuptools import find_packages, setup
|
|
4
4
|
|
|
5
|
-
from datalake_scripts.cli import Cli
|
|
6
|
-
|
|
7
5
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
8
6
|
|
|
9
7
|
with open("README.md", "r") as fh:
|
|
10
|
-
|
|
8
|
+
LONG_DESCRIPTION = fh.read()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def read_version_or_cli_name(option: str):
|
|
12
|
+
with open("datalake_scripts/cli.py") as f:
|
|
13
|
+
for line in f:
|
|
14
|
+
if option == "version" and line.strip().startswith("VERSION"):
|
|
15
|
+
return line.split("=")[1].strip().strip('"')
|
|
16
|
+
if option == "cli_name" and line.strip().startswith("CLI_NAME"):
|
|
17
|
+
return line.split("=")[1].strip().strip('"')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
CLI_NAME = read_version_or_cli_name("cli_name")
|
|
21
|
+
VERSION = read_version_or_cli_name("version")
|
|
11
22
|
|
|
12
23
|
setup(
|
|
13
24
|
name="datalake_scripts",
|
|
14
|
-
version=
|
|
25
|
+
version=VERSION,
|
|
15
26
|
author="OCD",
|
|
16
27
|
author_email="cert-contact.ocd@orange.com",
|
|
17
28
|
description="A collection of scripts to easily use the API of OCD Datalake",
|
|
18
|
-
long_description=
|
|
29
|
+
long_description=LONG_DESCRIPTION,
|
|
19
30
|
long_description_content_type="text/markdown",
|
|
20
31
|
url="https://github.com/cert-orangecyberdefense/datalake/",
|
|
21
32
|
package_dir={
|
|
@@ -25,7 +36,7 @@ setup(
|
|
|
25
36
|
install_requires=["requests", "halo", "prettytable"],
|
|
26
37
|
test_suite="nose.collector",
|
|
27
38
|
entry_points={
|
|
28
|
-
"console_scripts": (f"{
|
|
39
|
+
"console_scripts": (f"{CLI_NAME} = datalake_scripts.cli:main",),
|
|
29
40
|
},
|
|
30
41
|
classifiers=[
|
|
31
42
|
"Programming Language :: Python :: 3.7",
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import responses
|
|
3
|
+
from datalake import Datalake
|
|
4
|
+
from tests.common.fixture import TestData, datalake
|
|
5
|
+
|
|
6
|
+
mock_results_data = {
|
|
7
|
+
"email": "user@example.com",
|
|
8
|
+
"first_name": "string",
|
|
9
|
+
"force_password_change": True,
|
|
10
|
+
"full_name": "string",
|
|
11
|
+
"has_password": True,
|
|
12
|
+
"id": 0,
|
|
13
|
+
"is_active": True,
|
|
14
|
+
"is_deleted": False,
|
|
15
|
+
"last_login": "2025-10-15T08:43:33.390Z",
|
|
16
|
+
"last_name": "string",
|
|
17
|
+
"organization": {"id": 0, "name": "string", "path_names": ["string"]},
|
|
18
|
+
"previous_login": "2025-10-15T08:43:33.390Z",
|
|
19
|
+
"request_limit": 0,
|
|
20
|
+
"role": {
|
|
21
|
+
"administration_permissions": {"description": "string", "name": "string"},
|
|
22
|
+
"id": 0,
|
|
23
|
+
"name": "string",
|
|
24
|
+
},
|
|
25
|
+
"totp_enabled": True,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def mock_api_response():
|
|
30
|
+
responses.add(
|
|
31
|
+
responses.GET,
|
|
32
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
33
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
34
|
+
+ TestData.TEST_CONFIG["endpoints"]["users-me"],
|
|
35
|
+
json=mock_results_data, # Only return the "results" part
|
|
36
|
+
status=200,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@responses.activate
|
|
41
|
+
def test_my_account(datalake: Datalake):
|
|
42
|
+
mock_api_response()
|
|
43
|
+
response = datalake.MyAccount.me()
|
|
44
|
+
assert response == mock_results_data
|
|
45
|
+
assert isinstance(response, dict), "Results should be a dict"
|
|
46
|
+
assert responses.calls[0].request.url, (
|
|
47
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
48
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
49
|
+
+ TestData.TEST_CONFIG["endpoints"]["users-me"]
|
|
50
|
+
)
|
|
51
|
+
assert responses.calls[0].request.method, "GET"
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
from unittest.mock import patch
|
|
2
2
|
import pytest
|
|
3
3
|
import responses
|
|
4
|
-
from datetime import datetime
|
|
4
|
+
from datetime import datetime, timedelta, timezone
|
|
5
5
|
|
|
6
6
|
from responses import matchers
|
|
7
7
|
import os
|
|
8
|
+
import ast
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
from tests.common.fixture import TestData, datalake # noqa needed fixture import
|
|
11
12
|
from datalake import (
|
|
12
13
|
IpAtom,
|
|
13
14
|
FileAtom,
|
|
15
|
+
EmailAtom,
|
|
16
|
+
UrlAtom,
|
|
14
17
|
Hashes,
|
|
15
18
|
Jarm,
|
|
16
19
|
IpService,
|
|
@@ -448,6 +451,242 @@ def test_submit_sightings_bad_atom(datalake):
|
|
|
448
451
|
assert str(err.value) == '"atoms" needs to be a list of Atom subclasses.'
|
|
449
452
|
|
|
450
453
|
|
|
454
|
+
@responses.activate
|
|
455
|
+
def test_bulk_submit_sightings_success(datalake):
|
|
456
|
+
url = (
|
|
457
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
458
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
459
|
+
+ TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
f1 = FileAtom(
|
|
463
|
+
hashes=Hashes(
|
|
464
|
+
md5="d41d8cd98f00b204e9800998ecf8427e",
|
|
465
|
+
sha1="da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
466
|
+
sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
467
|
+
)
|
|
468
|
+
)
|
|
469
|
+
ip1 = IpAtom("52.48.79.33")
|
|
470
|
+
em1 = EmailAtom("hacker@hacker.fr")
|
|
471
|
+
url1 = UrlAtom("http://notfishing.com")
|
|
472
|
+
|
|
473
|
+
# You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
|
|
474
|
+
|
|
475
|
+
## Prepare threat types for this sighting
|
|
476
|
+
threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
|
|
477
|
+
|
|
478
|
+
## Prepare start and end timestamps for this sighting
|
|
479
|
+
start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
|
|
480
|
+
end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
|
|
481
|
+
input_sighting = {
|
|
482
|
+
"atoms": [ip1, f1, em1, url1],
|
|
483
|
+
"start_timestamp": start,
|
|
484
|
+
"end_timestamp": end,
|
|
485
|
+
"sighting_type": SightingType.POSITIVE,
|
|
486
|
+
"description_visibility": Visibility.PUBLIC,
|
|
487
|
+
"count": 1,
|
|
488
|
+
"threat_types": threat_types,
|
|
489
|
+
"tags": ["some_tag"],
|
|
490
|
+
"description": "some_description",
|
|
491
|
+
"editable": True,
|
|
492
|
+
}
|
|
493
|
+
expected_request = {
|
|
494
|
+
"data": [
|
|
495
|
+
{
|
|
496
|
+
"count": 1,
|
|
497
|
+
"description": "some_description",
|
|
498
|
+
"description_visibility": "PUBLIC",
|
|
499
|
+
"editable": True,
|
|
500
|
+
"email_list": [{"email": "hacker@hacker.fr"}],
|
|
501
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
502
|
+
"file_list": [
|
|
503
|
+
{
|
|
504
|
+
"hashes": {
|
|
505
|
+
"md5": "d41d8cd98f00b204e9800998ecf8427e",
|
|
506
|
+
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
507
|
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
],
|
|
511
|
+
"ip_list": [{"ip_address": "52.48.79.33"}],
|
|
512
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
513
|
+
"tags": ["some_tag"],
|
|
514
|
+
"threat_types": ["phishing", "scam"],
|
|
515
|
+
"type": "positive",
|
|
516
|
+
"url_list": [{"url": "http://notfishing.com"}],
|
|
517
|
+
}
|
|
518
|
+
]
|
|
519
|
+
}
|
|
520
|
+
expected_res = {
|
|
521
|
+
"count": 1,
|
|
522
|
+
"results": [
|
|
523
|
+
{
|
|
524
|
+
"count": 1,
|
|
525
|
+
"description": "some_description",
|
|
526
|
+
"description_visibility": "PUBLIC",
|
|
527
|
+
"editable": True,
|
|
528
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
529
|
+
"relation_type": "sighting",
|
|
530
|
+
"reliability": 100,
|
|
531
|
+
"sighting_version": "3.0.0",
|
|
532
|
+
"sightings": {
|
|
533
|
+
"email_list": [{"email": "hacker@hacker.fr"}],
|
|
534
|
+
"file_list": [
|
|
535
|
+
{
|
|
536
|
+
"hashes": {
|
|
537
|
+
"md5": "d41d8cd98f00b204e9800998ecf8427e",
|
|
538
|
+
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
539
|
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
540
|
+
"sha512": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
],
|
|
544
|
+
"ip_list": [{"ip_address": "52.48.79.33"}],
|
|
545
|
+
"url_list": [{"url": "http://notfishing.com"}],
|
|
546
|
+
},
|
|
547
|
+
"source_context": {
|
|
548
|
+
"source_id": "org: Orange Cyberdefense",
|
|
549
|
+
"source_uses": ["sightings"],
|
|
550
|
+
},
|
|
551
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
552
|
+
"tags": [{"categories": [], "name": "some_tag"}],
|
|
553
|
+
"threat_types": ["phishing", "scam"],
|
|
554
|
+
"timestamp_created": "2025-12-16T10:00:00Z",
|
|
555
|
+
"type": "positive",
|
|
556
|
+
"uid": "e14dd426-ed41-436c-9e0e-0d785e750428",
|
|
557
|
+
}
|
|
558
|
+
],
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
responses.post(
|
|
562
|
+
url=url,
|
|
563
|
+
json=expected_res,
|
|
564
|
+
status=200,
|
|
565
|
+
match=[matchers.json_params_matcher(expected_request)],
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
with patch.dict(
|
|
569
|
+
os.environ,
|
|
570
|
+
{
|
|
571
|
+
"IGNORE_SIGHTING_BUILDER_WARNING": "1",
|
|
572
|
+
},
|
|
573
|
+
clear=True,
|
|
574
|
+
):
|
|
575
|
+
res = datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
|
|
576
|
+
|
|
577
|
+
assert res == expected_res
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def test_bulk_submit_sightings_payload_error(datalake):
|
|
581
|
+
with pytest.raises(ValueError) as err:
|
|
582
|
+
datalake.Sightings.bulk_submit_sightings(
|
|
583
|
+
sightings=[
|
|
584
|
+
{
|
|
585
|
+
"atoms": [FileAtom(Hashes(md5="d41d8cd98f00b204e9800998ecf8427e"))],
|
|
586
|
+
"start_timestamp": start,
|
|
587
|
+
"end_timestamp": end,
|
|
588
|
+
"sighting_type": SightingType.NEUTRAL,
|
|
589
|
+
"description_visibility": Visibility.PUBLIC,
|
|
590
|
+
"count": 3,
|
|
591
|
+
"threat_types": threat_types,
|
|
592
|
+
"tags": ["some_tag_bis"],
|
|
593
|
+
"description": "some_description_bis",
|
|
594
|
+
"editable": False,
|
|
595
|
+
}
|
|
596
|
+
]
|
|
597
|
+
)
|
|
598
|
+
assert (
|
|
599
|
+
str(err.value) == """For NEUTRAL sightings, "threat_types" can't be passed."""
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
@responses.activate
|
|
604
|
+
def test_bulk_submit_sightings_failure(datalake):
|
|
605
|
+
url = (
|
|
606
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
607
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
608
|
+
+ TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
hashkeys = [
|
|
612
|
+
"d41d8cd98f00b204e9800998ecf8427e",
|
|
613
|
+
"a7b25b324871a7695aa2cc5d09681dda",
|
|
614
|
+
"1ed07771327e850255b09b042ad00e3d",
|
|
615
|
+
"bf1f33c3a56e1dfda6a2f4f3d3e4361a",
|
|
616
|
+
]
|
|
617
|
+
|
|
618
|
+
# You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
|
|
619
|
+
|
|
620
|
+
## Prepare threat types for this sighting
|
|
621
|
+
threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
|
|
622
|
+
|
|
623
|
+
## Prepare start and end timestamps for this sighting
|
|
624
|
+
start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
|
|
625
|
+
end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
|
|
626
|
+
input_sighting = {
|
|
627
|
+
"hashkeys": hashkeys,
|
|
628
|
+
"start_timestamp": start,
|
|
629
|
+
"end_timestamp": end,
|
|
630
|
+
"sighting_type": SightingType.POSITIVE,
|
|
631
|
+
"description_visibility": Visibility.PUBLIC,
|
|
632
|
+
"count": 1,
|
|
633
|
+
"threat_types": threat_types,
|
|
634
|
+
"tags": ["some_tag"],
|
|
635
|
+
"description": "some_description",
|
|
636
|
+
"editable": True,
|
|
637
|
+
}
|
|
638
|
+
expected_request = {
|
|
639
|
+
"data": [
|
|
640
|
+
{
|
|
641
|
+
"count": 1,
|
|
642
|
+
"description": "some_description",
|
|
643
|
+
"description_visibility": "PUBLIC",
|
|
644
|
+
"editable": True,
|
|
645
|
+
"end_timestamp": "2025-12-16T10:00:00Z",
|
|
646
|
+
"hashkeys": hashkeys,
|
|
647
|
+
"start_timestamp": "2025-12-16T09:00:00Z",
|
|
648
|
+
"tags": ["some_tag"],
|
|
649
|
+
"threat_types": ["phishing", "scam"],
|
|
650
|
+
"type": "positive",
|
|
651
|
+
}
|
|
652
|
+
]
|
|
653
|
+
}
|
|
654
|
+
expected_res = {
|
|
655
|
+
"data": {
|
|
656
|
+
"0": {
|
|
657
|
+
"hashkeys": [
|
|
658
|
+
"Some of the provided hashkeys has not been found: "
|
|
659
|
+
"['d41d8cd98f00b204e9800998ecf8427e']"
|
|
660
|
+
]
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
responses.post(
|
|
666
|
+
url=url,
|
|
667
|
+
json=expected_res,
|
|
668
|
+
status=422,
|
|
669
|
+
match=[matchers.json_params_matcher(expected_request)],
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
with patch.dict(
|
|
673
|
+
os.environ,
|
|
674
|
+
{
|
|
675
|
+
"IGNORE_SIGHTING_BUILDER_WARNING": "1",
|
|
676
|
+
},
|
|
677
|
+
clear=True,
|
|
678
|
+
):
|
|
679
|
+
with pytest.raises(ValueError) as exc:
|
|
680
|
+
datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
|
|
681
|
+
|
|
682
|
+
prefix = "422 HTTP code: "
|
|
683
|
+
msg = str(exc.value)
|
|
684
|
+
assert msg.startswith(prefix)
|
|
685
|
+
|
|
686
|
+
payload = ast.literal_eval(msg[len(prefix) :])
|
|
687
|
+
assert payload == expected_res
|
|
688
|
+
|
|
689
|
+
|
|
451
690
|
def test_sightings_filtered_bad_ordering(datalake):
|
|
452
691
|
with pytest.raises(ValueError) as err:
|
|
453
692
|
datalake.Sightings.sightings_filtered(ordering="bad_ordering")
|
|
@@ -16,6 +16,7 @@ from datalake_scripts.scripts import (
|
|
|
16
16
|
get_atom_values,
|
|
17
17
|
get_filtered_threat_entity,
|
|
18
18
|
search_watch,
|
|
19
|
+
get_my_user_info,
|
|
19
20
|
)
|
|
20
21
|
|
|
21
22
|
|
|
@@ -36,6 +37,7 @@ from datalake_scripts.scripts import (
|
|
|
36
37
|
get_atom_values,
|
|
37
38
|
get_filtered_threat_entity,
|
|
38
39
|
search_watch,
|
|
40
|
+
get_my_user_info,
|
|
39
41
|
],
|
|
40
42
|
)
|
|
41
43
|
def test_name_function(function_script):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/advanced_search.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/search_watch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/base_script.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/utils.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/__init__.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_comments.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_tags.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/edit_score.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/search_watch.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/requires.txt
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_add_new_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|