datalake-scripts 2.7.3__tar.gz → 2.7.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/PKG-INFO +25 -2
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/README.md +24 -1
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/token_manager.py +13 -2
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/datalake.py +20 -6
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/endpoint.py +25 -3
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/cli.py +1 -1
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/base_engine.py +15 -3
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/mixins.py +1 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/PKG-INFO +25 -2
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/api_objects/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/api_objects/bulk_search_task.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/atom.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/atom_type.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/config.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/logger.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/output.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/throttler.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/utils.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/warn.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/advanced_search.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/bulk_search.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/filtered_tag_subcategory.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/sightings.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/tags.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/miscellaneous/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/base_script.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/get_engine.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/post_engine.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/SOURCES.txt +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/entry_points.txt +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/setup.cfg +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/setup.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/fixture.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/test_output.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/test_utils.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/conftest.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_advanced_search.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_atom_type.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_bulk_search.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_filtered_tag_subcategory.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_search_watch.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_sightings.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_tags.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_base_engine.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_cli.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_csv_builder.py +0 -0
- {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_endpoint_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.4
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
|
|
|
40
40
|
$ pip3 install datalake-scripts
|
|
41
41
|
```
|
|
42
42
|
### Using as a library
|
|
43
|
-
|
|
43
|
+
```python
|
|
44
|
+
from datalake import Datalake, AtomType, Output
|
|
45
|
+
|
|
46
|
+
dtl = Datalake(username='username', password='password')
|
|
47
|
+
dtl.Threats.lookup(
|
|
48
|
+
atom_value='mayoclinic.org',
|
|
49
|
+
atom_type=AtomType.DOMAIN,
|
|
50
|
+
hashkey_only=False,
|
|
51
|
+
output=Output.JSON
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
44
56
|
|
|
45
57
|
### Using as a CLI
|
|
46
58
|
|
|
@@ -72,6 +84,17 @@ or
|
|
|
72
84
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
73
85
|
> These last two are independent and one can be used without the other if you wish.
|
|
74
86
|
|
|
87
|
+
#### Using a Proxy
|
|
88
|
+
|
|
89
|
+
You can set up following environment variables :
|
|
90
|
+
|
|
91
|
+
* `HTTP_PROXY`
|
|
92
|
+
* `HTTPS_PROXY`
|
|
93
|
+
|
|
94
|
+
We use the format accepted by the requests python library.
|
|
95
|
+
See its documenation for other possible kinds of proxy to set up.
|
|
96
|
+
|
|
97
|
+
|
|
75
98
|
#### Throttling
|
|
76
99
|
For throttling the request, those two environment variable can be used:
|
|
77
100
|
* `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
@@ -20,7 +20,19 @@ $ pip install datalake-scripts
|
|
|
20
20
|
$ pip3 install datalake-scripts
|
|
21
21
|
```
|
|
22
22
|
### Using as a library
|
|
23
|
-
|
|
23
|
+
```python
|
|
24
|
+
from datalake import Datalake, AtomType, Output
|
|
25
|
+
|
|
26
|
+
dtl = Datalake(username='username', password='password')
|
|
27
|
+
dtl.Threats.lookup(
|
|
28
|
+
atom_value='mayoclinic.org',
|
|
29
|
+
atom_type=AtomType.DOMAIN,
|
|
30
|
+
hashkey_only=False,
|
|
31
|
+
output=Output.JSON
|
|
32
|
+
)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
24
36
|
|
|
25
37
|
### Using as a CLI
|
|
26
38
|
|
|
@@ -52,6 +64,17 @@ or
|
|
|
52
64
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
53
65
|
> These last two are independent and one can be used without the other if you wish.
|
|
54
66
|
|
|
67
|
+
#### Using a Proxy
|
|
68
|
+
|
|
69
|
+
You can set up following environment variables :
|
|
70
|
+
|
|
71
|
+
* `HTTP_PROXY`
|
|
72
|
+
* `HTTPS_PROXY`
|
|
73
|
+
|
|
74
|
+
We use the format accepted by the requests python library.
|
|
75
|
+
See its documenation for other possible kinds of proxy to set up.
|
|
76
|
+
|
|
77
|
+
|
|
55
78
|
#### Throttling
|
|
56
79
|
For throttling the request, those two environment variable can be used:
|
|
57
80
|
* `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
@@ -25,6 +25,8 @@ class TokenManager:
|
|
|
25
25
|
username=None,
|
|
26
26
|
password=None,
|
|
27
27
|
longterm_token=None,
|
|
28
|
+
proxies: dict = None,
|
|
29
|
+
verify: bool = True,
|
|
28
30
|
):
|
|
29
31
|
"""environment can be either prod or preprod"""
|
|
30
32
|
base_url = urljoin(
|
|
@@ -40,6 +42,8 @@ class TokenManager:
|
|
|
40
42
|
self.username = username
|
|
41
43
|
self.password = password
|
|
42
44
|
self.longterm_token = longterm_token
|
|
45
|
+
self.proxies = proxies
|
|
46
|
+
self.verify = verify
|
|
43
47
|
self.access_token = None
|
|
44
48
|
self.refresh_token = None
|
|
45
49
|
self.get_token()
|
|
@@ -81,7 +85,9 @@ class TokenManager:
|
|
|
81
85
|
else:
|
|
82
86
|
data = {"email": self.username, "password": self.password}
|
|
83
87
|
|
|
84
|
-
response = requests.post(
|
|
88
|
+
response = requests.post(
|
|
89
|
+
url=self.url_token, json=data, proxies=self.proxies, verify=self.verify
|
|
90
|
+
)
|
|
85
91
|
json_response = json.loads(response.text)
|
|
86
92
|
try:
|
|
87
93
|
self.access_token = f'Token {json_response["access_token"]}'
|
|
@@ -96,7 +102,12 @@ class TokenManager:
|
|
|
96
102
|
def fetch_new_token(self):
|
|
97
103
|
logger.debug("Token will be refreshed")
|
|
98
104
|
headers = {"Authorization": self.refresh_token}
|
|
99
|
-
response = requests.post(
|
|
105
|
+
response = requests.post(
|
|
106
|
+
url=self.url_refresh,
|
|
107
|
+
headers=headers,
|
|
108
|
+
proxies=self.proxies,
|
|
109
|
+
verify=self.verify,
|
|
110
|
+
)
|
|
100
111
|
|
|
101
112
|
json_response = response.json()
|
|
102
113
|
if (
|
|
@@ -33,6 +33,8 @@ class Datalake:
|
|
|
33
33
|
longterm_token: str = None,
|
|
34
34
|
env="prod",
|
|
35
35
|
log_level=logging.WARNING,
|
|
36
|
+
proxies: dict = None,
|
|
37
|
+
verify: bool = True,
|
|
36
38
|
):
|
|
37
39
|
configure_logging(log_level)
|
|
38
40
|
endpoint_config = Config().load_config()
|
|
@@ -43,6 +45,8 @@ class Datalake:
|
|
|
43
45
|
username=username,
|
|
44
46
|
password=password,
|
|
45
47
|
longterm_token=longterm_token,
|
|
48
|
+
proxies=proxies,
|
|
49
|
+
verify=verify,
|
|
46
50
|
)
|
|
47
51
|
except Exception as e:
|
|
48
52
|
if "Failed to resolve" in str(e) or "Failed to establish" in str(e):
|
|
@@ -52,14 +56,24 @@ class Datalake:
|
|
|
52
56
|
else:
|
|
53
57
|
raise
|
|
54
58
|
|
|
55
|
-
self.Threats = Threats(
|
|
56
|
-
|
|
57
|
-
|
|
59
|
+
self.Threats = Threats(
|
|
60
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
61
|
+
)
|
|
62
|
+
self.BulkSearch = BulkSearch(
|
|
63
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
64
|
+
)
|
|
65
|
+
self.Tags = Tags(
|
|
66
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
67
|
+
)
|
|
58
68
|
self.FilteredTagSubcategory = FilteredTagSubcategory(
|
|
59
|
-
endpoint_config, env, token_manager
|
|
69
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
70
|
+
)
|
|
71
|
+
self.AdvancedSearch = AdvancedSearch(
|
|
72
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
73
|
+
)
|
|
74
|
+
self.Sightings = Sightings(
|
|
75
|
+
endpoint_config, env, token_manager, proxies=proxies, verify=verify
|
|
60
76
|
)
|
|
61
|
-
self.AdvancedSearch = AdvancedSearch(endpoint_config, env, token_manager)
|
|
62
|
-
self.Sightings = Sightings(endpoint_config, env, token_manager)
|
|
63
77
|
self.SearchWatch = SearchWatch(self.BulkSearch)
|
|
64
78
|
|
|
65
79
|
def sightings_filtered_from_atom_value(
|
|
@@ -28,13 +28,20 @@ class Endpoint:
|
|
|
28
28
|
SET_MAX_RETRY = 3
|
|
29
29
|
|
|
30
30
|
def __init__(
|
|
31
|
-
self,
|
|
31
|
+
self,
|
|
32
|
+
endpoint_config: dict,
|
|
33
|
+
environment: str,
|
|
34
|
+
token_manager: TokenManager,
|
|
35
|
+
proxies: dict = None,
|
|
36
|
+
verify: bool = True,
|
|
32
37
|
):
|
|
33
38
|
self.endpoint_config = endpoint_config
|
|
34
39
|
self.environment = environment
|
|
35
40
|
self.terminal_size = self._get_terminal_size()
|
|
36
41
|
self.token_manager = token_manager
|
|
37
42
|
self.SET_MAX_RETRY = 3
|
|
43
|
+
self.proxies = proxies
|
|
44
|
+
self.verify = verify
|
|
38
45
|
|
|
39
46
|
@staticmethod
|
|
40
47
|
def _get_terminal_size() -> int:
|
|
@@ -71,7 +78,13 @@ class Endpoint:
|
|
|
71
78
|
logger.debug(self._pretty_debug_request(url, method, post_body, headers))
|
|
72
79
|
|
|
73
80
|
response = self._send_request(
|
|
74
|
-
url,
|
|
81
|
+
url,
|
|
82
|
+
method,
|
|
83
|
+
headers,
|
|
84
|
+
post_body,
|
|
85
|
+
stream=stream,
|
|
86
|
+
proxies=self.proxies,
|
|
87
|
+
verify=self.verify,
|
|
75
88
|
)
|
|
76
89
|
|
|
77
90
|
if logger.isEnabledFor(logging.DEBUG):
|
|
@@ -111,7 +124,13 @@ class Endpoint:
|
|
|
111
124
|
|
|
112
125
|
@staticmethod
|
|
113
126
|
def _send_request(
|
|
114
|
-
url: str,
|
|
127
|
+
url: str,
|
|
128
|
+
method: str,
|
|
129
|
+
headers: dict,
|
|
130
|
+
data: dict,
|
|
131
|
+
stream=False,
|
|
132
|
+
proxies: dict = None,
|
|
133
|
+
verify: bool = True,
|
|
115
134
|
) -> Response:
|
|
116
135
|
"""
|
|
117
136
|
Send the correct http request to url from method [get, post, delete, patch, put].
|
|
@@ -121,7 +140,10 @@ class Endpoint:
|
|
|
121
140
|
"url": url,
|
|
122
141
|
"headers": headers,
|
|
123
142
|
"stream": stream,
|
|
143
|
+
"verify": verify,
|
|
124
144
|
}
|
|
145
|
+
if proxies:
|
|
146
|
+
common_kwargs["proxies"] = proxies
|
|
125
147
|
|
|
126
148
|
if method == "get":
|
|
127
149
|
api_response = requests.get(**common_kwargs)
|
|
@@ -33,14 +33,26 @@ class BaseEngine:
|
|
|
33
33
|
Json = Union[dict, list] # json like object that can be a dict or root level array
|
|
34
34
|
|
|
35
35
|
def __init__(
|
|
36
|
-
self,
|
|
36
|
+
self,
|
|
37
|
+
endpoint_config: dict,
|
|
38
|
+
environment: str,
|
|
39
|
+
token_manager: TokenManager,
|
|
40
|
+
proxies: dict = None,
|
|
41
|
+
verify: bool = True,
|
|
37
42
|
):
|
|
38
43
|
self.endpoint_config = endpoint_config
|
|
39
44
|
self.environment = environment
|
|
40
|
-
self.requests_ssl_verify =
|
|
45
|
+
self.requests_ssl_verify = (
|
|
46
|
+
verify
|
|
47
|
+
if suppress_insecure_request_warns(environment)
|
|
48
|
+
else suppress_insecure_request_warns(environment)
|
|
49
|
+
)
|
|
41
50
|
self.url = self._build_url(endpoint_config, environment)
|
|
42
51
|
self.token_manager = token_manager
|
|
43
|
-
self.
|
|
52
|
+
self.proxies = proxies
|
|
53
|
+
self.endpoint = Endpoint(
|
|
54
|
+
endpoint_config, environment, token_manager, proxies, verify
|
|
55
|
+
)
|
|
44
56
|
|
|
45
57
|
def datalake_requests(
|
|
46
58
|
self, url: str, method: str, headers: dict, post_body: dict = None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.4
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
|
|
|
40
40
|
$ pip3 install datalake-scripts
|
|
41
41
|
```
|
|
42
42
|
### Using as a library
|
|
43
|
-
|
|
43
|
+
```python
|
|
44
|
+
from datalake import Datalake, AtomType, Output
|
|
45
|
+
|
|
46
|
+
dtl = Datalake(username='username', password='password')
|
|
47
|
+
dtl.Threats.lookup(
|
|
48
|
+
atom_value='mayoclinic.org',
|
|
49
|
+
atom_type=AtomType.DOMAIN,
|
|
50
|
+
hashkey_only=False,
|
|
51
|
+
output=Output.JSON
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
44
56
|
|
|
45
57
|
### Using as a CLI
|
|
46
58
|
|
|
@@ -72,6 +84,17 @@ or
|
|
|
72
84
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
73
85
|
> These last two are independent and one can be used without the other if you wish.
|
|
74
86
|
|
|
87
|
+
#### Using a Proxy
|
|
88
|
+
|
|
89
|
+
You can set up following environment variables :
|
|
90
|
+
|
|
91
|
+
* `HTTP_PROXY`
|
|
92
|
+
* `HTTPS_PROXY`
|
|
93
|
+
|
|
94
|
+
We use the format accepted by the requests python library.
|
|
95
|
+
See its documenation for other possible kinds of proxy to set up.
|
|
96
|
+
|
|
97
|
+
|
|
75
98
|
#### Throttling
|
|
76
99
|
For throttling the request, those two environment variable can be used:
|
|
77
100
|
* `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/filtered_tag_subcategory.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/output_builder.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/bulk_lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_by_hashkey.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_filtered_tag_subcategory.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|