datalake-scripts 2.7.3__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/PKG-INFO +69 -25
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/README.md +54 -19
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/__init__.py +4 -9
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/atom.py +2 -9
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/atom_type.py +13 -165
- {datalake-scripts-2.7.3/datalake/api_objects → datalake_scripts-3.0.0/datalake/common}/bulk_search_task.py +6 -2
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/logger.py +4 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/token_manager.py +24 -11
- datalake_scripts-3.0.0/datalake/config/endpoints.json +35 -0
- datalake_scripts-3.0.0/datalake/datalake.py +139 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/endpoints/advanced_search.py +7 -4
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/endpoints/bulk_search.py +12 -3
- datalake_scripts-3.0.0/datalake/endpoints/comments.py +48 -0
- datalake_scripts-3.0.0/datalake/endpoints/endpoint.py +213 -0
- datalake-scripts-2.7.3/datalake/endpoints/filtered_tag_subcategory.py → datalake_scripts-3.0.0/datalake/endpoints/filtered_threat_entity.py +23 -20
- datalake_scripts-3.0.0/datalake/endpoints/my_account.py +17 -0
- datalake_scripts-3.0.0/datalake/endpoints/sightings.py +339 -0
- datalake_scripts-3.0.0/datalake/endpoints/sources.py +33 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/endpoints/tags.py +1 -1
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/endpoints/threats.py +114 -42
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/miscellaneous/search_watch.py +21 -17
- datalake_scripts-3.0.0/datalake_scripts/__init__.py +1 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/cli.py +16 -10
- datalake_scripts-3.0.0/datalake_scripts/common/base_script.py +79 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/helper_scripts/utils.py +8 -2
- datalake_scripts-3.0.0/datalake_scripts/scripts/add_comments.py +73 -0
- datalake_scripts-3.0.0/datalake_scripts/scripts/add_tags.py +80 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/add_threats.py +134 -149
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/advanced_search.py +13 -9
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/bulk_lookup_threats.py +102 -97
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/edit_score.py +28 -47
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_atom_values.py +24 -35
- datalake_scripts-3.0.0/datalake_scripts/scripts/get_filtered_threat_entity.py +67 -0
- datalake_scripts-3.0.0/datalake_scripts/scripts/get_my_user_info.py +32 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_query_hash.py +8 -8
- datalake_scripts-3.0.0/datalake_scripts/scripts/get_threats.py +58 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_threats_from_query_hash.py +21 -26
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/lookup_threats.py +22 -22
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/scripts/search_watch.py +13 -14
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/PKG-INFO +69 -25
- datalake_scripts-3.0.0/datalake_scripts.egg-info/SOURCES.txt +152 -0
- datalake_scripts-3.0.0/datalake_scripts.egg-info/entry_points.txt +2 -0
- datalake_scripts-3.0.0/pyproject.toml +3 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/setup.cfg +3 -5
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/setup.py +18 -15
- datalake_scripts-3.0.0/tests/common/fixture.py +96 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/conftest.py +3 -3
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_advanced_search.py +24 -11
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_auth_via_tokens.py +72 -33
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_bulk_search.py +96 -22
- datalake-scripts-2.7.3/tests/lib/test_filtered_tag_subcategory.py → datalake_scripts-3.0.0/tests/lib/test_filtered_threat_entity.py +23 -17
- datalake_scripts-3.0.0/tests/lib/test_my_account.py +51 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_search_watch.py +7 -3
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_sightings.py +291 -31
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_tags.py +37 -29
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_threats.py +73 -39
- datalake-scripts-2.7.3/tests/scripts/test_base_engine.py → datalake_scripts-3.0.0/tests/scripts/test_base_scripts.py +4 -4
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/scripts/test_cli.py +6 -7
- datalake-scripts-2.7.3/datalake/datalake.py +0 -113
- datalake-scripts-2.7.3/datalake/endpoints/endpoint.py +0 -164
- datalake-scripts-2.7.3/datalake/endpoints/sightings.py +0 -210
- datalake-scripts-2.7.3/datalake_scripts/__init__.py +0 -5
- datalake-scripts-2.7.3/datalake_scripts/common/base_engine.py +0 -111
- datalake-scripts-2.7.3/datalake_scripts/common/base_script.py +0 -83
- datalake-scripts-2.7.3/datalake_scripts/common/mixins.py +0 -85
- datalake-scripts-2.7.3/datalake_scripts/engines/get_engine.py +0 -104
- datalake-scripts-2.7.3/datalake_scripts/engines/post_engine.py +0 -199
- datalake-scripts-2.7.3/datalake_scripts/scripts/add_comments.py +0 -77
- datalake-scripts-2.7.3/datalake_scripts/scripts/add_tags.py +0 -84
- datalake-scripts-2.7.3/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -65
- datalake-scripts-2.7.3/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -60
- datalake-scripts-2.7.3/datalake_scripts.egg-info/SOURCES.txt +0 -81
- datalake-scripts-2.7.3/datalake_scripts.egg-info/entry_points.txt +0 -9
- datalake-scripts-2.7.3/tests/common/fixture.py +0 -57
- datalake-scripts-2.7.3/tests/lib/__init__.py +0 -0
- datalake-scripts-2.7.3/tests/scripts/__init__.py +0 -0
- datalake-scripts-2.7.3/tests/scripts/test_endpoint_config.py +0 -26
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/config.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/output.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/throttler.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/utils.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/common/warn.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake/endpoints/__init__.py +0 -0
- {datalake-scripts-2.7.3/datalake/api_objects → datalake_scripts-3.0.0/datalake/miscellaneous}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/common/__init__.py +0 -0
- {datalake-scripts-2.7.3/datalake/miscellaneous → datalake_scripts-3.0.0/datalake_scripts/helper_scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake-scripts-2.7.3/datalake_scripts/engines → datalake_scripts-3.0.0/datalake_scripts/scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake-scripts-2.7.3/datalake_scripts/helper_scripts → datalake_scripts-3.0.0/tests}/__init__.py +0 -0
- {datalake-scripts-2.7.3/datalake_scripts/scripts → datalake_scripts-3.0.0/tests/common}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/common/test_output.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/common/test_utils.py +0 -0
- {datalake-scripts-2.7.3/tests → datalake_scripts-3.0.0/tests/lib}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/lib/test_atom_type.py +0 -0
- {datalake-scripts-2.7.3/tests/common → datalake_scripts-3.0.0/tests/scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake-scripts-2.7.3 → datalake_scripts-3.0.0}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version:
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datalake_scripts
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
7
7
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
License: UNKNOWN
|
|
9
|
-
Platform: UNKNOWN
|
|
10
8
|
Classifier: Programming Language :: Python :: 3.7
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
|
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Requires-Python: >=3.6
|
|
19
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: halo
|
|
20
|
+
Requires-Dist: prettytable
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: requires-dist
|
|
28
|
+
Dynamic: requires-python
|
|
29
|
+
Dynamic: summary
|
|
20
30
|
|
|
21
31
|
|
|
22
32
|
____ _ _ _ ____ _ _
|
|
@@ -27,11 +37,12 @@ Description-Content-Type: text/markdown
|
|
|
27
37
|
|_|
|
|
28
38
|
|
|
29
39
|
|
|
30
|
-
# datalake
|
|
31
|
-
Datalake scripts
|
|
32
|
-
|
|
33
40
|
## How to use
|
|
34
41
|
|
|
42
|
+
Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
43
|
+
|
|
44
|
+
You can use this repository either as a library or as a CLI
|
|
45
|
+
|
|
35
46
|
### Installation
|
|
36
47
|
|
|
37
48
|
With Python 3.6+:
|
|
@@ -40,7 +51,25 @@ $ pip install datalake-scripts
|
|
|
40
51
|
$ pip3 install datalake-scripts
|
|
41
52
|
```
|
|
42
53
|
### Using as a library
|
|
43
|
-
|
|
54
|
+
The library requires to first create a Datalake instance and then to use the defined Classes' methods
|
|
55
|
+
|
|
56
|
+
The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
57
|
+
|
|
58
|
+
Example :
|
|
59
|
+
```python
|
|
60
|
+
from datalake import Datalake, AtomType, Output
|
|
61
|
+
|
|
62
|
+
dtl = Datalake(username='username', password='password')
|
|
63
|
+
dtl.Threats.lookup(
|
|
64
|
+
atom_value='mayoclinic.org',
|
|
65
|
+
atom_type=AtomType.DOMAIN,
|
|
66
|
+
hashkey_only=False,
|
|
67
|
+
output=Output.JSON
|
|
68
|
+
)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
44
73
|
|
|
45
74
|
### Using as a CLI
|
|
46
75
|
|
|
@@ -52,9 +81,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
|
|
|
52
81
|
|
|
53
82
|
You can also use a script directly by using the following command: `<script_name> <script_options>`.
|
|
54
83
|
|
|
55
|
-
> /!\ Make sure to use utf-8 **without BOM** when providing a file (
|
|
84
|
+
> /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
|
|
56
85
|
|
|
57
|
-
|
|
86
|
+
#### Cli parameters
|
|
87
|
+
|
|
88
|
+
Common parameters for all commands:
|
|
89
|
+
* `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
|
|
90
|
+
* `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
|
|
91
|
+
* `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
|
|
92
|
+
* `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
|
|
93
|
+
|
|
94
|
+
Commands can also have additionary mandatory or optional parameters
|
|
95
|
+
|
|
96
|
+
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
### Environment variables
|
|
58
100
|
|
|
59
101
|
#### Authentication
|
|
60
102
|
|
|
@@ -64,7 +106,7 @@ There are two methods of authentication:
|
|
|
64
106
|
|
|
65
107
|
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
66
108
|
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
67
|
-
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
109
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
68
110
|
|
|
69
111
|
or
|
|
70
112
|
|
|
@@ -72,25 +114,27 @@ or
|
|
|
72
114
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
73
115
|
> These last two are independent and one can be used without the other if you wish.
|
|
74
116
|
|
|
75
|
-
####
|
|
76
|
-
For throttling the request, those two environment variable can be used:
|
|
77
|
-
* `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
78
|
-
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` define the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
117
|
+
#### Using a Proxy
|
|
79
118
|
|
|
80
|
-
|
|
119
|
+
You can set up following environment variables :
|
|
81
120
|
|
|
82
|
-
|
|
121
|
+
* `HTTP_PROXY`
|
|
122
|
+
* `HTTPS_PROXY`
|
|
83
123
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
> -e to change the environment {preprod, prod}, default is **prod**
|
|
87
|
-
> -o will set the output file as the API gives it.
|
|
88
|
-
> -q will quiet the verbosity of the program (but still show errors / warnings)
|
|
124
|
+
We use the format accepted by the requests python library.
|
|
125
|
+
See its documenation for other possible kinds of proxy to set up.
|
|
89
126
|
|
|
90
|
-
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
91
127
|
|
|
128
|
+
#### Throttling and retries
|
|
129
|
+
For throttling the requests, those two environment variables can be used:
|
|
130
|
+
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
131
|
+
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
132
|
+
We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
|
|
133
|
+
|
|
134
|
+
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
135
|
+
|
|
136
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
92
137
|
|
|
93
138
|
### Contributing
|
|
94
139
|
|
|
95
140
|
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
96
|
-
|
|
@@ -7,11 +7,12 @@
|
|
|
7
7
|
|_|
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
# datalake
|
|
11
|
-
Datalake scripts
|
|
12
|
-
|
|
13
10
|
## How to use
|
|
14
11
|
|
|
12
|
+
Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
13
|
+
|
|
14
|
+
You can use this repository either as a library or as a CLI
|
|
15
|
+
|
|
15
16
|
### Installation
|
|
16
17
|
|
|
17
18
|
With Python 3.6+:
|
|
@@ -20,7 +21,25 @@ $ pip install datalake-scripts
|
|
|
20
21
|
$ pip3 install datalake-scripts
|
|
21
22
|
```
|
|
22
23
|
### Using as a library
|
|
23
|
-
|
|
24
|
+
The library requires to first create a Datalake instance and then to use the defined Classes' methods
|
|
25
|
+
|
|
26
|
+
The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
27
|
+
|
|
28
|
+
Example :
|
|
29
|
+
```python
|
|
30
|
+
from datalake import Datalake, AtomType, Output
|
|
31
|
+
|
|
32
|
+
dtl = Datalake(username='username', password='password')
|
|
33
|
+
dtl.Threats.lookup(
|
|
34
|
+
atom_value='mayoclinic.org',
|
|
35
|
+
atom_type=AtomType.DOMAIN,
|
|
36
|
+
hashkey_only=False,
|
|
37
|
+
output=Output.JSON
|
|
38
|
+
)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
24
43
|
|
|
25
44
|
### Using as a CLI
|
|
26
45
|
|
|
@@ -32,9 +51,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
|
|
|
32
51
|
|
|
33
52
|
You can also use a script directly by using the following command: `<script_name> <script_options>`.
|
|
34
53
|
|
|
35
|
-
> /!\ Make sure to use utf-8 **without BOM** when providing a file (
|
|
54
|
+
> /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
|
|
36
55
|
|
|
37
|
-
|
|
56
|
+
#### Cli parameters
|
|
57
|
+
|
|
58
|
+
Common parameters for all commands:
|
|
59
|
+
* `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
|
|
60
|
+
* `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
|
|
61
|
+
* `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
|
|
62
|
+
* `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
|
|
63
|
+
|
|
64
|
+
Commands can also have additionary mandatory or optional parameters
|
|
65
|
+
|
|
66
|
+
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
### Environment variables
|
|
38
70
|
|
|
39
71
|
#### Authentication
|
|
40
72
|
|
|
@@ -44,7 +76,7 @@ There are two methods of authentication:
|
|
|
44
76
|
|
|
45
77
|
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
46
78
|
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
47
|
-
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
79
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
48
80
|
|
|
49
81
|
or
|
|
50
82
|
|
|
@@ -52,23 +84,26 @@ or
|
|
|
52
84
|
* `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
|
|
53
85
|
> These last two are independent and one can be used without the other if you wish.
|
|
54
86
|
|
|
55
|
-
####
|
|
56
|
-
For throttling the request, those two environment variable can be used:
|
|
57
|
-
* `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
58
|
-
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` define the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
87
|
+
#### Using a Proxy
|
|
59
88
|
|
|
60
|
-
|
|
89
|
+
You can set up following environment variables :
|
|
61
90
|
|
|
62
|
-
|
|
91
|
+
* `HTTP_PROXY`
|
|
92
|
+
* `HTTPS_PROXY`
|
|
63
93
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
> -e to change the environment {preprod, prod}, default is **prod**
|
|
67
|
-
> -o will set the output file as the API gives it.
|
|
68
|
-
> -q will quiet the verbosity of the program (but still show errors / warnings)
|
|
94
|
+
We use the format accepted by the requests python library.
|
|
95
|
+
See its documenation for other possible kinds of proxy to set up.
|
|
69
96
|
|
|
70
|
-
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
71
97
|
|
|
98
|
+
#### Throttling and retries
|
|
99
|
+
For throttling the requests, those two environment variables can be used:
|
|
100
|
+
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
101
|
+
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
102
|
+
We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
|
|
103
|
+
|
|
104
|
+
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
105
|
+
|
|
106
|
+
Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
|
|
72
107
|
|
|
73
108
|
### Contributing
|
|
74
109
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
from .
|
|
1
|
+
from .common.bulk_search_task import (
|
|
2
2
|
BulkSearchTask,
|
|
3
3
|
BulkSearchTaskState,
|
|
4
4
|
BulkSearchFailedError,
|
|
5
5
|
BulkSearchNotFound,
|
|
6
6
|
)
|
|
7
|
+
|
|
7
8
|
from .common.atom import (
|
|
8
9
|
AtomType,
|
|
9
10
|
ThreatType,
|
|
@@ -12,29 +13,23 @@ from .common.atom import (
|
|
|
12
13
|
Visibility,
|
|
13
14
|
SightingRelation,
|
|
14
15
|
)
|
|
16
|
+
|
|
15
17
|
from .common.atom_type import (
|
|
16
18
|
Atom,
|
|
17
19
|
Hashes,
|
|
18
20
|
FileAtom,
|
|
19
21
|
AndroidApp,
|
|
20
|
-
ApkAtom,
|
|
21
22
|
AsAtom,
|
|
22
|
-
CcAtom,
|
|
23
23
|
CryptoAtom,
|
|
24
|
-
CveAtom,
|
|
25
24
|
Jarm,
|
|
26
25
|
DomainAtom,
|
|
27
26
|
EmailFlow,
|
|
28
27
|
EmailAtom,
|
|
29
|
-
FqdnAtom,
|
|
30
|
-
IbanAtom,
|
|
31
28
|
IpService,
|
|
32
29
|
IpAtom,
|
|
33
30
|
IpRangeAtom,
|
|
34
|
-
PasteAtom,
|
|
35
31
|
PhoneNumberAtom,
|
|
36
|
-
|
|
37
|
-
SslAtom,
|
|
32
|
+
CertificateAtom,
|
|
38
33
|
UrlAtom,
|
|
39
34
|
)
|
|
40
35
|
from .common.output import Output
|
|
@@ -1,27 +1,21 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Enums for atom properties
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
from enum import Enum
|
|
5
6
|
from typing import Dict, Union
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
class AtomType(Enum):
|
|
9
|
-
APK = "apk"
|
|
10
10
|
AS = "as"
|
|
11
|
-
CC = "cc"
|
|
12
11
|
CRYPTO = "crypto"
|
|
13
|
-
CVE = "cve"
|
|
14
12
|
DOMAIN = "domain"
|
|
15
13
|
EMAIL = "email"
|
|
16
14
|
FILE = "file"
|
|
17
|
-
FQDN = "fqdn"
|
|
18
|
-
IBAN = "iban"
|
|
19
15
|
IP = "ip"
|
|
20
16
|
IP_RANGE = "ip_range"
|
|
21
|
-
PASTE = "paste"
|
|
22
17
|
PHONE_NUMBER = "phone_number"
|
|
23
|
-
|
|
24
|
-
SSL = "ssl"
|
|
18
|
+
CERTIFICATE = "certificate"
|
|
25
19
|
URL = "url"
|
|
26
20
|
|
|
27
21
|
|
|
@@ -48,7 +42,6 @@ threat_type -> ThreatType
|
|
|
48
42
|
|
|
49
43
|
|
|
50
44
|
class OverrideType(Enum):
|
|
51
|
-
PERMANENT = "permanent"
|
|
52
45
|
TEMPORARY = "temporary"
|
|
53
46
|
LOCK = "lock"
|
|
54
47
|
|
|
@@ -66,28 +66,6 @@ class Hashes:
|
|
|
66
66
|
ssdeep: str = None
|
|
67
67
|
|
|
68
68
|
|
|
69
|
-
@dataclass
|
|
70
|
-
class FileAtom(Atom):
|
|
71
|
-
"""
|
|
72
|
-
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
73
|
-
"""
|
|
74
|
-
|
|
75
|
-
hashes: Hashes
|
|
76
|
-
external_analysis_link: List[str] = None
|
|
77
|
-
filesize: int = None
|
|
78
|
-
filetype: str = None
|
|
79
|
-
file_url: str = None
|
|
80
|
-
mimetype: str = None
|
|
81
|
-
filename: str = None
|
|
82
|
-
filepath: str = None
|
|
83
|
-
|
|
84
|
-
def _get_sightings_allowed_keys(self):
|
|
85
|
-
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
86
|
-
|
|
87
|
-
def _get_sightings_prefix(self):
|
|
88
|
-
return "file"
|
|
89
|
-
|
|
90
|
-
|
|
91
69
|
@dataclass
|
|
92
70
|
class AndroidApp:
|
|
93
71
|
package_name: str
|
|
@@ -98,35 +76,26 @@ class AndroidApp:
|
|
|
98
76
|
|
|
99
77
|
|
|
100
78
|
@dataclass
|
|
101
|
-
class
|
|
79
|
+
class FileAtom(Atom):
|
|
102
80
|
"""
|
|
103
|
-
Allowed sighting key: ['
|
|
81
|
+
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
104
82
|
"""
|
|
105
83
|
|
|
106
|
-
android: AndroidApp
|
|
107
84
|
hashes: Hashes
|
|
108
85
|
external_analysis_link: List[str] = None
|
|
109
86
|
filesize: int = None
|
|
110
87
|
filetype: str = None
|
|
111
88
|
file_url: str = None
|
|
112
89
|
mimetype: str = None
|
|
113
|
-
filename: str = None
|
|
90
|
+
filename: List[str] = None
|
|
114
91
|
filepath: str = None
|
|
92
|
+
android: AndroidApp = None
|
|
115
93
|
|
|
116
94
|
def _get_sightings_allowed_keys(self):
|
|
117
|
-
return [
|
|
118
|
-
"android",
|
|
119
|
-
"package_name",
|
|
120
|
-
"version_name",
|
|
121
|
-
"hashes",
|
|
122
|
-
"md5",
|
|
123
|
-
"sha1",
|
|
124
|
-
"sha256",
|
|
125
|
-
"sha512",
|
|
126
|
-
]
|
|
95
|
+
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
127
96
|
|
|
128
97
|
def _get_sightings_prefix(self):
|
|
129
|
-
return "
|
|
98
|
+
return "file"
|
|
130
99
|
|
|
131
100
|
|
|
132
101
|
@dataclass
|
|
@@ -150,28 +119,6 @@ class AsAtom(Atom):
|
|
|
150
119
|
return "as"
|
|
151
120
|
|
|
152
121
|
|
|
153
|
-
@dataclass
|
|
154
|
-
class CcAtom(Atom):
|
|
155
|
-
"""
|
|
156
|
-
Allowed sighting key: ['number']
|
|
157
|
-
number: minLength = 8 maxLength = 19
|
|
158
|
-
"""
|
|
159
|
-
|
|
160
|
-
number: str
|
|
161
|
-
external_analysis_link: List[str] = None
|
|
162
|
-
bin: int = None
|
|
163
|
-
brand: str = None
|
|
164
|
-
cvx: int = None
|
|
165
|
-
description: str = None
|
|
166
|
-
expiry_date: str = None
|
|
167
|
-
|
|
168
|
-
def _get_sightings_allowed_keys(self):
|
|
169
|
-
return ["number"]
|
|
170
|
-
|
|
171
|
-
def _get_sightings_prefix(self):
|
|
172
|
-
return "cc"
|
|
173
|
-
|
|
174
|
-
|
|
175
122
|
@dataclass
|
|
176
123
|
class CryptoAtom(Atom):
|
|
177
124
|
"""
|
|
@@ -197,28 +144,6 @@ class CryptoAtom(Atom):
|
|
|
197
144
|
return "crypto"
|
|
198
145
|
|
|
199
146
|
|
|
200
|
-
@dataclass
|
|
201
|
-
class CveAtom(Atom):
|
|
202
|
-
"""
|
|
203
|
-
Allowed sighting key: ['cve_id']
|
|
204
|
-
"""
|
|
205
|
-
|
|
206
|
-
"""
|
|
207
|
-
published_at expect the following datetime format: '%Y-%m-%dT%H:%M:%SZ'
|
|
208
|
-
"""
|
|
209
|
-
cve_id: str
|
|
210
|
-
cvss: int = None
|
|
211
|
-
cwe: str = None
|
|
212
|
-
external_analysis_link: List[str] = None
|
|
213
|
-
published_at: str = None
|
|
214
|
-
|
|
215
|
-
def _get_sightings_allowed_keys(self):
|
|
216
|
-
return ["cve_id"]
|
|
217
|
-
|
|
218
|
-
def _get_sightings_prefix(self):
|
|
219
|
-
return "cve"
|
|
220
|
-
|
|
221
|
-
|
|
222
147
|
@dataclass
|
|
223
148
|
class Jarm:
|
|
224
149
|
calculated_at: str = None
|
|
@@ -237,6 +162,8 @@ class DomainAtom(Atom):
|
|
|
237
162
|
external_analysis_link: List[str] = None
|
|
238
163
|
malware_family: str = None
|
|
239
164
|
jarm: Jarm = None
|
|
165
|
+
port: List[int] = None
|
|
166
|
+
ns: List[str] = None
|
|
240
167
|
|
|
241
168
|
def _get_sightings_allowed_keys(self):
|
|
242
169
|
return ["domain"]
|
|
@@ -267,47 +194,6 @@ class EmailAtom(Atom):
|
|
|
267
194
|
return "email"
|
|
268
195
|
|
|
269
196
|
|
|
270
|
-
@dataclass
|
|
271
|
-
class FqdnAtom(Atom):
|
|
272
|
-
"""
|
|
273
|
-
Allowed sighting key: ['fqdn']
|
|
274
|
-
"""
|
|
275
|
-
|
|
276
|
-
fqdn: str
|
|
277
|
-
jarm: Jarm = None
|
|
278
|
-
malware_family: str = None
|
|
279
|
-
port: List[int] = None
|
|
280
|
-
ns: List[str] = None
|
|
281
|
-
external_analysis_link: List[str] = None
|
|
282
|
-
|
|
283
|
-
def _get_sightings_allowed_keys(self):
|
|
284
|
-
return ["fqdn"]
|
|
285
|
-
|
|
286
|
-
def _get_sightings_prefix(self):
|
|
287
|
-
return "fqdn"
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
@dataclass
|
|
291
|
-
class IbanAtom(Atom):
|
|
292
|
-
"""
|
|
293
|
-
Allowed sighting key: ['iban']
|
|
294
|
-
"""
|
|
295
|
-
|
|
296
|
-
iban: str
|
|
297
|
-
holder_name: str = None
|
|
298
|
-
holder_address: str = None
|
|
299
|
-
external_analysis_link: List[str] = None
|
|
300
|
-
bic: str = None
|
|
301
|
-
bank_name: str = None
|
|
302
|
-
bank_address: str = None
|
|
303
|
-
|
|
304
|
-
def _get_sightings_allowed_keys(self):
|
|
305
|
-
return ["iban"]
|
|
306
|
-
|
|
307
|
-
def _get_sightings_prefix(self):
|
|
308
|
-
return "iban"
|
|
309
|
-
|
|
310
|
-
|
|
311
197
|
@dataclass
|
|
312
198
|
class IpService:
|
|
313
199
|
port: int
|
|
@@ -360,67 +246,29 @@ class IpRangeAtom(Atom):
|
|
|
360
246
|
return "ip_range"
|
|
361
247
|
|
|
362
248
|
|
|
363
|
-
@dataclass
|
|
364
|
-
class PasteAtom(Atom):
|
|
365
|
-
"""
|
|
366
|
-
Allowed sighting key: ['url']
|
|
367
|
-
"""
|
|
368
|
-
|
|
369
|
-
url: str
|
|
370
|
-
author: str = None
|
|
371
|
-
title: str = None
|
|
372
|
-
content: str = None
|
|
373
|
-
external_analysis_link: List[str] = None
|
|
374
|
-
|
|
375
|
-
def _get_sightings_allowed_keys(self):
|
|
376
|
-
return ["url"]
|
|
377
|
-
|
|
378
|
-
def _get_sightings_prefix(self):
|
|
379
|
-
return "paste"
|
|
380
|
-
|
|
381
|
-
|
|
382
249
|
@dataclass
|
|
383
250
|
class PhoneNumberAtom(Atom):
|
|
384
251
|
"""
|
|
385
|
-
Allowed sighting key:
|
|
252
|
+
Allowed sighting key: 'international_phone_number'
|
|
386
253
|
"""
|
|
387
254
|
|
|
388
255
|
"""
|
|
389
256
|
country: minLength = 2 maxLength = 2
|
|
390
257
|
"""
|
|
258
|
+
international_phone_number: str
|
|
391
259
|
company: str = None
|
|
392
260
|
country: str = None
|
|
393
261
|
external_analysis_link: List[str] = None
|
|
394
|
-
international_phone_number: str = None
|
|
395
|
-
national_phone_number: str = None
|
|
396
262
|
|
|
397
263
|
def _get_sightings_allowed_keys(self):
|
|
398
|
-
return ["international_phone_number"
|
|
264
|
+
return ["international_phone_number"]
|
|
399
265
|
|
|
400
266
|
def _get_sightings_prefix(self):
|
|
401
267
|
return "phone_number"
|
|
402
268
|
|
|
403
269
|
|
|
404
270
|
@dataclass
|
|
405
|
-
class
|
|
406
|
-
"""
|
|
407
|
-
Allowed sighting key: ['path']
|
|
408
|
-
"""
|
|
409
|
-
|
|
410
|
-
path: str
|
|
411
|
-
regkey_value: str = None
|
|
412
|
-
hive: str = None
|
|
413
|
-
external_analysis_link: List[str] = None
|
|
414
|
-
|
|
415
|
-
def _get_sightings_allowed_keys(self):
|
|
416
|
-
return ["path"]
|
|
417
|
-
|
|
418
|
-
def _get_sightings_prefix(self):
|
|
419
|
-
return "regkey"
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
@dataclass
|
|
423
|
-
class SslAtom(Atom):
|
|
271
|
+
class CertificateAtom(Atom):
|
|
424
272
|
"""
|
|
425
273
|
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
426
274
|
"""
|
|
@@ -440,7 +288,7 @@ class SslAtom(Atom):
|
|
|
440
288
|
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
441
289
|
|
|
442
290
|
def _get_sightings_prefix(self):
|
|
443
|
-
return "
|
|
291
|
+
return "certificate"
|
|
444
292
|
|
|
445
293
|
|
|
446
294
|
@dataclass
|
|
@@ -16,12 +16,16 @@ class BulkSearchTaskState(Enum):
|
|
|
16
16
|
CANCELLED = "CANCELLED"
|
|
17
17
|
FAILED_ERROR = "FAILED_ERROR"
|
|
18
18
|
FAILED_TIMEOUT = "FAILED_TIMEOUT"
|
|
19
|
+
FAILED_QUOTA_EXCEEDED = "FAILED_QUOTA_EXCEEDED"
|
|
20
|
+
FAILED_TOO_MANY_RESULTS = "FAILED_TOO_MANY_RESULTS"
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
BULK_SEARCH_FAILED_STATE = {
|
|
22
24
|
BulkSearchTaskState.CANCELLED,
|
|
23
25
|
BulkSearchTaskState.FAILED_TIMEOUT,
|
|
24
26
|
BulkSearchTaskState.FAILED_ERROR,
|
|
27
|
+
BulkSearchTaskState.FAILED_QUOTA_EXCEEDED,
|
|
28
|
+
BulkSearchTaskState.FAILED_TOO_MANY_RESULTS,
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
|
|
@@ -98,11 +102,11 @@ class BulkSearchTask:
|
|
|
98
102
|
Note that the requests library is used which is blocking.
|
|
99
103
|
timeout parameter is in seconds.
|
|
100
104
|
"""
|
|
101
|
-
start = datetime.datetime.
|
|
105
|
+
start = datetime.datetime.now(datetime.timezone.utc)
|
|
102
106
|
while self.state != BulkSearchTaskState.DONE:
|
|
103
107
|
if self.state in BULK_SEARCH_FAILED_STATE:
|
|
104
108
|
raise BulkSearchFailedError(self.state)
|
|
105
|
-
time_passed = datetime.datetime.
|
|
109
|
+
time_passed = datetime.datetime.now(datetime.timezone.utc) - start
|
|
106
110
|
if time_passed.total_seconds() > timeout:
|
|
107
111
|
raise TimeoutError()
|
|
108
112
|
|