datalake-scripts 2.7.4__tar.gz → 3.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake-scripts-2.7.4/datalake_scripts.egg-info → datalake_scripts-3.0.0rc1}/PKG-INFO +34 -23
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/README.md +32 -21
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/__init__.py +4 -9
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/atom.py +2 -9
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/atom_type.py +13 -165
- {datalake-scripts-2.7.4/datalake/api_objects → datalake_scripts-3.0.0rc1/datalake/common}/bulk_search_task.py +6 -2
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/logger.py +4 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/token_manager.py +11 -9
- datalake_scripts-3.0.0rc1/datalake/datalake.py +130 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/advanced_search.py +7 -4
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/bulk_search.py +4 -3
- datalake_scripts-3.0.0rc1/datalake/endpoints/comments.py +48 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/endpoint.py +31 -25
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/filtered_tag_subcategory.py +4 -6
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/sightings.py +123 -27
- datalake_scripts-3.0.0rc1/datalake/endpoints/sources.py +33 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/tags.py +1 -1
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/threats.py +114 -42
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/miscellaneous/search_watch.py +21 -17
- datalake_scripts-3.0.0rc1/datalake_scripts/__init__.py +1 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/cli.py +3 -5
- datalake_scripts-3.0.0rc1/datalake_scripts/common/base_script.py +79 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/helper_scripts/utils.py +8 -2
- datalake_scripts-3.0.0rc1/datalake_scripts/scripts/add_comments.py +73 -0
- datalake_scripts-3.0.0rc1/datalake_scripts/scripts/add_tags.py +80 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/add_threats.py +134 -149
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/advanced_search.py +13 -9
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/bulk_lookup_threats.py +102 -97
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/edit_score.py +28 -47
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/get_atom_values.py +24 -35
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/get_filtered_tag_subcategory.py +6 -19
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/get_query_hash.py +8 -8
- datalake_scripts-3.0.0rc1/datalake_scripts/scripts/get_threats.py +58 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/get_threats_from_query_hash.py +21 -26
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/lookup_threats.py +22 -22
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/scripts/search_watch.py +13 -14
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1/datalake_scripts.egg-info}/PKG-INFO +33 -22
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts.egg-info/SOURCES.txt +5 -10
- datalake_scripts-3.0.0rc1/datalake_scripts.egg-info/entry_points.txt +3 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/setup.py +2 -10
- datalake_scripts-3.0.0rc1/tests/common/fixture.py +116 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/conftest.py +3 -3
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_advanced_search.py +24 -11
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_auth_via_tokens.py +72 -33
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_bulk_search.py +96 -22
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_filtered_tag_subcategory.py +9 -5
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_search_watch.py +7 -3
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_sightings.py +50 -29
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_tags.py +37 -29
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_threats.py +73 -39
- datalake-scripts-2.7.4/tests/scripts/test_base_engine.py → datalake_scripts-3.0.0rc1/tests/scripts/test_base_scripts.py +4 -4
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/scripts/test_cli.py +2 -5
- datalake-scripts-2.7.4/datalake/datalake.py +0 -127
- datalake-scripts-2.7.4/datalake_scripts/__init__.py +0 -5
- datalake-scripts-2.7.4/datalake_scripts/common/base_engine.py +0 -123
- datalake-scripts-2.7.4/datalake_scripts/common/base_script.py +0 -83
- datalake-scripts-2.7.4/datalake_scripts/common/mixins.py +0 -86
- datalake-scripts-2.7.4/datalake_scripts/engines/get_engine.py +0 -104
- datalake-scripts-2.7.4/datalake_scripts/engines/post_engine.py +0 -199
- datalake-scripts-2.7.4/datalake_scripts/scripts/add_comments.py +0 -77
- datalake-scripts-2.7.4/datalake_scripts/scripts/add_tags.py +0 -84
- datalake-scripts-2.7.4/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -60
- datalake-scripts-2.7.4/datalake_scripts.egg-info/entry_points.txt +0 -9
- datalake-scripts-2.7.4/tests/common/fixture.py +0 -57
- datalake-scripts-2.7.4/tests/lib/__init__.py +0 -0
- datalake-scripts-2.7.4/tests/scripts/__init__.py +0 -0
- datalake-scripts-2.7.4/tests/scripts/test_endpoint_config.py +0 -26
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/config.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/output.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/throttler.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/utils.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/common/warn.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake/endpoints/__init__.py +0 -0
- {datalake-scripts-2.7.4/datalake/api_objects → datalake_scripts-3.0.0rc1/datalake/miscellaneous}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/common/__init__.py +0 -0
- {datalake-scripts-2.7.4/datalake/miscellaneous → datalake_scripts-3.0.0rc1/datalake_scripts/helper_scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake-scripts-2.7.4/datalake_scripts/engines → datalake_scripts-3.0.0rc1/datalake_scripts/scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/setup.cfg +0 -0
- {datalake-scripts-2.7.4/datalake_scripts/helper_scripts → datalake_scripts-3.0.0rc1/tests}/__init__.py +0 -0
- {datalake-scripts-2.7.4/datalake_scripts/scripts → datalake_scripts-3.0.0rc1/tests/common}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/common/test_output.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/common/test_utils.py +0 -0
- {datalake-scripts-2.7.4/tests → datalake_scripts-3.0.0rc1/tests/lib}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/lib/test_atom_type.py +0 -0
- {datalake-scripts-2.7.4/tests/common → datalake_scripts-3.0.0rc1/tests/scripts}/__init__.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake-scripts-2.7.4 → datalake_scripts-3.0.0rc1}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
|
-
Name:
|
|
3
|
-
Version:
|
|
2
|
+
Name: datalake_scripts
|
|
3
|
+
Version: 3.0.0rc1
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -27,11 +27,12 @@ Description-Content-Type: text/markdown
|
|
|
27
27
|
|_|
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
# datalake
|
|
31
|
-
Datalake scripts
|
|
32
|
-
|
|
33
30
|
## How to use
|
|
34
31
|
|
|
32
|
+
Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
33
|
+
|
|
34
|
+
You can use this repository either as a library or as a CLI
|
|
35
|
+
|
|
35
36
|
### Installation
|
|
36
37
|
|
|
37
38
|
With Python 3.6+:
|
|
@@ -40,6 +41,11 @@ $ pip install datalake-scripts
|
|
|
40
41
|
$ pip3 install datalake-scripts
|
|
41
42
|
```
|
|
42
43
|
### Using as a library
|
|
44
|
+
The library requires to first create a Datalake instance and then to use the defined Classes' methods
|
|
45
|
+
|
|
46
|
+
The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
47
|
+
|
|
48
|
+
Example :
|
|
43
49
|
```python
|
|
44
50
|
from datalake import Datalake, AtomType, Output
|
|
45
51
|
|
|
@@ -52,7 +58,8 @@ dtl.Threats.lookup(
|
|
|
52
58
|
)
|
|
53
59
|
```
|
|
54
60
|
|
|
55
|
-
|
|
61
|
+
|
|
62
|
+
|
|
56
63
|
|
|
57
64
|
### Using as a CLI
|
|
58
65
|
|
|
@@ -64,9 +71,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
|
|
|
64
71
|
|
|
65
72
|
You can also use a script directly by using the following command: `<script_name> <script_options>`.
|
|
66
73
|
|
|
67
|
-
> /!\ Make sure to use utf-8 **without BOM** when providing a file (
|
|
74
|
+
> /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
|
|
75
|
+
|
|
76
|
+
#### Cli parameters
|
|
77
|
+
|
|
78
|
+
Common parameters for all commands:
|
|
79
|
+
* `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
|
|
80
|
+
* `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
|
|
81
|
+
* `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
|
|
82
|
+
* `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
|
|
68
83
|
|
|
69
|
-
|
|
84
|
+
Commands can also have additionary mandatory or optional parameters
|
|
85
|
+
|
|
86
|
+
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
### Environment variables
|
|
70
90
|
|
|
71
91
|
#### Authentication
|
|
72
92
|
|
|
@@ -76,7 +96,7 @@ There are two methods of authentication:
|
|
|
76
96
|
|
|
77
97
|
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
78
98
|
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
79
|
-
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
99
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
80
100
|
|
|
81
101
|
or
|
|
82
102
|
|
|
@@ -96,21 +116,12 @@ See its documenation for other possible kinds of proxy to set up.
|
|
|
96
116
|
|
|
97
117
|
|
|
98
118
|
#### Throttling
|
|
99
|
-
For throttling the
|
|
100
|
-
* `OCD_DTL_QUOTA_TIME`
|
|
101
|
-
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME`
|
|
119
|
+
For throttling the requests, those two environment variables can be used:
|
|
120
|
+
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
121
|
+
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
122
|
+
We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
|
|
102
123
|
|
|
103
|
-
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
104
|
-
|
|
105
|
-
## Cli parameters
|
|
106
|
-
|
|
107
|
-
Parameters common and optional for all commands:
|
|
108
|
-
> --debug display more information for debugging purposes
|
|
109
|
-
> -e to change the environment {preprod, prod}, default is **prod**
|
|
110
|
-
> -o will set the output file as the API gives it.
|
|
111
|
-
> -q will quiet the verbosity of the program (but still show errors / warnings)
|
|
112
|
-
|
|
113
|
-
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
124
|
+
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
114
125
|
|
|
115
126
|
|
|
116
127
|
### Contributing
|
|
@@ -7,11 +7,12 @@
|
|
|
7
7
|
|_|
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
# datalake
|
|
11
|
-
Datalake scripts
|
|
12
|
-
|
|
13
10
|
## How to use
|
|
14
11
|
|
|
12
|
+
Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
13
|
+
|
|
14
|
+
You can use this repository either as a library or as a CLI
|
|
15
|
+
|
|
15
16
|
### Installation
|
|
16
17
|
|
|
17
18
|
With Python 3.6+:
|
|
@@ -20,6 +21,11 @@ $ pip install datalake-scripts
|
|
|
20
21
|
$ pip3 install datalake-scripts
|
|
21
22
|
```
|
|
22
23
|
### Using as a library
|
|
24
|
+
The library requires to first create a Datalake instance and then to use the defined Classes' methods
|
|
25
|
+
|
|
26
|
+
The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
|
|
27
|
+
|
|
28
|
+
Example :
|
|
23
29
|
```python
|
|
24
30
|
from datalake import Datalake, AtomType, Output
|
|
25
31
|
|
|
@@ -32,7 +38,8 @@ dtl.Threats.lookup(
|
|
|
32
38
|
)
|
|
33
39
|
```
|
|
34
40
|
|
|
35
|
-
|
|
41
|
+
|
|
42
|
+
|
|
36
43
|
|
|
37
44
|
### Using as a CLI
|
|
38
45
|
|
|
@@ -44,9 +51,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
|
|
|
44
51
|
|
|
45
52
|
You can also use a script directly by using the following command: `<script_name> <script_options>`.
|
|
46
53
|
|
|
47
|
-
> /!\ Make sure to use utf-8 **without BOM** when providing a file (
|
|
54
|
+
> /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
|
|
55
|
+
|
|
56
|
+
#### Cli parameters
|
|
57
|
+
|
|
58
|
+
Common parameters for all commands:
|
|
59
|
+
* `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
|
|
60
|
+
* `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
|
|
61
|
+
* `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
|
|
62
|
+
* `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
|
|
48
63
|
|
|
49
|
-
|
|
64
|
+
Commands can also have additionary mandatory or optional parameters
|
|
65
|
+
|
|
66
|
+
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
### Environment variables
|
|
50
70
|
|
|
51
71
|
#### Authentication
|
|
52
72
|
|
|
@@ -56,7 +76,7 @@ There are two methods of authentication:
|
|
|
56
76
|
|
|
57
77
|
In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
|
|
58
78
|
* `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
|
|
59
|
-
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
79
|
+
Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
|
|
60
80
|
|
|
61
81
|
or
|
|
62
82
|
|
|
@@ -76,21 +96,12 @@ See its documenation for other possible kinds of proxy to set up.
|
|
|
76
96
|
|
|
77
97
|
|
|
78
98
|
#### Throttling
|
|
79
|
-
For throttling the
|
|
80
|
-
* `OCD_DTL_QUOTA_TIME`
|
|
81
|
-
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME`
|
|
99
|
+
For throttling the requests, those two environment variables can be used:
|
|
100
|
+
* `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
|
|
101
|
+
* `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
|
|
102
|
+
We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
|
|
82
103
|
|
|
83
|
-
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/
|
|
84
|
-
|
|
85
|
-
## Cli parameters
|
|
86
|
-
|
|
87
|
-
Parameters common and optional for all commands:
|
|
88
|
-
> --debug display more information for debugging purposes
|
|
89
|
-
> -e to change the environment {preprod, prod}, default is **prod**
|
|
90
|
-
> -o will set the output file as the API gives it.
|
|
91
|
-
> -q will quiet the verbosity of the program (but still show errors / warnings)
|
|
92
|
-
|
|
93
|
-
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
104
|
+
> Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
|
|
94
105
|
|
|
95
106
|
|
|
96
107
|
### Contributing
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
from .
|
|
1
|
+
from .common.bulk_search_task import (
|
|
2
2
|
BulkSearchTask,
|
|
3
3
|
BulkSearchTaskState,
|
|
4
4
|
BulkSearchFailedError,
|
|
5
5
|
BulkSearchNotFound,
|
|
6
6
|
)
|
|
7
|
+
|
|
7
8
|
from .common.atom import (
|
|
8
9
|
AtomType,
|
|
9
10
|
ThreatType,
|
|
@@ -12,29 +13,23 @@ from .common.atom import (
|
|
|
12
13
|
Visibility,
|
|
13
14
|
SightingRelation,
|
|
14
15
|
)
|
|
16
|
+
|
|
15
17
|
from .common.atom_type import (
|
|
16
18
|
Atom,
|
|
17
19
|
Hashes,
|
|
18
20
|
FileAtom,
|
|
19
21
|
AndroidApp,
|
|
20
|
-
ApkAtom,
|
|
21
22
|
AsAtom,
|
|
22
|
-
CcAtom,
|
|
23
23
|
CryptoAtom,
|
|
24
|
-
CveAtom,
|
|
25
24
|
Jarm,
|
|
26
25
|
DomainAtom,
|
|
27
26
|
EmailFlow,
|
|
28
27
|
EmailAtom,
|
|
29
|
-
FqdnAtom,
|
|
30
|
-
IbanAtom,
|
|
31
28
|
IpService,
|
|
32
29
|
IpAtom,
|
|
33
30
|
IpRangeAtom,
|
|
34
|
-
PasteAtom,
|
|
35
31
|
PhoneNumberAtom,
|
|
36
|
-
|
|
37
|
-
SslAtom,
|
|
32
|
+
CertificateAtom,
|
|
38
33
|
UrlAtom,
|
|
39
34
|
)
|
|
40
35
|
from .common.output import Output
|
|
@@ -1,27 +1,21 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Enums for atom properties
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
from enum import Enum
|
|
5
6
|
from typing import Dict, Union
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
class AtomType(Enum):
|
|
9
|
-
APK = "apk"
|
|
10
10
|
AS = "as"
|
|
11
|
-
CC = "cc"
|
|
12
11
|
CRYPTO = "crypto"
|
|
13
|
-
CVE = "cve"
|
|
14
12
|
DOMAIN = "domain"
|
|
15
13
|
EMAIL = "email"
|
|
16
14
|
FILE = "file"
|
|
17
|
-
FQDN = "fqdn"
|
|
18
|
-
IBAN = "iban"
|
|
19
15
|
IP = "ip"
|
|
20
16
|
IP_RANGE = "ip_range"
|
|
21
|
-
PASTE = "paste"
|
|
22
17
|
PHONE_NUMBER = "phone_number"
|
|
23
|
-
|
|
24
|
-
SSL = "ssl"
|
|
18
|
+
CERTIFICATE = "certificate"
|
|
25
19
|
URL = "url"
|
|
26
20
|
|
|
27
21
|
|
|
@@ -48,7 +42,6 @@ threat_type -> ThreatType
|
|
|
48
42
|
|
|
49
43
|
|
|
50
44
|
class OverrideType(Enum):
|
|
51
|
-
PERMANENT = "permanent"
|
|
52
45
|
TEMPORARY = "temporary"
|
|
53
46
|
LOCK = "lock"
|
|
54
47
|
|
|
@@ -66,28 +66,6 @@ class Hashes:
|
|
|
66
66
|
ssdeep: str = None
|
|
67
67
|
|
|
68
68
|
|
|
69
|
-
@dataclass
|
|
70
|
-
class FileAtom(Atom):
|
|
71
|
-
"""
|
|
72
|
-
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
73
|
-
"""
|
|
74
|
-
|
|
75
|
-
hashes: Hashes
|
|
76
|
-
external_analysis_link: List[str] = None
|
|
77
|
-
filesize: int = None
|
|
78
|
-
filetype: str = None
|
|
79
|
-
file_url: str = None
|
|
80
|
-
mimetype: str = None
|
|
81
|
-
filename: str = None
|
|
82
|
-
filepath: str = None
|
|
83
|
-
|
|
84
|
-
def _get_sightings_allowed_keys(self):
|
|
85
|
-
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
86
|
-
|
|
87
|
-
def _get_sightings_prefix(self):
|
|
88
|
-
return "file"
|
|
89
|
-
|
|
90
|
-
|
|
91
69
|
@dataclass
|
|
92
70
|
class AndroidApp:
|
|
93
71
|
package_name: str
|
|
@@ -98,35 +76,26 @@ class AndroidApp:
|
|
|
98
76
|
|
|
99
77
|
|
|
100
78
|
@dataclass
|
|
101
|
-
class
|
|
79
|
+
class FileAtom(Atom):
|
|
102
80
|
"""
|
|
103
|
-
Allowed sighting key: ['
|
|
81
|
+
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
104
82
|
"""
|
|
105
83
|
|
|
106
|
-
android: AndroidApp
|
|
107
84
|
hashes: Hashes
|
|
108
85
|
external_analysis_link: List[str] = None
|
|
109
86
|
filesize: int = None
|
|
110
87
|
filetype: str = None
|
|
111
88
|
file_url: str = None
|
|
112
89
|
mimetype: str = None
|
|
113
|
-
filename: str = None
|
|
90
|
+
filename: List[str] = None
|
|
114
91
|
filepath: str = None
|
|
92
|
+
android: AndroidApp = None
|
|
115
93
|
|
|
116
94
|
def _get_sightings_allowed_keys(self):
|
|
117
|
-
return [
|
|
118
|
-
"android",
|
|
119
|
-
"package_name",
|
|
120
|
-
"version_name",
|
|
121
|
-
"hashes",
|
|
122
|
-
"md5",
|
|
123
|
-
"sha1",
|
|
124
|
-
"sha256",
|
|
125
|
-
"sha512",
|
|
126
|
-
]
|
|
95
|
+
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
127
96
|
|
|
128
97
|
def _get_sightings_prefix(self):
|
|
129
|
-
return "
|
|
98
|
+
return "file"
|
|
130
99
|
|
|
131
100
|
|
|
132
101
|
@dataclass
|
|
@@ -150,28 +119,6 @@ class AsAtom(Atom):
|
|
|
150
119
|
return "as"
|
|
151
120
|
|
|
152
121
|
|
|
153
|
-
@dataclass
|
|
154
|
-
class CcAtom(Atom):
|
|
155
|
-
"""
|
|
156
|
-
Allowed sighting key: ['number']
|
|
157
|
-
number: minLength = 8 maxLength = 19
|
|
158
|
-
"""
|
|
159
|
-
|
|
160
|
-
number: str
|
|
161
|
-
external_analysis_link: List[str] = None
|
|
162
|
-
bin: int = None
|
|
163
|
-
brand: str = None
|
|
164
|
-
cvx: int = None
|
|
165
|
-
description: str = None
|
|
166
|
-
expiry_date: str = None
|
|
167
|
-
|
|
168
|
-
def _get_sightings_allowed_keys(self):
|
|
169
|
-
return ["number"]
|
|
170
|
-
|
|
171
|
-
def _get_sightings_prefix(self):
|
|
172
|
-
return "cc"
|
|
173
|
-
|
|
174
|
-
|
|
175
122
|
@dataclass
|
|
176
123
|
class CryptoAtom(Atom):
|
|
177
124
|
"""
|
|
@@ -197,28 +144,6 @@ class CryptoAtom(Atom):
|
|
|
197
144
|
return "crypto"
|
|
198
145
|
|
|
199
146
|
|
|
200
|
-
@dataclass
|
|
201
|
-
class CveAtom(Atom):
|
|
202
|
-
"""
|
|
203
|
-
Allowed sighting key: ['cve_id']
|
|
204
|
-
"""
|
|
205
|
-
|
|
206
|
-
"""
|
|
207
|
-
published_at expect the following datetime format: '%Y-%m-%dT%H:%M:%SZ'
|
|
208
|
-
"""
|
|
209
|
-
cve_id: str
|
|
210
|
-
cvss: int = None
|
|
211
|
-
cwe: str = None
|
|
212
|
-
external_analysis_link: List[str] = None
|
|
213
|
-
published_at: str = None
|
|
214
|
-
|
|
215
|
-
def _get_sightings_allowed_keys(self):
|
|
216
|
-
return ["cve_id"]
|
|
217
|
-
|
|
218
|
-
def _get_sightings_prefix(self):
|
|
219
|
-
return "cve"
|
|
220
|
-
|
|
221
|
-
|
|
222
147
|
@dataclass
|
|
223
148
|
class Jarm:
|
|
224
149
|
calculated_at: str = None
|
|
@@ -237,6 +162,8 @@ class DomainAtom(Atom):
|
|
|
237
162
|
external_analysis_link: List[str] = None
|
|
238
163
|
malware_family: str = None
|
|
239
164
|
jarm: Jarm = None
|
|
165
|
+
port: List[int] = None
|
|
166
|
+
ns: List[str] = None
|
|
240
167
|
|
|
241
168
|
def _get_sightings_allowed_keys(self):
|
|
242
169
|
return ["domain"]
|
|
@@ -267,47 +194,6 @@ class EmailAtom(Atom):
|
|
|
267
194
|
return "email"
|
|
268
195
|
|
|
269
196
|
|
|
270
|
-
@dataclass
|
|
271
|
-
class FqdnAtom(Atom):
|
|
272
|
-
"""
|
|
273
|
-
Allowed sighting key: ['fqdn']
|
|
274
|
-
"""
|
|
275
|
-
|
|
276
|
-
fqdn: str
|
|
277
|
-
jarm: Jarm = None
|
|
278
|
-
malware_family: str = None
|
|
279
|
-
port: List[int] = None
|
|
280
|
-
ns: List[str] = None
|
|
281
|
-
external_analysis_link: List[str] = None
|
|
282
|
-
|
|
283
|
-
def _get_sightings_allowed_keys(self):
|
|
284
|
-
return ["fqdn"]
|
|
285
|
-
|
|
286
|
-
def _get_sightings_prefix(self):
|
|
287
|
-
return "fqdn"
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
@dataclass
|
|
291
|
-
class IbanAtom(Atom):
|
|
292
|
-
"""
|
|
293
|
-
Allowed sighting key: ['iban']
|
|
294
|
-
"""
|
|
295
|
-
|
|
296
|
-
iban: str
|
|
297
|
-
holder_name: str = None
|
|
298
|
-
holder_address: str = None
|
|
299
|
-
external_analysis_link: List[str] = None
|
|
300
|
-
bic: str = None
|
|
301
|
-
bank_name: str = None
|
|
302
|
-
bank_address: str = None
|
|
303
|
-
|
|
304
|
-
def _get_sightings_allowed_keys(self):
|
|
305
|
-
return ["iban"]
|
|
306
|
-
|
|
307
|
-
def _get_sightings_prefix(self):
|
|
308
|
-
return "iban"
|
|
309
|
-
|
|
310
|
-
|
|
311
197
|
@dataclass
|
|
312
198
|
class IpService:
|
|
313
199
|
port: int
|
|
@@ -360,67 +246,29 @@ class IpRangeAtom(Atom):
|
|
|
360
246
|
return "ip_range"
|
|
361
247
|
|
|
362
248
|
|
|
363
|
-
@dataclass
|
|
364
|
-
class PasteAtom(Atom):
|
|
365
|
-
"""
|
|
366
|
-
Allowed sighting key: ['url']
|
|
367
|
-
"""
|
|
368
|
-
|
|
369
|
-
url: str
|
|
370
|
-
author: str = None
|
|
371
|
-
title: str = None
|
|
372
|
-
content: str = None
|
|
373
|
-
external_analysis_link: List[str] = None
|
|
374
|
-
|
|
375
|
-
def _get_sightings_allowed_keys(self):
|
|
376
|
-
return ["url"]
|
|
377
|
-
|
|
378
|
-
def _get_sightings_prefix(self):
|
|
379
|
-
return "paste"
|
|
380
|
-
|
|
381
|
-
|
|
382
249
|
@dataclass
|
|
383
250
|
class PhoneNumberAtom(Atom):
|
|
384
251
|
"""
|
|
385
|
-
Allowed sighting key:
|
|
252
|
+
Allowed sighting key: 'international_phone_number'
|
|
386
253
|
"""
|
|
387
254
|
|
|
388
255
|
"""
|
|
389
256
|
country: minLength = 2 maxLength = 2
|
|
390
257
|
"""
|
|
258
|
+
international_phone_number: str
|
|
391
259
|
company: str = None
|
|
392
260
|
country: str = None
|
|
393
261
|
external_analysis_link: List[str] = None
|
|
394
|
-
international_phone_number: str = None
|
|
395
|
-
national_phone_number: str = None
|
|
396
262
|
|
|
397
263
|
def _get_sightings_allowed_keys(self):
|
|
398
|
-
return ["international_phone_number"
|
|
264
|
+
return ["international_phone_number"]
|
|
399
265
|
|
|
400
266
|
def _get_sightings_prefix(self):
|
|
401
267
|
return "phone_number"
|
|
402
268
|
|
|
403
269
|
|
|
404
270
|
@dataclass
|
|
405
|
-
class
|
|
406
|
-
"""
|
|
407
|
-
Allowed sighting key: ['path']
|
|
408
|
-
"""
|
|
409
|
-
|
|
410
|
-
path: str
|
|
411
|
-
regkey_value: str = None
|
|
412
|
-
hive: str = None
|
|
413
|
-
external_analysis_link: List[str] = None
|
|
414
|
-
|
|
415
|
-
def _get_sightings_allowed_keys(self):
|
|
416
|
-
return ["path"]
|
|
417
|
-
|
|
418
|
-
def _get_sightings_prefix(self):
|
|
419
|
-
return "regkey"
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
@dataclass
|
|
423
|
-
class SslAtom(Atom):
|
|
271
|
+
class CertificateAtom(Atom):
|
|
424
272
|
"""
|
|
425
273
|
Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
|
|
426
274
|
"""
|
|
@@ -440,7 +288,7 @@ class SslAtom(Atom):
|
|
|
440
288
|
return ["hashes", "md5", "sha1", "sha256", "sha512"]
|
|
441
289
|
|
|
442
290
|
def _get_sightings_prefix(self):
|
|
443
|
-
return "
|
|
291
|
+
return "certificate"
|
|
444
292
|
|
|
445
293
|
|
|
446
294
|
@dataclass
|
|
@@ -16,12 +16,16 @@ class BulkSearchTaskState(Enum):
|
|
|
16
16
|
CANCELLED = "CANCELLED"
|
|
17
17
|
FAILED_ERROR = "FAILED_ERROR"
|
|
18
18
|
FAILED_TIMEOUT = "FAILED_TIMEOUT"
|
|
19
|
+
FAILED_QUOTA_EXCEEDED = "FAILED_QUOTA_EXCEEDED"
|
|
20
|
+
FAILED_TOO_MANY_RESULTS = "FAILED_TOO_MANY_RESULTS"
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
BULK_SEARCH_FAILED_STATE = {
|
|
22
24
|
BulkSearchTaskState.CANCELLED,
|
|
23
25
|
BulkSearchTaskState.FAILED_TIMEOUT,
|
|
24
26
|
BulkSearchTaskState.FAILED_ERROR,
|
|
27
|
+
BulkSearchTaskState.FAILED_QUOTA_EXCEEDED,
|
|
28
|
+
BulkSearchTaskState.FAILED_TOO_MANY_RESULTS,
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
|
|
@@ -98,11 +102,11 @@ class BulkSearchTask:
|
|
|
98
102
|
Note that the requests library is used which is blocking.
|
|
99
103
|
timeout parameter is in seconds.
|
|
100
104
|
"""
|
|
101
|
-
start = datetime.datetime.
|
|
105
|
+
start = datetime.datetime.now(datetime.timezone.utc)
|
|
102
106
|
while self.state != BulkSearchTaskState.DONE:
|
|
103
107
|
if self.state in BULK_SEARCH_FAILED_STATE:
|
|
104
108
|
raise BulkSearchFailedError(self.state)
|
|
105
|
-
time_passed = datetime.datetime.
|
|
109
|
+
time_passed = datetime.datetime.now(datetime.timezone.utc) - start
|
|
106
110
|
if time_passed.total_seconds() > timeout:
|
|
107
111
|
raise TimeoutError()
|
|
108
112
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Token manager will manage tokens for the scripts.
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
import json
|
|
5
6
|
import os
|
|
6
7
|
from getpass import getpass
|
|
@@ -8,7 +9,6 @@ from urllib.parse import urljoin
|
|
|
8
9
|
|
|
9
10
|
import requests
|
|
10
11
|
|
|
11
|
-
from datalake.common.logger import logger
|
|
12
12
|
from datalake.common.utils import get_error_message
|
|
13
13
|
|
|
14
14
|
|
|
@@ -21,6 +21,7 @@ class TokenManager:
|
|
|
21
21
|
self,
|
|
22
22
|
endpoint_config: dict,
|
|
23
23
|
*,
|
|
24
|
+
logger,
|
|
24
25
|
environment: str,
|
|
25
26
|
username=None,
|
|
26
27
|
password=None,
|
|
@@ -36,9 +37,9 @@ class TokenManager:
|
|
|
36
37
|
|
|
37
38
|
self.url_token = urljoin(base_url, endpoints["token"], allow_fragments=True)
|
|
38
39
|
self.url_refresh = urljoin(
|
|
39
|
-
base_url, endpoints["
|
|
40
|
+
base_url, endpoints["refresh-token"], allow_fragments=True
|
|
40
41
|
)
|
|
41
|
-
|
|
42
|
+
self.logger = logger
|
|
42
43
|
self.username = username
|
|
43
44
|
self.password = password
|
|
44
45
|
self.longterm_token = longterm_token
|
|
@@ -61,6 +62,7 @@ class TokenManager:
|
|
|
61
62
|
self.longterm_token = self.longterm_token or os.getenv(
|
|
62
63
|
"OCD_DTL_LONGTERM_TOKEN"
|
|
63
64
|
)
|
|
65
|
+
self.logger.debug("Using Long Term Token")
|
|
64
66
|
|
|
65
67
|
self.username = self.username or os.getenv("OCD_DTL_USERNAME")
|
|
66
68
|
self.password = self.password or os.getenv("OCD_DTL_PASSWORD")
|
|
@@ -79,7 +81,7 @@ class TokenManager:
|
|
|
79
81
|
if self.longterm_token:
|
|
80
82
|
self.longterm_token = f"Token {self.longterm_token}"
|
|
81
83
|
if self.username or self.password:
|
|
82
|
-
logger.warning(
|
|
84
|
+
self.logger.warning(
|
|
83
85
|
f"Using provided Long Term Token for Authentication to the Datalake API. Ignoring username and/or password."
|
|
84
86
|
)
|
|
85
87
|
else:
|
|
@@ -93,14 +95,14 @@ class TokenManager:
|
|
|
93
95
|
self.access_token = f'Token {json_response["access_token"]}'
|
|
94
96
|
self.refresh_token = f'Token {json_response["refresh_token"]}'
|
|
95
97
|
except KeyError:
|
|
96
|
-
logger.error(
|
|
98
|
+
self.logger.error(
|
|
97
99
|
f"An error occurred while retrieving an access token, for URL: {self.url_token}\n"
|
|
98
100
|
f"response of the API: {response.text}"
|
|
99
101
|
)
|
|
100
102
|
raise ValueError(f"Could not login: {response.text}")
|
|
101
103
|
|
|
102
104
|
def fetch_new_token(self):
|
|
103
|
-
logger.debug("Token will be refreshed")
|
|
105
|
+
self.logger.debug("Token will be refreshed")
|
|
104
106
|
headers = {"Authorization": self.refresh_token}
|
|
105
107
|
response = requests.post(
|
|
106
108
|
url=self.url_refresh,
|
|
@@ -112,15 +114,15 @@ class TokenManager:
|
|
|
112
114
|
json_response = response.json()
|
|
113
115
|
if (
|
|
114
116
|
response.status_code == 401
|
|
115
|
-
and json_response
|
|
117
|
+
and get_error_message(json_response) == "Token has expired"
|
|
116
118
|
):
|
|
117
|
-
logger.info("Refreshing the refresh token")
|
|
119
|
+
self.logger.info("Refreshing the refresh token")
|
|
118
120
|
# Refresh token is also expired, we need to restart the authentication from scratch
|
|
119
121
|
self.get_token()
|
|
120
122
|
elif "access_token" in json_response:
|
|
121
123
|
self.access_token = f'Token {json_response["access_token"]}'
|
|
122
124
|
else: # an error occurred
|
|
123
|
-
logger.error(
|
|
125
|
+
self.logger.error(
|
|
124
126
|
f"An error occurred while refreshing the refresh token, for URL: {self.url_refresh}\n"
|
|
125
127
|
f"response of the API: {response.text}"
|
|
126
128
|
)
|