datalake-scripts 2.7.4__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/PKG-INFO +49 -28
  2. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/README.md +34 -22
  3. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/__init__.py +4 -9
  4. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/atom.py +2 -9
  5. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/atom_type.py +13 -165
  6. {datalake-scripts-2.7.4/datalake/api_objects → datalake_scripts-3.0.0/datalake/common}/bulk_search_task.py +6 -2
  7. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/logger.py +4 -0
  8. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/token_manager.py +11 -9
  9. datalake_scripts-3.0.0/datalake/config/endpoints.json +35 -0
  10. datalake_scripts-3.0.0/datalake/datalake.py +139 -0
  11. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/advanced_search.py +7 -4
  12. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/bulk_search.py +12 -3
  13. datalake_scripts-3.0.0/datalake/endpoints/comments.py +48 -0
  14. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/endpoint.py +74 -47
  15. datalake-scripts-2.7.4/datalake/endpoints/filtered_tag_subcategory.py → datalake_scripts-3.0.0/datalake/endpoints/filtered_threat_entity.py +23 -20
  16. datalake_scripts-3.0.0/datalake/endpoints/my_account.py +17 -0
  17. datalake_scripts-3.0.0/datalake/endpoints/sightings.py +339 -0
  18. datalake_scripts-3.0.0/datalake/endpoints/sources.py +33 -0
  19. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/tags.py +1 -1
  20. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/threats.py +114 -42
  21. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/miscellaneous/search_watch.py +21 -17
  22. datalake_scripts-3.0.0/datalake_scripts/__init__.py +1 -0
  23. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/cli.py +16 -10
  24. datalake_scripts-3.0.0/datalake_scripts/common/base_script.py +79 -0
  25. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/helper_scripts/utils.py +8 -2
  26. datalake_scripts-3.0.0/datalake_scripts/scripts/add_comments.py +73 -0
  27. datalake_scripts-3.0.0/datalake_scripts/scripts/add_tags.py +80 -0
  28. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/add_threats.py +134 -149
  29. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/advanced_search.py +13 -9
  30. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/bulk_lookup_threats.py +102 -97
  31. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/edit_score.py +28 -47
  32. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_atom_values.py +24 -35
  33. datalake_scripts-3.0.0/datalake_scripts/scripts/get_filtered_threat_entity.py +67 -0
  34. datalake_scripts-3.0.0/datalake_scripts/scripts/get_my_user_info.py +32 -0
  35. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_query_hash.py +8 -8
  36. datalake_scripts-3.0.0/datalake_scripts/scripts/get_threats.py +58 -0
  37. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/get_threats_from_query_hash.py +21 -26
  38. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/lookup_threats.py +22 -22
  39. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/scripts/search_watch.py +13 -14
  40. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/PKG-INFO +49 -28
  41. datalake_scripts-3.0.0/datalake_scripts.egg-info/SOURCES.txt +152 -0
  42. datalake_scripts-3.0.0/datalake_scripts.egg-info/entry_points.txt +2 -0
  43. datalake_scripts-3.0.0/pyproject.toml +3 -0
  44. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/setup.cfg +3 -5
  45. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/setup.py +18 -15
  46. datalake_scripts-3.0.0/tests/common/fixture.py +96 -0
  47. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/conftest.py +3 -3
  48. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_advanced_search.py +24 -11
  49. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_auth_via_tokens.py +72 -33
  50. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_bulk_search.py +96 -22
  51. datalake-scripts-2.7.4/tests/lib/test_filtered_tag_subcategory.py → datalake_scripts-3.0.0/tests/lib/test_filtered_threat_entity.py +23 -17
  52. datalake_scripts-3.0.0/tests/lib/test_my_account.py +51 -0
  53. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_search_watch.py +7 -3
  54. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_sightings.py +291 -31
  55. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_tags.py +37 -29
  56. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_threats.py +73 -39
  57. datalake-scripts-2.7.4/tests/scripts/test_base_engine.py → datalake_scripts-3.0.0/tests/scripts/test_base_scripts.py +4 -4
  58. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/scripts/test_cli.py +6 -7
  59. datalake-scripts-2.7.4/datalake/datalake.py +0 -127
  60. datalake-scripts-2.7.4/datalake/endpoints/sightings.py +0 -210
  61. datalake-scripts-2.7.4/datalake_scripts/__init__.py +0 -5
  62. datalake-scripts-2.7.4/datalake_scripts/common/base_engine.py +0 -123
  63. datalake-scripts-2.7.4/datalake_scripts/common/base_script.py +0 -83
  64. datalake-scripts-2.7.4/datalake_scripts/common/mixins.py +0 -86
  65. datalake-scripts-2.7.4/datalake_scripts/engines/get_engine.py +0 -104
  66. datalake-scripts-2.7.4/datalake_scripts/engines/post_engine.py +0 -199
  67. datalake-scripts-2.7.4/datalake_scripts/scripts/add_comments.py +0 -77
  68. datalake-scripts-2.7.4/datalake_scripts/scripts/add_tags.py +0 -84
  69. datalake-scripts-2.7.4/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -65
  70. datalake-scripts-2.7.4/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -60
  71. datalake-scripts-2.7.4/datalake_scripts.egg-info/SOURCES.txt +0 -81
  72. datalake-scripts-2.7.4/datalake_scripts.egg-info/entry_points.txt +0 -9
  73. datalake-scripts-2.7.4/tests/common/fixture.py +0 -57
  74. datalake-scripts-2.7.4/tests/lib/__init__.py +0 -0
  75. datalake-scripts-2.7.4/tests/scripts/__init__.py +0 -0
  76. datalake-scripts-2.7.4/tests/scripts/test_endpoint_config.py +0 -26
  77. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/__init__.py +0 -0
  78. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/config.py +0 -0
  79. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/output.py +0 -0
  80. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/throttler.py +0 -0
  81. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/utils.py +0 -0
  82. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/common/warn.py +0 -0
  83. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake/endpoints/__init__.py +0 -0
  84. {datalake-scripts-2.7.4/datalake/api_objects → datalake_scripts-3.0.0/datalake/miscellaneous}/__init__.py +0 -0
  85. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/common/__init__.py +0 -0
  86. {datalake-scripts-2.7.4/datalake/miscellaneous → datalake_scripts-3.0.0/datalake_scripts/helper_scripts}/__init__.py +0 -0
  87. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  88. {datalake-scripts-2.7.4/datalake_scripts/engines → datalake_scripts-3.0.0/datalake_scripts/scripts}/__init__.py +0 -0
  89. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  90. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/requires.txt +0 -0
  91. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/datalake_scripts.egg-info/top_level.txt +0 -0
  92. {datalake-scripts-2.7.4/datalake_scripts/helper_scripts → datalake_scripts-3.0.0/tests}/__init__.py +0 -0
  93. {datalake-scripts-2.7.4/datalake_scripts/scripts → datalake_scripts-3.0.0/tests/common}/__init__.py +0 -0
  94. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/common/test_output.py +0 -0
  95. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/common/test_utils.py +0 -0
  96. {datalake-scripts-2.7.4/tests → datalake_scripts-3.0.0/tests/lib}/__init__.py +0 -0
  97. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/lib/test_atom_type.py +0 -0
  98. {datalake-scripts-2.7.4/tests/common → datalake_scripts-3.0.0/tests/scripts}/__init__.py +0 -0
  99. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/scripts/test_add_new_threats.py +0 -0
  100. {datalake-scripts-2.7.4 → datalake_scripts-3.0.0}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
2
- Name: datalake-scripts
3
- Version: 2.7.4
1
+ Metadata-Version: 2.4
2
+ Name: datalake_scripts
3
+ Version: 3.0.0
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -27,11 +37,12 @@ Description-Content-Type: text/markdown
27
37
  |_|
28
38
 
29
39
 
30
- # datalake
31
- Datalake scripts
32
-
33
40
  ## How to use
34
41
 
42
+ Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
43
+
44
+ You can use this repository either as a library or as a CLI
45
+
35
46
  ### Installation
36
47
 
37
48
  With Python 3.6+:
@@ -40,6 +51,11 @@ $ pip install datalake-scripts
40
51
  $ pip3 install datalake-scripts
41
52
  ```
42
53
  ### Using as a library
54
+ The library requires to first create a Datalake instance and then to use the defined Classes' methods
55
+
56
+ The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
57
+
58
+ Example :
43
59
  ```python
44
60
  from datalake import Datalake, AtomType, Output
45
61
 
@@ -52,7 +68,8 @@ dtl.Threats.lookup(
52
68
  )
53
69
  ```
54
70
 
55
- see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
71
+
72
+
56
73
 
57
74
  ### Using as a CLI
58
75
 
@@ -64,9 +81,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
64
81
 
65
82
  You can also use a script directly by using the following command: `<script_name> <script_options>`.
66
83
 
67
- > /!\ Make sure to use utf-8 **without BOM** when providing a file (-i option)
84
+ > /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
85
+
86
+ #### Cli parameters
87
+
88
+ Common parameters for all commands:
89
+ * `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
90
+ * `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
91
+ * `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
92
+ * `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
68
93
 
69
- ## Environment variables
94
+ Commands can also have additionary mandatory or optional parameters
95
+
96
+ For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
97
+
98
+
99
+ ### Environment variables
70
100
 
71
101
  #### Authentication
72
102
 
@@ -76,7 +106,7 @@ There are two methods of authentication:
76
106
 
77
107
  In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
78
108
  * `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
79
- Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
109
+ Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
80
110
 
81
111
  or
82
112
 
@@ -95,25 +125,16 @@ We use the format accepted by the requests python library.
95
125
  See its documenation for other possible kinds of proxy to set up.
96
126
 
97
127
 
98
- #### Throttling
99
- For throttling the request, those two environment variable can be used:
100
- * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
101
- * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` define the number of request to do at maximum for the given time, *default is 5 queries*.
102
-
103
- > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/) for each endpoint
128
+ #### Throttling and retries
129
+ For throttling the requests, those two environment variables can be used:
130
+ * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
131
+ * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
132
+ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
104
133
 
105
- ## Cli parameters
106
-
107
- Parameters common and optional for all commands:
108
- > --debug display more information for debugging purposes
109
- > -e to change the environment {preprod, prod}, default is **prod**
110
- > -o will set the output file as the API gives it.
111
- > -q will quiet the verbosity of the program (but still show errors / warnings)
112
-
113
- For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
134
+ > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
114
135
 
136
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
115
137
 
116
138
  ### Contributing
117
139
 
118
140
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
119
-
@@ -7,11 +7,12 @@
7
7
  |_|
8
8
 
9
9
 
10
- # datalake
11
- Datalake scripts
12
-
13
10
  ## How to use
14
11
 
12
+ Datalake scripts is developed by Datalake developers to help use the Datalake [API](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
13
+
14
+ You can use this repository either as a library or as a CLI
15
+
15
16
  ### Installation
16
17
 
17
18
  With Python 3.6+:
@@ -20,6 +21,11 @@ $ pip install datalake-scripts
20
21
  $ pip3 install datalake-scripts
21
22
  ```
22
23
  ### Using as a library
24
+ The library requires to first create a Datalake instance and then to use the defined Classes' methods
25
+
26
+ The library tutorial is available in [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
27
+
28
+ Example :
23
29
  ```python
24
30
  from datalake import Datalake, AtomType, Output
25
31
 
@@ -32,7 +38,8 @@ dtl.Threats.lookup(
32
38
  )
33
39
  ```
34
40
 
35
- see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
41
+
42
+
36
43
 
37
44
  ### Using as a CLI
38
45
 
@@ -44,9 +51,22 @@ Check `ocd-dtl -h` for help, including the list of commands available.
44
51
 
45
52
  You can also use a script directly by using the following command: `<script_name> <script_options>`.
46
53
 
47
- > /!\ Make sure to use utf-8 **without BOM** when providing a file (-i option)
54
+ > /!\ Make sure to use utf-8 **without BOM** when providing a file as input (`-i, --input` parameter)
55
+
56
+ #### Cli parameters
57
+
58
+ Common parameters for all commands:
59
+ * `-e, --env <preprod|prod>` : Datalake environment. Default is **prod**
60
+ * `-o, --output <OUTPUT_PATH>` : will set the output file as the API gives it. No default
61
+ * `-D, --debug` : will raise the verbosity of the program (by displaying additional DEBUG messages). Default log level is INFO
62
+ * `-q, --quiet` : will quiet the verbosity of the program (but will still show ERROR / WARNING messages). Default log level is INFO
48
63
 
49
- ## Environment variables
64
+ Commands can also have additionary mandatory or optional parameters
65
+
66
+ For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
67
+
68
+
69
+ ### Environment variables
50
70
 
51
71
  #### Authentication
52
72
 
@@ -56,7 +76,7 @@ There are two methods of authentication:
56
76
 
57
77
  In case you don't want to enter credentials for each commands and you are on a secured terminal, set those variables:
58
78
  * `OCD_DTL_LONGTERM_TOKEN` a long term token associated to your Datalake account.
59
- Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/)
79
+ Please note that if this variable is set, then the long term token will be used for every request to the Datalake API, even if you set the username and passsword environment variables below. This is important because some endpoints / requests do not accept long term tokens but need fresh tokens (ie a Datalake instance with username and password). Check for the need of fresh tokens in each endpoint description [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/)
60
80
 
61
81
  or
62
82
 
@@ -75,23 +95,15 @@ We use the format accepted by the requests python library.
75
95
  See its documenation for other possible kinds of proxy to set up.
76
96
 
77
97
 
78
- #### Throttling
79
- For throttling the request, those two environment variable can be used:
80
- * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
81
- * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` define the number of request to do at maximum for the given time, *default is 5 queries*.
98
+ #### Throttling and retries
99
+ For throttling the requests, those two environment variables can be used:
100
+ * `OCD_DTL_QUOTA_TIME` defines, in seconds, the time before resetting the requests limit, *default is 1 second*.
101
+ * `OCD_DTL_REQUESTS_PER_QUOTA_TIME` defines the number of request to do at maximum for the given time, *default is 5 queries*.
102
+ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encounter too many 429 errors.
82
103
 
83
- > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v2/docs/) for each endpoint
84
-
85
- ## Cli parameters
86
-
87
- Parameters common and optional for all commands:
88
- > --debug display more information for debugging purposes
89
- > -e to change the environment {preprod, prod}, default is **prod**
90
- > -o will set the output file as the API gives it.
91
- > -q will quiet the verbosity of the program (but still show errors / warnings)
92
-
93
- For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
104
+ > Please don't exceed the quota marked [here](https://datalake.cert.orangecyberdefense.com/api/v3/docs/) for each endpoint
94
105
 
106
+ Only network errors and HTTP response code 429, 500, 502, 503 and 504 trigger retries. You may control the number of retries using the environment variable `OCD_DTL_MAX_RETRIES`, which defaults to 3.
95
107
 
96
108
  ### Contributing
97
109
 
@@ -1,9 +1,10 @@
1
- from .api_objects.bulk_search_task import (
1
+ from .common.bulk_search_task import (
2
2
  BulkSearchTask,
3
3
  BulkSearchTaskState,
4
4
  BulkSearchFailedError,
5
5
  BulkSearchNotFound,
6
6
  )
7
+
7
8
  from .common.atom import (
8
9
  AtomType,
9
10
  ThreatType,
@@ -12,29 +13,23 @@ from .common.atom import (
12
13
  Visibility,
13
14
  SightingRelation,
14
15
  )
16
+
15
17
  from .common.atom_type import (
16
18
  Atom,
17
19
  Hashes,
18
20
  FileAtom,
19
21
  AndroidApp,
20
- ApkAtom,
21
22
  AsAtom,
22
- CcAtom,
23
23
  CryptoAtom,
24
- CveAtom,
25
24
  Jarm,
26
25
  DomainAtom,
27
26
  EmailFlow,
28
27
  EmailAtom,
29
- FqdnAtom,
30
- IbanAtom,
31
28
  IpService,
32
29
  IpAtom,
33
30
  IpRangeAtom,
34
- PasteAtom,
35
31
  PhoneNumberAtom,
36
- RegKeyAtom,
37
- SslAtom,
32
+ CertificateAtom,
38
33
  UrlAtom,
39
34
  )
40
35
  from .common.output import Output
@@ -1,27 +1,21 @@
1
1
  """
2
2
  Enums for atom properties
3
3
  """
4
+
4
5
  from enum import Enum
5
6
  from typing import Dict, Union
6
7
 
7
8
 
8
9
  class AtomType(Enum):
9
- APK = "apk"
10
10
  AS = "as"
11
- CC = "cc"
12
11
  CRYPTO = "crypto"
13
- CVE = "cve"
14
12
  DOMAIN = "domain"
15
13
  EMAIL = "email"
16
14
  FILE = "file"
17
- FQDN = "fqdn"
18
- IBAN = "iban"
19
15
  IP = "ip"
20
16
  IP_RANGE = "ip_range"
21
- PASTE = "paste"
22
17
  PHONE_NUMBER = "phone_number"
23
- REGKEY = "regkey"
24
- SSL = "ssl"
18
+ CERTIFICATE = "certificate"
25
19
  URL = "url"
26
20
 
27
21
 
@@ -48,7 +42,6 @@ threat_type -> ThreatType
48
42
 
49
43
 
50
44
  class OverrideType(Enum):
51
- PERMANENT = "permanent"
52
45
  TEMPORARY = "temporary"
53
46
  LOCK = "lock"
54
47
 
@@ -66,28 +66,6 @@ class Hashes:
66
66
  ssdeep: str = None
67
67
 
68
68
 
69
- @dataclass
70
- class FileAtom(Atom):
71
- """
72
- Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
73
- """
74
-
75
- hashes: Hashes
76
- external_analysis_link: List[str] = None
77
- filesize: int = None
78
- filetype: str = None
79
- file_url: str = None
80
- mimetype: str = None
81
- filename: str = None
82
- filepath: str = None
83
-
84
- def _get_sightings_allowed_keys(self):
85
- return ["hashes", "md5", "sha1", "sha256", "sha512"]
86
-
87
- def _get_sightings_prefix(self):
88
- return "file"
89
-
90
-
91
69
  @dataclass
92
70
  class AndroidApp:
93
71
  package_name: str
@@ -98,35 +76,26 @@ class AndroidApp:
98
76
 
99
77
 
100
78
  @dataclass
101
- class ApkAtom(Atom):
79
+ class FileAtom(Atom):
102
80
  """
103
- Allowed sighting key: ['android', 'package_name', 'version_name', 'hashes', 'md5', 'sha1', 'sha256', 'sha512']
81
+ Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
104
82
  """
105
83
 
106
- android: AndroidApp
107
84
  hashes: Hashes
108
85
  external_analysis_link: List[str] = None
109
86
  filesize: int = None
110
87
  filetype: str = None
111
88
  file_url: str = None
112
89
  mimetype: str = None
113
- filename: str = None
90
+ filename: List[str] = None
114
91
  filepath: str = None
92
+ android: AndroidApp = None
115
93
 
116
94
  def _get_sightings_allowed_keys(self):
117
- return [
118
- "android",
119
- "package_name",
120
- "version_name",
121
- "hashes",
122
- "md5",
123
- "sha1",
124
- "sha256",
125
- "sha512",
126
- ]
95
+ return ["hashes", "md5", "sha1", "sha256", "sha512"]
127
96
 
128
97
  def _get_sightings_prefix(self):
129
- return "apk"
98
+ return "file"
130
99
 
131
100
 
132
101
  @dataclass
@@ -150,28 +119,6 @@ class AsAtom(Atom):
150
119
  return "as"
151
120
 
152
121
 
153
- @dataclass
154
- class CcAtom(Atom):
155
- """
156
- Allowed sighting key: ['number']
157
- number: minLength = 8 maxLength = 19
158
- """
159
-
160
- number: str
161
- external_analysis_link: List[str] = None
162
- bin: int = None
163
- brand: str = None
164
- cvx: int = None
165
- description: str = None
166
- expiry_date: str = None
167
-
168
- def _get_sightings_allowed_keys(self):
169
- return ["number"]
170
-
171
- def _get_sightings_prefix(self):
172
- return "cc"
173
-
174
-
175
122
  @dataclass
176
123
  class CryptoAtom(Atom):
177
124
  """
@@ -197,28 +144,6 @@ class CryptoAtom(Atom):
197
144
  return "crypto"
198
145
 
199
146
 
200
- @dataclass
201
- class CveAtom(Atom):
202
- """
203
- Allowed sighting key: ['cve_id']
204
- """
205
-
206
- """
207
- published_at expect the following datetime format: '%Y-%m-%dT%H:%M:%SZ'
208
- """
209
- cve_id: str
210
- cvss: int = None
211
- cwe: str = None
212
- external_analysis_link: List[str] = None
213
- published_at: str = None
214
-
215
- def _get_sightings_allowed_keys(self):
216
- return ["cve_id"]
217
-
218
- def _get_sightings_prefix(self):
219
- return "cve"
220
-
221
-
222
147
  @dataclass
223
148
  class Jarm:
224
149
  calculated_at: str = None
@@ -237,6 +162,8 @@ class DomainAtom(Atom):
237
162
  external_analysis_link: List[str] = None
238
163
  malware_family: str = None
239
164
  jarm: Jarm = None
165
+ port: List[int] = None
166
+ ns: List[str] = None
240
167
 
241
168
  def _get_sightings_allowed_keys(self):
242
169
  return ["domain"]
@@ -267,47 +194,6 @@ class EmailAtom(Atom):
267
194
  return "email"
268
195
 
269
196
 
270
- @dataclass
271
- class FqdnAtom(Atom):
272
- """
273
- Allowed sighting key: ['fqdn']
274
- """
275
-
276
- fqdn: str
277
- jarm: Jarm = None
278
- malware_family: str = None
279
- port: List[int] = None
280
- ns: List[str] = None
281
- external_analysis_link: List[str] = None
282
-
283
- def _get_sightings_allowed_keys(self):
284
- return ["fqdn"]
285
-
286
- def _get_sightings_prefix(self):
287
- return "fqdn"
288
-
289
-
290
- @dataclass
291
- class IbanAtom(Atom):
292
- """
293
- Allowed sighting key: ['iban']
294
- """
295
-
296
- iban: str
297
- holder_name: str = None
298
- holder_address: str = None
299
- external_analysis_link: List[str] = None
300
- bic: str = None
301
- bank_name: str = None
302
- bank_address: str = None
303
-
304
- def _get_sightings_allowed_keys(self):
305
- return ["iban"]
306
-
307
- def _get_sightings_prefix(self):
308
- return "iban"
309
-
310
-
311
197
  @dataclass
312
198
  class IpService:
313
199
  port: int
@@ -360,67 +246,29 @@ class IpRangeAtom(Atom):
360
246
  return "ip_range"
361
247
 
362
248
 
363
- @dataclass
364
- class PasteAtom(Atom):
365
- """
366
- Allowed sighting key: ['url']
367
- """
368
-
369
- url: str
370
- author: str = None
371
- title: str = None
372
- content: str = None
373
- external_analysis_link: List[str] = None
374
-
375
- def _get_sightings_allowed_keys(self):
376
- return ["url"]
377
-
378
- def _get_sightings_prefix(self):
379
- return "paste"
380
-
381
-
382
249
  @dataclass
383
250
  class PhoneNumberAtom(Atom):
384
251
  """
385
- Allowed sighting key: ['international_phone_number', 'national_phone_number']
252
+ Allowed sighting key: 'international_phone_number'
386
253
  """
387
254
 
388
255
  """
389
256
  country: minLength = 2 maxLength = 2
390
257
  """
258
+ international_phone_number: str
391
259
  company: str = None
392
260
  country: str = None
393
261
  external_analysis_link: List[str] = None
394
- international_phone_number: str = None
395
- national_phone_number: str = None
396
262
 
397
263
  def _get_sightings_allowed_keys(self):
398
- return ["international_phone_number", "national_phone_number"]
264
+ return ["international_phone_number"]
399
265
 
400
266
  def _get_sightings_prefix(self):
401
267
  return "phone_number"
402
268
 
403
269
 
404
270
  @dataclass
405
- class RegKeyAtom(Atom):
406
- """
407
- Allowed sighting key: ['path']
408
- """
409
-
410
- path: str
411
- regkey_value: str = None
412
- hive: str = None
413
- external_analysis_link: List[str] = None
414
-
415
- def _get_sightings_allowed_keys(self):
416
- return ["path"]
417
-
418
- def _get_sightings_prefix(self):
419
- return "regkey"
420
-
421
-
422
- @dataclass
423
- class SslAtom(Atom):
271
+ class CertificateAtom(Atom):
424
272
  """
425
273
  Allowed sighting key: ['hashes', 'md5', 'sha1', 'sha256', 'sha512']
426
274
  """
@@ -440,7 +288,7 @@ class SslAtom(Atom):
440
288
  return ["hashes", "md5", "sha1", "sha256", "sha512"]
441
289
 
442
290
  def _get_sightings_prefix(self):
443
- return "ssl"
291
+ return "certificate"
444
292
 
445
293
 
446
294
  @dataclass
@@ -16,12 +16,16 @@ class BulkSearchTaskState(Enum):
16
16
  CANCELLED = "CANCELLED"
17
17
  FAILED_ERROR = "FAILED_ERROR"
18
18
  FAILED_TIMEOUT = "FAILED_TIMEOUT"
19
+ FAILED_QUOTA_EXCEEDED = "FAILED_QUOTA_EXCEEDED"
20
+ FAILED_TOO_MANY_RESULTS = "FAILED_TOO_MANY_RESULTS"
19
21
 
20
22
 
21
23
  BULK_SEARCH_FAILED_STATE = {
22
24
  BulkSearchTaskState.CANCELLED,
23
25
  BulkSearchTaskState.FAILED_TIMEOUT,
24
26
  BulkSearchTaskState.FAILED_ERROR,
27
+ BulkSearchTaskState.FAILED_QUOTA_EXCEEDED,
28
+ BulkSearchTaskState.FAILED_TOO_MANY_RESULTS,
25
29
  }
26
30
 
27
31
 
@@ -98,11 +102,11 @@ class BulkSearchTask:
98
102
  Note that the requests library is used which is blocking.
99
103
  timeout parameter is in seconds.
100
104
  """
101
- start = datetime.datetime.utcnow()
105
+ start = datetime.datetime.now(datetime.timezone.utc)
102
106
  while self.state != BulkSearchTaskState.DONE:
103
107
  if self.state in BULK_SEARCH_FAILED_STATE:
104
108
  raise BulkSearchFailedError(self.state)
105
- time_passed = datetime.datetime.utcnow() - start
109
+ time_passed = datetime.datetime.now(datetime.timezone.utc) - start
106
110
  if time_passed.total_seconds() > timeout:
107
111
  raise TimeoutError()
108
112
 
@@ -16,3 +16,7 @@ def configure_logging(loglevel: int):
16
16
  handler.setFormatter(formatter)
17
17
  logger.addHandler(handler)
18
18
  logger.setLevel(loglevel)
19
+
20
+ logger.debug(f"Logging configured with level {str(loglevel)}")
21
+
22
+ return logger