datalake-scripts 3.0.0rc4__tar.gz → 3.0.0rc6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/PKG-INFO +14 -5
  2. datalake_scripts-3.0.0rc6/datalake/config/endpoints.json +35 -0
  3. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sightings.py +35 -2
  4. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/cli.py +1 -1
  5. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/PKG-INFO +15 -6
  6. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/SOURCES.txt +74 -1
  7. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/entry_points.txt +0 -1
  8. datalake_scripts-3.0.0rc6/pyproject.toml +3 -0
  9. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/setup.cfg +3 -5
  10. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/setup.py +17 -6
  11. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_sightings.py +240 -1
  12. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/README.md +0 -0
  13. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/__init__.py +0 -0
  14. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/__init__.py +0 -0
  15. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/atom.py +0 -0
  16. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/atom_type.py +0 -0
  17. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/bulk_search_task.py +0 -0
  18. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/config.py +0 -0
  19. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/logger.py +0 -0
  20. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/output.py +0 -0
  21. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/throttler.py +0 -0
  22. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/token_manager.py +0 -0
  23. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/utils.py +0 -0
  24. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/common/warn.py +0 -0
  25. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/datalake.py +0 -0
  26. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/__init__.py +0 -0
  27. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/advanced_search.py +0 -0
  28. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/bulk_search.py +0 -0
  29. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/comments.py +0 -0
  30. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/endpoint.py +0 -0
  31. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/filtered_threat_entity.py +0 -0
  32. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/my_account.py +0 -0
  33. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sources.py +0 -0
  34. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/tags.py +0 -0
  35. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/endpoints/threats.py +0 -0
  36. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/__init__.py +0 -0
  37. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/search_watch.py +0 -0
  38. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/__init__.py +0 -0
  39. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/__init__.py +0 -0
  40. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/base_script.py +0 -0
  41. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/__init__.py +0 -0
  42. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  43. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/utils.py +0 -0
  44. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/__init__.py +0 -0
  45. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_comments.py +0 -0
  46. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_tags.py +0 -0
  47. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_threats.py +0 -0
  48. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/advanced_search.py +0 -0
  49. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  50. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/edit_score.py +0 -0
  51. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_atom_values.py +0 -0
  52. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
  53. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_my_user_info.py +0 -0
  54. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_query_hash.py +0 -0
  55. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats.py +0 -0
  56. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  57. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/lookup_threats.py +0 -0
  58. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/search_watch.py +0 -0
  59. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  60. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/requires.txt +0 -0
  61. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/top_level.txt +0 -0
  62. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/__init__.py +0 -0
  63. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/common/__init__.py +0 -0
  64. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/common/fixture.py +0 -0
  65. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/common/test_output.py +0 -0
  66. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/common/test_utils.py +0 -0
  67. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/conftest.py +0 -0
  68. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/__init__.py +0 -0
  69. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_advanced_search.py +0 -0
  70. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_atom_type.py +0 -0
  71. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_auth_via_tokens.py +0 -0
  72. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_bulk_search.py +0 -0
  73. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_filtered_threat_entity.py +0 -0
  74. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_my_account.py +0 -0
  75. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_search_watch.py +0 -0
  76. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_tags.py +0 -0
  77. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/lib/test_threats.py +0 -0
  78. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/scripts/__init__.py +0 -0
  79. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/scripts/test_add_new_threats.py +0 -0
  80. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/scripts/test_base_scripts.py +0 -0
  81. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/scripts/test_cli.py +0 -0
  82. {datalake_scripts-3.0.0rc4 → datalake_scripts-3.0.0rc6}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc4
3
+ Version: 3.0.0rc6
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
127
137
  ### Contributing
128
138
 
129
139
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -0,0 +1,35 @@
1
+ {
2
+ "main": {
3
+ "preprod": "https://ti2.extranet.mrti-center.com/api/",
4
+ "prod": "https://datalake.cert.orangecyberdefense.com/api/"
5
+ },
6
+ "endpoints": {
7
+ "token": "auth/token/",
8
+ "refresh-token": "auth/refresh-token/",
9
+ "advanced-queries-threats": "mrti/advanced-queries/threats/",
10
+ "advanced-queries-threats-query-hash": "mrti/advanced-queries/threats/{query_hash}/",
11
+ "atom-values": "mrti/atom-values/",
12
+ "bulk-search": "mrti/bulk-search/",
13
+ "bulk-search-tasks": "mrti/bulk-search/tasks/",
14
+ "bulk-search-task": "mrti/bulk-search/task/{task_uuid}/",
15
+ "bulk-manual-threats": "mrti/bulk-manual-threats/",
16
+ "bulk-manual-threats-task": "mrti/bulk-manual-threats/task/{task_uuid}/",
17
+ "sources": "mrti/sources/",
18
+ "threat-entity-filtered":"mrti/threat-entity/filtered/",
19
+ "threats-manual": "mrti/threats-manual/",
20
+ "threats-bulk-lookup": "mrti/threats/bulk-lookup/",
21
+ "threats": "mrti/threats/{hashkey}/",
22
+ "threats-lookup": "mrti/threats/lookup/",
23
+ "threats-comments": "mrti/threats/{hashkey}/comments/",
24
+ "threats-tags": "mrti/threats/{hashkey}/tags/",
25
+ "threats-atom-values-extract": "mrti/threats/atom-values-extract/",
26
+ "threats-bulk-scoring-edits": "mrti/threats/bulk-scoring-edits/",
27
+ "threats-sighting": "mrti/threats/sighting/",
28
+ "threats-bulk-sighting": "mrti/threats/bulk-sighting/",
29
+ "threats-sighting-file": "mrti/threats/sighting-file/{hashkey}/",
30
+ "threats-sighting-filtered": "mrti/threats/sighting/filtered/",
31
+ "users-me": "users/me/"
32
+ },
33
+ "threats-manual-bulk-size": 100,
34
+ "api_version": "v3/"
35
+ }
@@ -46,9 +46,9 @@ class Sightings(Endpoint):
46
46
  impersonate_id: int = None,
47
47
  ):
48
48
  """
49
- Submit a list of sightings.
49
+ Submit a list of atoms as 1 sighting.
50
50
  Either threat "hashkeys" or list of atom objects in "atoms" is required.
51
- Possible sightings type: POSITIVE, NEGATIVE, NEUTRAL.
51
+ Possible sighting's type: POSITIVE, NEGATIVE, NEUTRAL.
52
52
  For POSITIVE and NEGATIVE sightings, "threat_types"
53
53
  field is required.
54
54
  End date timestamp should always be in the past.
@@ -73,6 +73,39 @@ class Sightings(Endpoint):
73
73
  res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
74
74
  return res
75
75
 
76
+ def bulk_submit_sightings(
77
+ self,
78
+ sightings: List[dict] = [],
79
+ ):
80
+ """
81
+ Submit a list of Sightings.
82
+ Input must be a list of valid dict.
83
+ Each dict is a sighting.
84
+ """
85
+ payload = {}
86
+ list_sightings_for_url_call = []
87
+ for sighting in sightings:
88
+ payload_item = self._prepare_sightings_payload(
89
+ sighting.get("atoms"),
90
+ sighting.get("hashkeys"),
91
+ sighting.get("start_timestamp"),
92
+ sighting.get("end_timestamp"),
93
+ sighting.get("sighting_type"),
94
+ sighting.get("description_visibility"),
95
+ sighting.get("count"),
96
+ sighting.get("threat_types"),
97
+ sighting.get("tags"),
98
+ sighting.get("description"),
99
+ sighting.get("relation"),
100
+ sighting.get("editable"),
101
+ sighting.get("impersonate_id"),
102
+ )
103
+ list_sightings_for_url_call.append(payload_item)
104
+ payload["data"] = list_sightings_for_url_call
105
+ url = self._build_url_for_endpoint("threats-bulk-sighting")
106
+ res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
107
+ return res
108
+
76
109
  @staticmethod
77
110
  def _check_sightings_payload_parameters(
78
111
  atoms,
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
21
21
 
22
22
  class Cli:
23
23
  CLI_NAME = "ocd-dtl"
24
- VERSION = "3.0.0rc4"
24
+ VERSION = "3.0.0rc6"
25
25
 
26
26
  def __init__(self):
27
27
  parser = argparse.ArgumentParser(
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
2
- Name: datalake-scripts
3
- Version: 3.0.0rc4
1
+ Metadata-Version: 2.4
2
+ Name: datalake_scripts
3
+ Version: 3.0.0rc6
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
127
137
  ### Contributing
128
138
 
129
139
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -1,4 +1,5 @@
1
1
  README.md
2
+ pyproject.toml
2
3
  setup.cfg
3
4
  setup.py
4
5
  ./datalake/__init__.py
@@ -14,6 +15,7 @@ setup.py
14
15
  ./datalake/common/token_manager.py
15
16
  ./datalake/common/utils.py
16
17
  ./datalake/common/warn.py
18
+ ./datalake/config/endpoints.json
17
19
  ./datalake/endpoints/__init__.py
18
20
  ./datalake/endpoints/advanced_search.py
19
21
  ./datalake/endpoints/bulk_search.py
@@ -71,9 +73,80 @@ setup.py
71
73
  ./tests/scripts/test_base_scripts.py
72
74
  ./tests/scripts/test_cli.py
73
75
  ./tests/scripts/test_csv_builder.py
76
+ datalake/__init__.py
77
+ datalake/datalake.py
78
+ datalake/common/__init__.py
79
+ datalake/common/atom.py
80
+ datalake/common/atom_type.py
81
+ datalake/common/bulk_search_task.py
82
+ datalake/common/config.py
83
+ datalake/common/logger.py
84
+ datalake/common/output.py
85
+ datalake/common/throttler.py
86
+ datalake/common/token_manager.py
87
+ datalake/common/utils.py
88
+ datalake/common/warn.py
89
+ datalake/config/endpoints.json
90
+ datalake/endpoints/__init__.py
91
+ datalake/endpoints/advanced_search.py
92
+ datalake/endpoints/bulk_search.py
93
+ datalake/endpoints/comments.py
94
+ datalake/endpoints/endpoint.py
95
+ datalake/endpoints/filtered_threat_entity.py
96
+ datalake/endpoints/my_account.py
97
+ datalake/endpoints/sightings.py
98
+ datalake/endpoints/sources.py
99
+ datalake/endpoints/tags.py
100
+ datalake/endpoints/threats.py
101
+ datalake/miscellaneous/__init__.py
102
+ datalake/miscellaneous/search_watch.py
103
+ datalake_scripts/__init__.py
104
+ datalake_scripts/cli.py
74
105
  datalake_scripts.egg-info/PKG-INFO
75
106
  datalake_scripts.egg-info/SOURCES.txt
76
107
  datalake_scripts.egg-info/dependency_links.txt
77
108
  datalake_scripts.egg-info/entry_points.txt
78
109
  datalake_scripts.egg-info/requires.txt
79
- datalake_scripts.egg-info/top_level.txt
110
+ datalake_scripts.egg-info/top_level.txt
111
+ datalake_scripts/common/__init__.py
112
+ datalake_scripts/common/base_script.py
113
+ datalake_scripts/helper_scripts/__init__.py
114
+ datalake_scripts/helper_scripts/output_builder.py
115
+ datalake_scripts/helper_scripts/utils.py
116
+ datalake_scripts/scripts/__init__.py
117
+ datalake_scripts/scripts/add_comments.py
118
+ datalake_scripts/scripts/add_tags.py
119
+ datalake_scripts/scripts/add_threats.py
120
+ datalake_scripts/scripts/advanced_search.py
121
+ datalake_scripts/scripts/bulk_lookup_threats.py
122
+ datalake_scripts/scripts/edit_score.py
123
+ datalake_scripts/scripts/get_atom_values.py
124
+ datalake_scripts/scripts/get_filtered_threat_entity.py
125
+ datalake_scripts/scripts/get_my_user_info.py
126
+ datalake_scripts/scripts/get_query_hash.py
127
+ datalake_scripts/scripts/get_threats.py
128
+ datalake_scripts/scripts/get_threats_from_query_hash.py
129
+ datalake_scripts/scripts/lookup_threats.py
130
+ datalake_scripts/scripts/search_watch.py
131
+ tests/__init__.py
132
+ tests/conftest.py
133
+ tests/common/__init__.py
134
+ tests/common/fixture.py
135
+ tests/common/test_output.py
136
+ tests/common/test_utils.py
137
+ tests/lib/__init__.py
138
+ tests/lib/test_advanced_search.py
139
+ tests/lib/test_atom_type.py
140
+ tests/lib/test_auth_via_tokens.py
141
+ tests/lib/test_bulk_search.py
142
+ tests/lib/test_filtered_threat_entity.py
143
+ tests/lib/test_my_account.py
144
+ tests/lib/test_search_watch.py
145
+ tests/lib/test_sightings.py
146
+ tests/lib/test_tags.py
147
+ tests/lib/test_threats.py
148
+ tests/scripts/__init__.py
149
+ tests/scripts/test_add_new_threats.py
150
+ tests/scripts/test_base_scripts.py
151
+ tests/scripts/test_cli.py
152
+ tests/scripts/test_csv_builder.py
@@ -1,3 +1,2 @@
1
1
  [console_scripts]
2
2
  ocd-dtl = datalake_scripts.cli:main
3
-
@@ -0,0 +1,3 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
@@ -2,12 +2,10 @@
2
2
  upgrade = true
3
3
 
4
4
  [egg_info]
5
- tag-date = false
6
- tag-build =
7
- tag_svn_revision = false
8
- tag_build =
9
5
  tag_date = 0
6
+ tag_build =
7
+ tag_svn_revision = false
10
8
 
11
9
  [metadata]
12
- description-file = README.md
10
+ description_file = README.md
13
11
 
@@ -2,20 +2,31 @@ import os
2
2
 
3
3
  from setuptools import find_packages, setup
4
4
 
5
- from datalake_scripts.cli import Cli
6
-
7
5
  here = os.path.abspath(os.path.dirname(__file__))
8
6
 
9
7
  with open("README.md", "r") as fh:
10
- long_description = fh.read()
8
+ LONG_DESCRIPTION = fh.read()
9
+
10
+
11
+ def read_version_or_cli_name(option: str):
12
+ with open("datalake_scripts/cli.py") as f:
13
+ for line in f:
14
+ if option == "version" and line.strip().startswith("VERSION"):
15
+ return line.split("=")[1].strip().strip('"')
16
+ if option == "cli_name" and line.strip().startswith("CLI_NAME"):
17
+ return line.split("=")[1].strip().strip('"')
18
+
19
+
20
+ CLI_NAME = read_version_or_cli_name("cli_name")
21
+ VERSION = read_version_or_cli_name("version")
11
22
 
12
23
  setup(
13
24
  name="datalake_scripts",
14
- version=Cli.VERSION,
25
+ version=VERSION,
15
26
  author="OCD",
16
27
  author_email="cert-contact.ocd@orange.com",
17
28
  description="A collection of scripts to easily use the API of OCD Datalake",
18
- long_description=long_description,
29
+ long_description=LONG_DESCRIPTION,
19
30
  long_description_content_type="text/markdown",
20
31
  url="https://github.com/cert-orangecyberdefense/datalake/",
21
32
  package_dir={
@@ -25,7 +36,7 @@ setup(
25
36
  install_requires=["requests", "halo", "prettytable"],
26
37
  test_suite="nose.collector",
27
38
  entry_points={
28
- "console_scripts": (f"{Cli.CLI_NAME} = datalake_scripts.cli:main",),
39
+ "console_scripts": (f"{CLI_NAME} = datalake_scripts.cli:main",),
29
40
  },
30
41
  classifiers=[
31
42
  "Programming Language :: Python :: 3.7",
@@ -1,16 +1,19 @@
1
1
  from unittest.mock import patch
2
2
  import pytest
3
3
  import responses
4
- from datetime import datetime
4
+ from datetime import datetime, timedelta, timezone
5
5
 
6
6
  from responses import matchers
7
7
  import os
8
+ import ast
8
9
 
9
10
 
10
11
  from tests.common.fixture import TestData, datalake # noqa needed fixture import
11
12
  from datalake import (
12
13
  IpAtom,
13
14
  FileAtom,
15
+ EmailAtom,
16
+ UrlAtom,
14
17
  Hashes,
15
18
  Jarm,
16
19
  IpService,
@@ -448,6 +451,242 @@ def test_submit_sightings_bad_atom(datalake):
448
451
  assert str(err.value) == '"atoms" needs to be a list of Atom subclasses.'
449
452
 
450
453
 
454
+ @responses.activate
455
+ def test_bulk_submit_sightings_success(datalake):
456
+ url = (
457
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
458
+ + TestData.TEST_CONFIG["api_version"]
459
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
460
+ )
461
+
462
+ f1 = FileAtom(
463
+ hashes=Hashes(
464
+ md5="d41d8cd98f00b204e9800998ecf8427e",
465
+ sha1="da39a3ee5e6b4b0d3255bfef95601890afd80709",
466
+ sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
467
+ )
468
+ )
469
+ ip1 = IpAtom("52.48.79.33")
470
+ em1 = EmailAtom("hacker@hacker.fr")
471
+ url1 = UrlAtom("http://notfishing.com")
472
+
473
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
474
+
475
+ ## Prepare threat types for this sighting
476
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
477
+
478
+ ## Prepare start and end timestamps for this sighting
479
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
480
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
481
+ input_sighting = {
482
+ "atoms": [ip1, f1, em1, url1],
483
+ "start_timestamp": start,
484
+ "end_timestamp": end,
485
+ "sighting_type": SightingType.POSITIVE,
486
+ "description_visibility": Visibility.PUBLIC,
487
+ "count": 1,
488
+ "threat_types": threat_types,
489
+ "tags": ["some_tag"],
490
+ "description": "some_description",
491
+ "editable": True,
492
+ }
493
+ expected_request = {
494
+ "data": [
495
+ {
496
+ "count": 1,
497
+ "description": "some_description",
498
+ "description_visibility": "PUBLIC",
499
+ "editable": True,
500
+ "email_list": [{"email": "hacker@hacker.fr"}],
501
+ "end_timestamp": "2025-12-16T10:00:00Z",
502
+ "file_list": [
503
+ {
504
+ "hashes": {
505
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
506
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
507
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
508
+ }
509
+ }
510
+ ],
511
+ "ip_list": [{"ip_address": "52.48.79.33"}],
512
+ "start_timestamp": "2025-12-16T09:00:00Z",
513
+ "tags": ["some_tag"],
514
+ "threat_types": ["phishing", "scam"],
515
+ "type": "positive",
516
+ "url_list": [{"url": "http://notfishing.com"}],
517
+ }
518
+ ]
519
+ }
520
+ expected_res = {
521
+ "count": 1,
522
+ "results": [
523
+ {
524
+ "count": 1,
525
+ "description": "some_description",
526
+ "description_visibility": "PUBLIC",
527
+ "editable": True,
528
+ "end_timestamp": "2025-12-16T10:00:00Z",
529
+ "relation_type": "sighting",
530
+ "reliability": 100,
531
+ "sighting_version": "3.0.0",
532
+ "sightings": {
533
+ "email_list": [{"email": "hacker@hacker.fr"}],
534
+ "file_list": [
535
+ {
536
+ "hashes": {
537
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
538
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
539
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
540
+ "sha512": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
541
+ }
542
+ }
543
+ ],
544
+ "ip_list": [{"ip_address": "52.48.79.33"}],
545
+ "url_list": [{"url": "http://notfishing.com"}],
546
+ },
547
+ "source_context": {
548
+ "source_id": "org: Orange Cyberdefense",
549
+ "source_uses": ["sightings"],
550
+ },
551
+ "start_timestamp": "2025-12-16T09:00:00Z",
552
+ "tags": [{"categories": [], "name": "some_tag"}],
553
+ "threat_types": ["phishing", "scam"],
554
+ "timestamp_created": "2025-12-16T10:00:00Z",
555
+ "type": "positive",
556
+ "uid": "e14dd426-ed41-436c-9e0e-0d785e750428",
557
+ }
558
+ ],
559
+ }
560
+
561
+ responses.post(
562
+ url=url,
563
+ json=expected_res,
564
+ status=200,
565
+ match=[matchers.json_params_matcher(expected_request)],
566
+ )
567
+
568
+ with patch.dict(
569
+ os.environ,
570
+ {
571
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
572
+ },
573
+ clear=True,
574
+ ):
575
+ res = datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
576
+
577
+ assert res == expected_res
578
+
579
+
580
+ def test_bulk_submit_sightings_payload_error(datalake):
581
+ with pytest.raises(ValueError) as err:
582
+ datalake.Sightings.bulk_submit_sightings(
583
+ sightings=[
584
+ {
585
+ "atoms": [FileAtom(Hashes(md5="d41d8cd98f00b204e9800998ecf8427e"))],
586
+ "start_timestamp": start,
587
+ "end_timestamp": end,
588
+ "sighting_type": SightingType.NEUTRAL,
589
+ "description_visibility": Visibility.PUBLIC,
590
+ "count": 3,
591
+ "threat_types": threat_types,
592
+ "tags": ["some_tag_bis"],
593
+ "description": "some_description_bis",
594
+ "editable": False,
595
+ }
596
+ ]
597
+ )
598
+ assert (
599
+ str(err.value) == """For NEUTRAL sightings, "threat_types" can't be passed."""
600
+ )
601
+
602
+
603
+ @responses.activate
604
+ def test_bulk_submit_sightings_failure(datalake):
605
+ url = (
606
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
607
+ + TestData.TEST_CONFIG["api_version"]
608
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
609
+ )
610
+
611
+ hashkeys = [
612
+ "d41d8cd98f00b204e9800998ecf8427e",
613
+ "a7b25b324871a7695aa2cc5d09681dda",
614
+ "1ed07771327e850255b09b042ad00e3d",
615
+ "bf1f33c3a56e1dfda6a2f4f3d3e4361a",
616
+ ]
617
+
618
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
619
+
620
+ ## Prepare threat types for this sighting
621
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
622
+
623
+ ## Prepare start and end timestamps for this sighting
624
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
625
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
626
+ input_sighting = {
627
+ "hashkeys": hashkeys,
628
+ "start_timestamp": start,
629
+ "end_timestamp": end,
630
+ "sighting_type": SightingType.POSITIVE,
631
+ "description_visibility": Visibility.PUBLIC,
632
+ "count": 1,
633
+ "threat_types": threat_types,
634
+ "tags": ["some_tag"],
635
+ "description": "some_description",
636
+ "editable": True,
637
+ }
638
+ expected_request = {
639
+ "data": [
640
+ {
641
+ "count": 1,
642
+ "description": "some_description",
643
+ "description_visibility": "PUBLIC",
644
+ "editable": True,
645
+ "end_timestamp": "2025-12-16T10:00:00Z",
646
+ "hashkeys": hashkeys,
647
+ "start_timestamp": "2025-12-16T09:00:00Z",
648
+ "tags": ["some_tag"],
649
+ "threat_types": ["phishing", "scam"],
650
+ "type": "positive",
651
+ }
652
+ ]
653
+ }
654
+ expected_res = {
655
+ "data": {
656
+ "0": {
657
+ "hashkeys": [
658
+ "Some of the provided hashkeys has not been found: "
659
+ "['d41d8cd98f00b204e9800998ecf8427e']"
660
+ ]
661
+ }
662
+ }
663
+ }
664
+
665
+ responses.post(
666
+ url=url,
667
+ json=expected_res,
668
+ status=422,
669
+ match=[matchers.json_params_matcher(expected_request)],
670
+ )
671
+
672
+ with patch.dict(
673
+ os.environ,
674
+ {
675
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
676
+ },
677
+ clear=True,
678
+ ):
679
+ with pytest.raises(ValueError) as exc:
680
+ datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
681
+
682
+ prefix = "422 HTTP code: "
683
+ msg = str(exc.value)
684
+ assert msg.startswith(prefix)
685
+
686
+ payload = ast.literal_eval(msg[len(prefix) :])
687
+ assert payload == expected_res
688
+
689
+
451
690
  def test_sightings_filtered_bad_ordering(datalake):
452
691
  with pytest.raises(ValueError) as err:
453
692
  datalake.Sightings.sightings_filtered(ordering="bad_ordering")