datalake-scripts 3.0.0rc3__tar.gz → 3.0.0rc6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/PKG-INFO +14 -5
  2. datalake_scripts-3.0.0rc6/datalake/config/endpoints.json +35 -0
  3. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/datalake.py +9 -0
  4. datalake_scripts-3.0.0rc6/datalake/endpoints/my_account.py +17 -0
  5. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sightings.py +35 -2
  6. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/cli.py +11 -3
  7. datalake_scripts-3.0.0rc6/datalake_scripts/scripts/get_my_user_info.py +32 -0
  8. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/PKG-INFO +15 -6
  9. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/SOURCES.txt +77 -1
  10. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/entry_points.txt +0 -1
  11. datalake_scripts-3.0.0rc6/pyproject.toml +3 -0
  12. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/setup.cfg +3 -5
  13. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/setup.py +17 -6
  14. datalake_scripts-3.0.0rc6/tests/lib/test_my_account.py +51 -0
  15. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_sightings.py +240 -1
  16. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_cli.py +2 -0
  17. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/README.md +0 -0
  18. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/__init__.py +0 -0
  19. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/__init__.py +0 -0
  20. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/atom.py +0 -0
  21. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/atom_type.py +0 -0
  22. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/bulk_search_task.py +0 -0
  23. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/config.py +0 -0
  24. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/logger.py +0 -0
  25. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/output.py +0 -0
  26. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/throttler.py +0 -0
  27. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/token_manager.py +0 -0
  28. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/utils.py +0 -0
  29. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/common/warn.py +0 -0
  30. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/__init__.py +0 -0
  31. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/advanced_search.py +0 -0
  32. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/bulk_search.py +0 -0
  33. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/comments.py +0 -0
  34. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/endpoint.py +0 -0
  35. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/filtered_threat_entity.py +0 -0
  36. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/sources.py +0 -0
  37. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/tags.py +0 -0
  38. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/endpoints/threats.py +0 -0
  39. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/__init__.py +0 -0
  40. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake/miscellaneous/search_watch.py +0 -0
  41. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/__init__.py +0 -0
  42. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/__init__.py +0 -0
  43. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/common/base_script.py +0 -0
  44. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/__init__.py +0 -0
  45. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  46. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/helper_scripts/utils.py +0 -0
  47. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/__init__.py +0 -0
  48. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_comments.py +0 -0
  49. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_tags.py +0 -0
  50. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/add_threats.py +0 -0
  51. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/advanced_search.py +0 -0
  52. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  53. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/edit_score.py +0 -0
  54. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_atom_values.py +0 -0
  55. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
  56. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_query_hash.py +0 -0
  57. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats.py +0 -0
  58. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  59. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/lookup_threats.py +0 -0
  60. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts/scripts/search_watch.py +0 -0
  61. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  62. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/requires.txt +0 -0
  63. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/datalake_scripts.egg-info/top_level.txt +0 -0
  64. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/__init__.py +0 -0
  65. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/__init__.py +0 -0
  66. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/fixture.py +0 -0
  67. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/test_output.py +0 -0
  68. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/common/test_utils.py +0 -0
  69. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/conftest.py +0 -0
  70. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/__init__.py +0 -0
  71. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_advanced_search.py +0 -0
  72. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_atom_type.py +0 -0
  73. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_auth_via_tokens.py +0 -0
  74. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_bulk_search.py +0 -0
  75. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_filtered_threat_entity.py +0 -0
  76. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_search_watch.py +0 -0
  77. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_tags.py +0 -0
  78. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/lib/test_threats.py +0 -0
  79. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/__init__.py +0 -0
  80. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_add_new_threats.py +0 -0
  81. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_base_scripts.py +0 -0
  82. {datalake_scripts-3.0.0rc3 → datalake_scripts-3.0.0rc6}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc3
3
+ Version: 3.0.0rc6
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
127
137
  ### Contributing
128
138
 
129
139
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -0,0 +1,35 @@
1
+ {
2
+ "main": {
3
+ "preprod": "https://ti2.extranet.mrti-center.com/api/",
4
+ "prod": "https://datalake.cert.orangecyberdefense.com/api/"
5
+ },
6
+ "endpoints": {
7
+ "token": "auth/token/",
8
+ "refresh-token": "auth/refresh-token/",
9
+ "advanced-queries-threats": "mrti/advanced-queries/threats/",
10
+ "advanced-queries-threats-query-hash": "mrti/advanced-queries/threats/{query_hash}/",
11
+ "atom-values": "mrti/atom-values/",
12
+ "bulk-search": "mrti/bulk-search/",
13
+ "bulk-search-tasks": "mrti/bulk-search/tasks/",
14
+ "bulk-search-task": "mrti/bulk-search/task/{task_uuid}/",
15
+ "bulk-manual-threats": "mrti/bulk-manual-threats/",
16
+ "bulk-manual-threats-task": "mrti/bulk-manual-threats/task/{task_uuid}/",
17
+ "sources": "mrti/sources/",
18
+ "threat-entity-filtered":"mrti/threat-entity/filtered/",
19
+ "threats-manual": "mrti/threats-manual/",
20
+ "threats-bulk-lookup": "mrti/threats/bulk-lookup/",
21
+ "threats": "mrti/threats/{hashkey}/",
22
+ "threats-lookup": "mrti/threats/lookup/",
23
+ "threats-comments": "mrti/threats/{hashkey}/comments/",
24
+ "threats-tags": "mrti/threats/{hashkey}/tags/",
25
+ "threats-atom-values-extract": "mrti/threats/atom-values-extract/",
26
+ "threats-bulk-scoring-edits": "mrti/threats/bulk-scoring-edits/",
27
+ "threats-sighting": "mrti/threats/sighting/",
28
+ "threats-bulk-sighting": "mrti/threats/bulk-sighting/",
29
+ "threats-sighting-file": "mrti/threats/sighting-file/{hashkey}/",
30
+ "threats-sighting-filtered": "mrti/threats/sighting/filtered/",
31
+ "users-me": "users/me/"
32
+ },
33
+ "threats-manual-bulk-size": 100,
34
+ "api_version": "v3/"
35
+ }
@@ -11,6 +11,7 @@ from datalake.endpoints.advanced_search import AdvancedSearch
11
11
  from datalake.endpoints.sightings import Sightings
12
12
  from datalake.endpoints.sources import Sources
13
13
  from datalake.endpoints.filtered_threat_entity import FilteredThreatEntity
14
+ from datalake.endpoints.my_account import MyAccount
14
15
  from datalake.miscellaneous.search_watch import SearchWatch
15
16
 
16
17
 
@@ -58,6 +59,14 @@ class Datalake:
58
59
  raise
59
60
 
60
61
  # Endpoints
62
+ self.MyAccount = MyAccount(
63
+ self.logger,
64
+ endpoint_config,
65
+ env,
66
+ token_manager,
67
+ proxies=proxies,
68
+ verify=verify,
69
+ )
61
70
  self.AdvancedSearch = AdvancedSearch(
62
71
  self.logger,
63
72
  endpoint_config,
@@ -0,0 +1,17 @@
1
+ from typing import List, Tuple
2
+
3
+ from datalake.endpoints.endpoint import Endpoint
4
+ from datalake.common.output import parse_response
5
+
6
+
7
+ class MyAccount(Endpoint):
8
+
9
+ def me(self):
10
+ """
11
+ Gets details of the currently logged in user.
12
+ """
13
+ url = self._build_url_for_endpoint("users-me")
14
+ response = parse_response(
15
+ self.datalake_requests(url, "get", self._get_headers())
16
+ )
17
+ return response
@@ -46,9 +46,9 @@ class Sightings(Endpoint):
46
46
  impersonate_id: int = None,
47
47
  ):
48
48
  """
49
- Submit a list of sightings.
49
+ Submit a list of atoms as 1 sighting.
50
50
  Either threat "hashkeys" or list of atom objects in "atoms" is required.
51
- Possible sightings type: POSITIVE, NEGATIVE, NEUTRAL.
51
+ Possible sighting's type: POSITIVE, NEGATIVE, NEUTRAL.
52
52
  For POSITIVE and NEGATIVE sightings, "threat_types"
53
53
  field is required.
54
54
  End date timestamp should always be in the past.
@@ -73,6 +73,39 @@ class Sightings(Endpoint):
73
73
  res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
74
74
  return res
75
75
 
76
+ def bulk_submit_sightings(
77
+ self,
78
+ sightings: List[dict] = [],
79
+ ):
80
+ """
81
+ Submit a list of Sightings.
82
+ Input must be a list of valid dict.
83
+ Each dict is a sighting.
84
+ """
85
+ payload = {}
86
+ list_sightings_for_url_call = []
87
+ for sighting in sightings:
88
+ payload_item = self._prepare_sightings_payload(
89
+ sighting.get("atoms"),
90
+ sighting.get("hashkeys"),
91
+ sighting.get("start_timestamp"),
92
+ sighting.get("end_timestamp"),
93
+ sighting.get("sighting_type"),
94
+ sighting.get("description_visibility"),
95
+ sighting.get("count"),
96
+ sighting.get("threat_types"),
97
+ sighting.get("tags"),
98
+ sighting.get("description"),
99
+ sighting.get("relation"),
100
+ sighting.get("editable"),
101
+ sighting.get("impersonate_id"),
102
+ )
103
+ list_sightings_for_url_call.append(payload_item)
104
+ payload["data"] = list_sightings_for_url_call
105
+ url = self._build_url_for_endpoint("threats-bulk-sighting")
106
+ res = self.datalake_requests(url, "post", self._post_headers(), payload).json()
107
+ return res
108
+
76
109
  @staticmethod
77
110
  def _check_sightings_payload_parameters(
78
111
  atoms,
@@ -15,12 +15,13 @@ from datalake_scripts.scripts import (
15
15
  get_atom_values,
16
16
  get_filtered_threat_entity,
17
17
  search_watch,
18
+ get_my_user_info,
18
19
  )
19
20
 
20
21
 
21
22
  class Cli:
22
23
  CLI_NAME = "ocd-dtl"
23
- VERSION = "3.0.0rc3"
24
+ VERSION = "3.0.0rc6"
24
25
 
25
26
  def __init__(self):
26
27
  parser = argparse.ArgumentParser(
@@ -54,8 +55,15 @@ class Cli:
54
55
  get_filtered_threat_entity.main,
55
56
  )
56
57
  self._add_command_subparser(subparsers, "search_watch", search_watch.main)
58
+ self._add_command_subparser(
59
+ subparsers, "get_my_user_info", get_my_user_info.main
60
+ )
61
+
62
+ # args = parser.parse_args(sys.argv[1:2])
63
+ command_called = sys.argv[1:2]
64
+ command_arguments = sys.argv[2:]
57
65
 
58
- args = parser.parse_args(sys.argv[1:2])
66
+ args = parser.parse_args(command_called)
59
67
 
60
68
  if args.version:
61
69
  print(self.VERSION)
@@ -67,7 +75,7 @@ class Cli:
67
75
  exit(1)
68
76
 
69
77
  # Call the subcommand method
70
- args.func(sys.argv[2:])
78
+ args.func(command_arguments)
71
79
 
72
80
  def _add_command_subparser(self, subparsers, name, method):
73
81
  command_parser = subparsers.add_parser(name)
@@ -0,0 +1,32 @@
1
+ import sys
2
+
3
+ from datalake import Datalake
4
+ from datalake_scripts.common.base_script import BaseScripts
5
+ from datalake_scripts.helper_scripts.utils import save_output
6
+
7
+
8
+ def get_my_user_info_func(dtl: Datalake):
9
+ response = dtl.MyAccount.me()
10
+ return response
11
+
12
+
13
+ def main(override_args=None):
14
+ """Method to start the script"""
15
+ # Load initial args
16
+ parser = BaseScripts.start("Gets details of the currently logged in user.")
17
+
18
+ args = parser.parse_args(override_args or [])
19
+
20
+ dtl = Datalake(env=args.env, log_level=args.loglevel)
21
+ response = get_my_user_info_func(dtl)
22
+
23
+ if args.output:
24
+ save_output(args.output, response)
25
+ dtl.logger.debug(f"Results saved in {args.output}\n")
26
+ else:
27
+ print(response)
28
+ dtl.logger.debug(f"END: get_my_user_info.py")
29
+
30
+
31
+ if __name__ == "__main__":
32
+ sys.exit(main())
@@ -1,12 +1,10 @@
1
- Metadata-Version: 2.1
2
- Name: datalake-scripts
3
- Version: 3.0.0rc3
1
+ Metadata-Version: 2.4
2
+ Name: datalake_scripts
3
+ Version: 3.0.0rc6
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
7
7
  Author-email: cert-contact.ocd@orange.com
8
- License: UNKNOWN
9
- Platform: UNKNOWN
10
8
  Classifier: Programming Language :: Python :: 3.7
11
9
  Classifier: Programming Language :: Python :: 3.8
12
10
  Classifier: Programming Language :: Python :: 3.9
@@ -17,6 +15,18 @@ Classifier: Topic :: Security
17
15
  Classifier: Natural Language :: English
18
16
  Requires-Python: >=3.6
19
17
  Description-Content-Type: text/markdown
18
+ Requires-Dist: requests
19
+ Requires-Dist: halo
20
+ Requires-Dist: prettytable
21
+ Dynamic: author
22
+ Dynamic: author-email
23
+ Dynamic: classifier
24
+ Dynamic: description
25
+ Dynamic: description-content-type
26
+ Dynamic: home-page
27
+ Dynamic: requires-dist
28
+ Dynamic: requires-python
29
+ Dynamic: summary
20
30
 
21
31
 
22
32
  ____ _ _ _ ____ _ _
@@ -127,4 +137,3 @@ We recommend to lower the `OCD_DTL_REQUESTS_PER_QUOTA_TIME` value, if you encoun
127
137
  ### Contributing
128
138
 
129
139
  To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
130
-
@@ -1,4 +1,5 @@
1
1
  README.md
2
+ pyproject.toml
2
3
  setup.cfg
3
4
  setup.py
4
5
  ./datalake/__init__.py
@@ -14,12 +15,14 @@ setup.py
14
15
  ./datalake/common/token_manager.py
15
16
  ./datalake/common/utils.py
16
17
  ./datalake/common/warn.py
18
+ ./datalake/config/endpoints.json
17
19
  ./datalake/endpoints/__init__.py
18
20
  ./datalake/endpoints/advanced_search.py
19
21
  ./datalake/endpoints/bulk_search.py
20
22
  ./datalake/endpoints/comments.py
21
23
  ./datalake/endpoints/endpoint.py
22
24
  ./datalake/endpoints/filtered_threat_entity.py
25
+ ./datalake/endpoints/my_account.py
23
26
  ./datalake/endpoints/sightings.py
24
27
  ./datalake/endpoints/sources.py
25
28
  ./datalake/endpoints/tags.py
@@ -42,6 +45,7 @@ setup.py
42
45
  ./datalake_scripts/scripts/edit_score.py
43
46
  ./datalake_scripts/scripts/get_atom_values.py
44
47
  ./datalake_scripts/scripts/get_filtered_threat_entity.py
48
+ ./datalake_scripts/scripts/get_my_user_info.py
45
49
  ./datalake_scripts/scripts/get_query_hash.py
46
50
  ./datalake_scripts/scripts/get_threats.py
47
51
  ./datalake_scripts/scripts/get_threats_from_query_hash.py
@@ -59,6 +63,7 @@ setup.py
59
63
  ./tests/lib/test_auth_via_tokens.py
60
64
  ./tests/lib/test_bulk_search.py
61
65
  ./tests/lib/test_filtered_threat_entity.py
66
+ ./tests/lib/test_my_account.py
62
67
  ./tests/lib/test_search_watch.py
63
68
  ./tests/lib/test_sightings.py
64
69
  ./tests/lib/test_tags.py
@@ -68,9 +73,80 @@ setup.py
68
73
  ./tests/scripts/test_base_scripts.py
69
74
  ./tests/scripts/test_cli.py
70
75
  ./tests/scripts/test_csv_builder.py
76
+ datalake/__init__.py
77
+ datalake/datalake.py
78
+ datalake/common/__init__.py
79
+ datalake/common/atom.py
80
+ datalake/common/atom_type.py
81
+ datalake/common/bulk_search_task.py
82
+ datalake/common/config.py
83
+ datalake/common/logger.py
84
+ datalake/common/output.py
85
+ datalake/common/throttler.py
86
+ datalake/common/token_manager.py
87
+ datalake/common/utils.py
88
+ datalake/common/warn.py
89
+ datalake/config/endpoints.json
90
+ datalake/endpoints/__init__.py
91
+ datalake/endpoints/advanced_search.py
92
+ datalake/endpoints/bulk_search.py
93
+ datalake/endpoints/comments.py
94
+ datalake/endpoints/endpoint.py
95
+ datalake/endpoints/filtered_threat_entity.py
96
+ datalake/endpoints/my_account.py
97
+ datalake/endpoints/sightings.py
98
+ datalake/endpoints/sources.py
99
+ datalake/endpoints/tags.py
100
+ datalake/endpoints/threats.py
101
+ datalake/miscellaneous/__init__.py
102
+ datalake/miscellaneous/search_watch.py
103
+ datalake_scripts/__init__.py
104
+ datalake_scripts/cli.py
71
105
  datalake_scripts.egg-info/PKG-INFO
72
106
  datalake_scripts.egg-info/SOURCES.txt
73
107
  datalake_scripts.egg-info/dependency_links.txt
74
108
  datalake_scripts.egg-info/entry_points.txt
75
109
  datalake_scripts.egg-info/requires.txt
76
- datalake_scripts.egg-info/top_level.txt
110
+ datalake_scripts.egg-info/top_level.txt
111
+ datalake_scripts/common/__init__.py
112
+ datalake_scripts/common/base_script.py
113
+ datalake_scripts/helper_scripts/__init__.py
114
+ datalake_scripts/helper_scripts/output_builder.py
115
+ datalake_scripts/helper_scripts/utils.py
116
+ datalake_scripts/scripts/__init__.py
117
+ datalake_scripts/scripts/add_comments.py
118
+ datalake_scripts/scripts/add_tags.py
119
+ datalake_scripts/scripts/add_threats.py
120
+ datalake_scripts/scripts/advanced_search.py
121
+ datalake_scripts/scripts/bulk_lookup_threats.py
122
+ datalake_scripts/scripts/edit_score.py
123
+ datalake_scripts/scripts/get_atom_values.py
124
+ datalake_scripts/scripts/get_filtered_threat_entity.py
125
+ datalake_scripts/scripts/get_my_user_info.py
126
+ datalake_scripts/scripts/get_query_hash.py
127
+ datalake_scripts/scripts/get_threats.py
128
+ datalake_scripts/scripts/get_threats_from_query_hash.py
129
+ datalake_scripts/scripts/lookup_threats.py
130
+ datalake_scripts/scripts/search_watch.py
131
+ tests/__init__.py
132
+ tests/conftest.py
133
+ tests/common/__init__.py
134
+ tests/common/fixture.py
135
+ tests/common/test_output.py
136
+ tests/common/test_utils.py
137
+ tests/lib/__init__.py
138
+ tests/lib/test_advanced_search.py
139
+ tests/lib/test_atom_type.py
140
+ tests/lib/test_auth_via_tokens.py
141
+ tests/lib/test_bulk_search.py
142
+ tests/lib/test_filtered_threat_entity.py
143
+ tests/lib/test_my_account.py
144
+ tests/lib/test_search_watch.py
145
+ tests/lib/test_sightings.py
146
+ tests/lib/test_tags.py
147
+ tests/lib/test_threats.py
148
+ tests/scripts/__init__.py
149
+ tests/scripts/test_add_new_threats.py
150
+ tests/scripts/test_base_scripts.py
151
+ tests/scripts/test_cli.py
152
+ tests/scripts/test_csv_builder.py
@@ -1,3 +1,2 @@
1
1
  [console_scripts]
2
2
  ocd-dtl = datalake_scripts.cli:main
3
-
@@ -0,0 +1,3 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
@@ -2,12 +2,10 @@
2
2
  upgrade = true
3
3
 
4
4
  [egg_info]
5
- tag-date = false
6
- tag-build =
7
- tag_svn_revision = false
8
- tag_build =
9
5
  tag_date = 0
6
+ tag_build =
7
+ tag_svn_revision = false
10
8
 
11
9
  [metadata]
12
- description-file = README.md
10
+ description_file = README.md
13
11
 
@@ -2,20 +2,31 @@ import os
2
2
 
3
3
  from setuptools import find_packages, setup
4
4
 
5
- from datalake_scripts.cli import Cli
6
-
7
5
  here = os.path.abspath(os.path.dirname(__file__))
8
6
 
9
7
  with open("README.md", "r") as fh:
10
- long_description = fh.read()
8
+ LONG_DESCRIPTION = fh.read()
9
+
10
+
11
+ def read_version_or_cli_name(option: str):
12
+ with open("datalake_scripts/cli.py") as f:
13
+ for line in f:
14
+ if option == "version" and line.strip().startswith("VERSION"):
15
+ return line.split("=")[1].strip().strip('"')
16
+ if option == "cli_name" and line.strip().startswith("CLI_NAME"):
17
+ return line.split("=")[1].strip().strip('"')
18
+
19
+
20
+ CLI_NAME = read_version_or_cli_name("cli_name")
21
+ VERSION = read_version_or_cli_name("version")
11
22
 
12
23
  setup(
13
24
  name="datalake_scripts",
14
- version=Cli.VERSION,
25
+ version=VERSION,
15
26
  author="OCD",
16
27
  author_email="cert-contact.ocd@orange.com",
17
28
  description="A collection of scripts to easily use the API of OCD Datalake",
18
- long_description=long_description,
29
+ long_description=LONG_DESCRIPTION,
19
30
  long_description_content_type="text/markdown",
20
31
  url="https://github.com/cert-orangecyberdefense/datalake/",
21
32
  package_dir={
@@ -25,7 +36,7 @@ setup(
25
36
  install_requires=["requests", "halo", "prettytable"],
26
37
  test_suite="nose.collector",
27
38
  entry_points={
28
- "console_scripts": (f"{Cli.CLI_NAME} = datalake_scripts.cli:main",),
39
+ "console_scripts": (f"{CLI_NAME} = datalake_scripts.cli:main",),
29
40
  },
30
41
  classifiers=[
31
42
  "Programming Language :: Python :: 3.7",
@@ -0,0 +1,51 @@
1
+ import pytest
2
+ import responses
3
+ from datalake import Datalake
4
+ from tests.common.fixture import TestData, datalake
5
+
6
+ mock_results_data = {
7
+ "email": "user@example.com",
8
+ "first_name": "string",
9
+ "force_password_change": True,
10
+ "full_name": "string",
11
+ "has_password": True,
12
+ "id": 0,
13
+ "is_active": True,
14
+ "is_deleted": False,
15
+ "last_login": "2025-10-15T08:43:33.390Z",
16
+ "last_name": "string",
17
+ "organization": {"id": 0, "name": "string", "path_names": ["string"]},
18
+ "previous_login": "2025-10-15T08:43:33.390Z",
19
+ "request_limit": 0,
20
+ "role": {
21
+ "administration_permissions": {"description": "string", "name": "string"},
22
+ "id": 0,
23
+ "name": "string",
24
+ },
25
+ "totp_enabled": True,
26
+ }
27
+
28
+
29
+ def mock_api_response():
30
+ responses.add(
31
+ responses.GET,
32
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
33
+ + TestData.TEST_CONFIG["api_version"]
34
+ + TestData.TEST_CONFIG["endpoints"]["users-me"],
35
+ json=mock_results_data, # Only return the "results" part
36
+ status=200,
37
+ )
38
+
39
+
40
+ @responses.activate
41
+ def test_my_account(datalake: Datalake):
42
+ mock_api_response()
43
+ response = datalake.MyAccount.me()
44
+ assert response == mock_results_data
45
+ assert isinstance(response, dict), "Results should be a dict"
46
+ assert responses.calls[0].request.url, (
47
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
48
+ + TestData.TEST_CONFIG["api_version"]
49
+ + TestData.TEST_CONFIG["endpoints"]["users-me"]
50
+ )
51
+ assert responses.calls[0].request.method, "GET"
@@ -1,16 +1,19 @@
1
1
  from unittest.mock import patch
2
2
  import pytest
3
3
  import responses
4
- from datetime import datetime
4
+ from datetime import datetime, timedelta, timezone
5
5
 
6
6
  from responses import matchers
7
7
  import os
8
+ import ast
8
9
 
9
10
 
10
11
  from tests.common.fixture import TestData, datalake # noqa needed fixture import
11
12
  from datalake import (
12
13
  IpAtom,
13
14
  FileAtom,
15
+ EmailAtom,
16
+ UrlAtom,
14
17
  Hashes,
15
18
  Jarm,
16
19
  IpService,
@@ -448,6 +451,242 @@ def test_submit_sightings_bad_atom(datalake):
448
451
  assert str(err.value) == '"atoms" needs to be a list of Atom subclasses.'
449
452
 
450
453
 
454
+ @responses.activate
455
+ def test_bulk_submit_sightings_success(datalake):
456
+ url = (
457
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
458
+ + TestData.TEST_CONFIG["api_version"]
459
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
460
+ )
461
+
462
+ f1 = FileAtom(
463
+ hashes=Hashes(
464
+ md5="d41d8cd98f00b204e9800998ecf8427e",
465
+ sha1="da39a3ee5e6b4b0d3255bfef95601890afd80709",
466
+ sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
467
+ )
468
+ )
469
+ ip1 = IpAtom("52.48.79.33")
470
+ em1 = EmailAtom("hacker@hacker.fr")
471
+ url1 = UrlAtom("http://notfishing.com")
472
+
473
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
474
+
475
+ ## Prepare threat types for this sighting
476
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
477
+
478
+ ## Prepare start and end timestamps for this sighting
479
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
480
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
481
+ input_sighting = {
482
+ "atoms": [ip1, f1, em1, url1],
483
+ "start_timestamp": start,
484
+ "end_timestamp": end,
485
+ "sighting_type": SightingType.POSITIVE,
486
+ "description_visibility": Visibility.PUBLIC,
487
+ "count": 1,
488
+ "threat_types": threat_types,
489
+ "tags": ["some_tag"],
490
+ "description": "some_description",
491
+ "editable": True,
492
+ }
493
+ expected_request = {
494
+ "data": [
495
+ {
496
+ "count": 1,
497
+ "description": "some_description",
498
+ "description_visibility": "PUBLIC",
499
+ "editable": True,
500
+ "email_list": [{"email": "hacker@hacker.fr"}],
501
+ "end_timestamp": "2025-12-16T10:00:00Z",
502
+ "file_list": [
503
+ {
504
+ "hashes": {
505
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
506
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
507
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
508
+ }
509
+ }
510
+ ],
511
+ "ip_list": [{"ip_address": "52.48.79.33"}],
512
+ "start_timestamp": "2025-12-16T09:00:00Z",
513
+ "tags": ["some_tag"],
514
+ "threat_types": ["phishing", "scam"],
515
+ "type": "positive",
516
+ "url_list": [{"url": "http://notfishing.com"}],
517
+ }
518
+ ]
519
+ }
520
+ expected_res = {
521
+ "count": 1,
522
+ "results": [
523
+ {
524
+ "count": 1,
525
+ "description": "some_description",
526
+ "description_visibility": "PUBLIC",
527
+ "editable": True,
528
+ "end_timestamp": "2025-12-16T10:00:00Z",
529
+ "relation_type": "sighting",
530
+ "reliability": 100,
531
+ "sighting_version": "3.0.0",
532
+ "sightings": {
533
+ "email_list": [{"email": "hacker@hacker.fr"}],
534
+ "file_list": [
535
+ {
536
+ "hashes": {
537
+ "md5": "d41d8cd98f00b204e9800998ecf8427e",
538
+ "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
539
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
540
+ "sha512": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
541
+ }
542
+ }
543
+ ],
544
+ "ip_list": [{"ip_address": "52.48.79.33"}],
545
+ "url_list": [{"url": "http://notfishing.com"}],
546
+ },
547
+ "source_context": {
548
+ "source_id": "org: Orange Cyberdefense",
549
+ "source_uses": ["sightings"],
550
+ },
551
+ "start_timestamp": "2025-12-16T09:00:00Z",
552
+ "tags": [{"categories": [], "name": "some_tag"}],
553
+ "threat_types": ["phishing", "scam"],
554
+ "timestamp_created": "2025-12-16T10:00:00Z",
555
+ "type": "positive",
556
+ "uid": "e14dd426-ed41-436c-9e0e-0d785e750428",
557
+ }
558
+ ],
559
+ }
560
+
561
+ responses.post(
562
+ url=url,
563
+ json=expected_res,
564
+ status=200,
565
+ match=[matchers.json_params_matcher(expected_request)],
566
+ )
567
+
568
+ with patch.dict(
569
+ os.environ,
570
+ {
571
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
572
+ },
573
+ clear=True,
574
+ ):
575
+ res = datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
576
+
577
+ assert res == expected_res
578
+
579
+
580
+ def test_bulk_submit_sightings_payload_error(datalake):
581
+ with pytest.raises(ValueError) as err:
582
+ datalake.Sightings.bulk_submit_sightings(
583
+ sightings=[
584
+ {
585
+ "atoms": [FileAtom(Hashes(md5="d41d8cd98f00b204e9800998ecf8427e"))],
586
+ "start_timestamp": start,
587
+ "end_timestamp": end,
588
+ "sighting_type": SightingType.NEUTRAL,
589
+ "description_visibility": Visibility.PUBLIC,
590
+ "count": 3,
591
+ "threat_types": threat_types,
592
+ "tags": ["some_tag_bis"],
593
+ "description": "some_description_bis",
594
+ "editable": False,
595
+ }
596
+ ]
597
+ )
598
+ assert (
599
+ str(err.value) == """For NEUTRAL sightings, "threat_types" can't be passed."""
600
+ )
601
+
602
+
603
+ @responses.activate
604
+ def test_bulk_submit_sightings_failure(datalake):
605
+ url = (
606
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
607
+ + TestData.TEST_CONFIG["api_version"]
608
+ + TestData.TEST_CONFIG["endpoints"]["threats-bulk-sighting"]
609
+ )
610
+
611
+ hashkeys = [
612
+ "d41d8cd98f00b204e9800998ecf8427e",
613
+ "a7b25b324871a7695aa2cc5d09681dda",
614
+ "1ed07771327e850255b09b042ad00e3d",
615
+ "bf1f33c3a56e1dfda6a2f4f3d3e4361a",
616
+ ]
617
+
618
+ # You also need to define additional properties for these atoms, shared by all atoms within the same sighting.
619
+
620
+ ## Prepare threat types for this sighting
621
+ threat_types = [ThreatType.PHISHING, ThreatType.SCAM]
622
+
623
+ ## Prepare start and end timestamps for this sighting
624
+ start = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc) - timedelta(hours=1)
625
+ end = datetime(2025, 12, 16, 10, 0, 0, tzinfo=timezone.utc)
626
+ input_sighting = {
627
+ "hashkeys": hashkeys,
628
+ "start_timestamp": start,
629
+ "end_timestamp": end,
630
+ "sighting_type": SightingType.POSITIVE,
631
+ "description_visibility": Visibility.PUBLIC,
632
+ "count": 1,
633
+ "threat_types": threat_types,
634
+ "tags": ["some_tag"],
635
+ "description": "some_description",
636
+ "editable": True,
637
+ }
638
+ expected_request = {
639
+ "data": [
640
+ {
641
+ "count": 1,
642
+ "description": "some_description",
643
+ "description_visibility": "PUBLIC",
644
+ "editable": True,
645
+ "end_timestamp": "2025-12-16T10:00:00Z",
646
+ "hashkeys": hashkeys,
647
+ "start_timestamp": "2025-12-16T09:00:00Z",
648
+ "tags": ["some_tag"],
649
+ "threat_types": ["phishing", "scam"],
650
+ "type": "positive",
651
+ }
652
+ ]
653
+ }
654
+ expected_res = {
655
+ "data": {
656
+ "0": {
657
+ "hashkeys": [
658
+ "Some of the provided hashkeys has not been found: "
659
+ "['d41d8cd98f00b204e9800998ecf8427e']"
660
+ ]
661
+ }
662
+ }
663
+ }
664
+
665
+ responses.post(
666
+ url=url,
667
+ json=expected_res,
668
+ status=422,
669
+ match=[matchers.json_params_matcher(expected_request)],
670
+ )
671
+
672
+ with patch.dict(
673
+ os.environ,
674
+ {
675
+ "IGNORE_SIGHTING_BUILDER_WARNING": "1",
676
+ },
677
+ clear=True,
678
+ ):
679
+ with pytest.raises(ValueError) as exc:
680
+ datalake.Sightings.bulk_submit_sightings(sightings=[input_sighting])
681
+
682
+ prefix = "422 HTTP code: "
683
+ msg = str(exc.value)
684
+ assert msg.startswith(prefix)
685
+
686
+ payload = ast.literal_eval(msg[len(prefix) :])
687
+ assert payload == expected_res
688
+
689
+
451
690
  def test_sightings_filtered_bad_ordering(datalake):
452
691
  with pytest.raises(ValueError) as err:
453
692
  datalake.Sightings.sightings_filtered(ordering="bad_ordering")
@@ -16,6 +16,7 @@ from datalake_scripts.scripts import (
16
16
  get_atom_values,
17
17
  get_filtered_threat_entity,
18
18
  search_watch,
19
+ get_my_user_info,
19
20
  )
20
21
 
21
22
 
@@ -36,6 +37,7 @@ from datalake_scripts.scripts import (
36
37
  get_atom_values,
37
38
  get_filtered_threat_entity,
38
39
  search_watch,
40
+ get_my_user_info,
39
41
  ],
40
42
  )
41
43
  def test_name_function(function_script):