datalake-scripts 3.0.0rc2__tar.gz → 3.0.0rc4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/PKG-INFO +1 -1
  2. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/datalake.py +9 -0
  3. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/filtered_threat_entity.py +7 -2
  4. datalake_scripts-3.0.0rc4/datalake/endpoints/my_account.py +17 -0
  5. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/cli.py +11 -3
  6. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/get_filtered_threat_entity.py +14 -1
  7. datalake_scripts-3.0.0rc4/datalake_scripts/scripts/get_my_user_info.py +32 -0
  8. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/PKG-INFO +1 -1
  9. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/SOURCES.txt +3 -0
  10. datalake_scripts-3.0.0rc4/tests/lib/test_my_account.py +51 -0
  11. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/scripts/test_cli.py +2 -0
  12. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/README.md +0 -0
  13. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/__init__.py +0 -0
  14. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/__init__.py +0 -0
  15. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/atom.py +0 -0
  16. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/atom_type.py +0 -0
  17. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/bulk_search_task.py +0 -0
  18. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/config.py +0 -0
  19. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/logger.py +0 -0
  20. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/output.py +0 -0
  21. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/throttler.py +0 -0
  22. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/token_manager.py +0 -0
  23. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/utils.py +0 -0
  24. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/common/warn.py +0 -0
  25. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/__init__.py +0 -0
  26. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/advanced_search.py +0 -0
  27. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/bulk_search.py +0 -0
  28. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/comments.py +0 -0
  29. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/endpoint.py +0 -0
  30. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/sightings.py +0 -0
  31. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/sources.py +0 -0
  32. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/tags.py +0 -0
  33. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/endpoints/threats.py +0 -0
  34. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/miscellaneous/__init__.py +0 -0
  35. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake/miscellaneous/search_watch.py +0 -0
  36. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/__init__.py +0 -0
  37. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/common/__init__.py +0 -0
  38. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/common/base_script.py +0 -0
  39. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/helper_scripts/__init__.py +0 -0
  40. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  41. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/helper_scripts/utils.py +0 -0
  42. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/__init__.py +0 -0
  43. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/add_comments.py +0 -0
  44. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/add_tags.py +0 -0
  45. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/add_threats.py +0 -0
  46. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/advanced_search.py +0 -0
  47. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  48. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/edit_score.py +0 -0
  49. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/get_atom_values.py +0 -0
  50. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/get_query_hash.py +0 -0
  51. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/get_threats.py +0 -0
  52. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  53. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/lookup_threats.py +0 -0
  54. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts/scripts/search_watch.py +0 -0
  55. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  56. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/entry_points.txt +0 -0
  57. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/requires.txt +0 -0
  58. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/datalake_scripts.egg-info/top_level.txt +0 -0
  59. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/setup.cfg +0 -0
  60. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/setup.py +0 -0
  61. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/__init__.py +0 -0
  62. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/common/__init__.py +0 -0
  63. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/common/fixture.py +0 -0
  64. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/common/test_output.py +0 -0
  65. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/common/test_utils.py +0 -0
  66. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/conftest.py +0 -0
  67. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/__init__.py +0 -0
  68. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_advanced_search.py +0 -0
  69. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_atom_type.py +0 -0
  70. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_auth_via_tokens.py +0 -0
  71. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_bulk_search.py +0 -0
  72. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_filtered_threat_entity.py +0 -0
  73. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_search_watch.py +0 -0
  74. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_sightings.py +0 -0
  75. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_tags.py +0 -0
  76. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/lib/test_threats.py +0 -0
  77. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/scripts/__init__.py +0 -0
  78. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/scripts/test_add_new_threats.py +0 -0
  79. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/scripts/test_base_scripts.py +0 -0
  80. {datalake_scripts-3.0.0rc2 → datalake_scripts-3.0.0rc4}/tests/scripts/test_csv_builder.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake_scripts
3
- Version: 3.0.0rc2
3
+ Version: 3.0.0rc4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -11,6 +11,7 @@ from datalake.endpoints.advanced_search import AdvancedSearch
11
11
  from datalake.endpoints.sightings import Sightings
12
12
  from datalake.endpoints.sources import Sources
13
13
  from datalake.endpoints.filtered_threat_entity import FilteredThreatEntity
14
+ from datalake.endpoints.my_account import MyAccount
14
15
  from datalake.miscellaneous.search_watch import SearchWatch
15
16
 
16
17
 
@@ -58,6 +59,14 @@ class Datalake:
58
59
  raise
59
60
 
60
61
  # Endpoints
62
+ self.MyAccount = MyAccount(
63
+ self.logger,
64
+ endpoint_config,
65
+ env,
66
+ token_manager,
67
+ proxies=proxies,
68
+ verify=verify,
69
+ )
61
70
  self.AdvancedSearch = AdvancedSearch(
62
71
  self.logger,
63
72
  endpoint_config,
@@ -1,10 +1,11 @@
1
1
  from datalake.endpoints.endpoint import Endpoint
2
- from datalake.common.output import parse_response
2
+ from datalake.common.output import parse_response, Output, output_supported
3
3
  import re
4
4
  import sys
5
5
 
6
6
 
7
7
  class FilteredThreatEntity(Endpoint):
8
+ @output_supported({Output.JSON, Output.STIX})
8
9
  def get_filtered_and_sorted_list(
9
10
  self,
10
11
  threat_category_name=None,
@@ -17,6 +18,7 @@ class FilteredThreatEntity(Endpoint):
17
18
  ordering="threat_category_name",
18
19
  stix_uuid=None,
19
20
  tag=None,
21
+ output=Output.JSON,
20
22
  ):
21
23
  """
22
24
  Retrieve threat entities, with available filtering options as input parameters
@@ -46,7 +48,10 @@ class FilteredThreatEntity(Endpoint):
46
48
 
47
49
  try:
48
50
  response = self.datalake_requests(
49
- url=url, method="post", headers=self._post_headers(), post_body=body
51
+ url=url,
52
+ method="post",
53
+ headers=self._post_headers(output=output),
54
+ post_body=body,
50
55
  )
51
56
  return parse_response(response)
52
57
  except ValueError as ve:
@@ -0,0 +1,17 @@
1
+ from typing import List, Tuple
2
+
3
+ from datalake.endpoints.endpoint import Endpoint
4
+ from datalake.common.output import parse_response
5
+
6
+
7
+ class MyAccount(Endpoint):
8
+
9
+ def me(self):
10
+ """
11
+ Gets details of the currently logged in user.
12
+ """
13
+ url = self._build_url_for_endpoint("users-me")
14
+ response = parse_response(
15
+ self.datalake_requests(url, "get", self._get_headers())
16
+ )
17
+ return response
@@ -15,12 +15,13 @@ from datalake_scripts.scripts import (
15
15
  get_atom_values,
16
16
  get_filtered_threat_entity,
17
17
  search_watch,
18
+ get_my_user_info,
18
19
  )
19
20
 
20
21
 
21
22
  class Cli:
22
23
  CLI_NAME = "ocd-dtl"
23
- VERSION = "3.0.0rc2"
24
+ VERSION = "3.0.0rc4"
24
25
 
25
26
  def __init__(self):
26
27
  parser = argparse.ArgumentParser(
@@ -54,8 +55,15 @@ class Cli:
54
55
  get_filtered_threat_entity.main,
55
56
  )
56
57
  self._add_command_subparser(subparsers, "search_watch", search_watch.main)
58
+ self._add_command_subparser(
59
+ subparsers, "get_my_user_info", get_my_user_info.main
60
+ )
61
+
62
+ # args = parser.parse_args(sys.argv[1:2])
63
+ command_called = sys.argv[1:2]
64
+ command_arguments = sys.argv[2:]
57
65
 
58
- args = parser.parse_args(sys.argv[1:2])
66
+ args = parser.parse_args(command_called)
59
67
 
60
68
  if args.version:
61
69
  print(self.VERSION)
@@ -67,7 +75,7 @@ class Cli:
67
75
  exit(1)
68
76
 
69
77
  # Call the subcommand method
70
- args.func(sys.argv[2:])
78
+ args.func(command_arguments)
71
79
 
72
80
  def _add_command_subparser(self, subparsers, name, method):
73
81
  command_parser = subparsers.add_parser(name)
@@ -1,6 +1,6 @@
1
1
  import sys
2
2
  import json
3
- from datalake import Datalake
3
+ from datalake import Datalake, Output
4
4
  from datalake_scripts.common.base_script import BaseScripts
5
5
 
6
6
 
@@ -24,6 +24,9 @@ def main(args=None):
24
24
  parser.add_argument(
25
25
  "-t", "--tag", help="Filter Tag associated with threat entity", default=None
26
26
  )
27
+ parser.add_argument(
28
+ "-ot", "--output-type", help="Desired output type", default=None
29
+ )
27
30
 
28
31
  if args is None:
29
32
  args = parser.parse_args()
@@ -31,6 +34,15 @@ def main(args=None):
31
34
  args = parser.parse_args(args)
32
35
 
33
36
  dtl = Datalake(env=args.env, log_level=args.loglevel)
37
+
38
+ output_type = Output.JSON
39
+ if args.output_type:
40
+ try:
41
+ output_type = Output[args.output_type.upper()]
42
+ except KeyError:
43
+ dtl.logger.error("Not supported output, please use either json or stix")
44
+ exit(1)
45
+
34
46
  threat_entities = dtl.FilteredThreatEntity.get_filtered_and_sorted_list(
35
47
  threat_category_name=args.threat_category_name,
36
48
  alias=args.alias,
@@ -41,6 +53,7 @@ def main(args=None):
41
53
  ordering=args.ordering,
42
54
  stix_uuid=args.stix_uuid,
43
55
  tag=args.tag,
56
+ output=output_type,
44
57
  )
45
58
 
46
59
  if args.output and threat_entities is not None:
@@ -0,0 +1,32 @@
1
+ import sys
2
+
3
+ from datalake import Datalake
4
+ from datalake_scripts.common.base_script import BaseScripts
5
+ from datalake_scripts.helper_scripts.utils import save_output
6
+
7
+
8
+ def get_my_user_info_func(dtl: Datalake):
9
+ response = dtl.MyAccount.me()
10
+ return response
11
+
12
+
13
+ def main(override_args=None):
14
+ """Method to start the script"""
15
+ # Load initial args
16
+ parser = BaseScripts.start("Gets details of the currently logged in user.")
17
+
18
+ args = parser.parse_args(override_args or [])
19
+
20
+ dtl = Datalake(env=args.env, log_level=args.loglevel)
21
+ response = get_my_user_info_func(dtl)
22
+
23
+ if args.output:
24
+ save_output(args.output, response)
25
+ dtl.logger.debug(f"Results saved in {args.output}\n")
26
+ else:
27
+ print(response)
28
+ dtl.logger.debug(f"END: get_my_user_info.py")
29
+
30
+
31
+ if __name__ == "__main__":
32
+ sys.exit(main())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 3.0.0rc2
3
+ Version: 3.0.0rc4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -20,6 +20,7 @@ setup.py
20
20
  ./datalake/endpoints/comments.py
21
21
  ./datalake/endpoints/endpoint.py
22
22
  ./datalake/endpoints/filtered_threat_entity.py
23
+ ./datalake/endpoints/my_account.py
23
24
  ./datalake/endpoints/sightings.py
24
25
  ./datalake/endpoints/sources.py
25
26
  ./datalake/endpoints/tags.py
@@ -42,6 +43,7 @@ setup.py
42
43
  ./datalake_scripts/scripts/edit_score.py
43
44
  ./datalake_scripts/scripts/get_atom_values.py
44
45
  ./datalake_scripts/scripts/get_filtered_threat_entity.py
46
+ ./datalake_scripts/scripts/get_my_user_info.py
45
47
  ./datalake_scripts/scripts/get_query_hash.py
46
48
  ./datalake_scripts/scripts/get_threats.py
47
49
  ./datalake_scripts/scripts/get_threats_from_query_hash.py
@@ -59,6 +61,7 @@ setup.py
59
61
  ./tests/lib/test_auth_via_tokens.py
60
62
  ./tests/lib/test_bulk_search.py
61
63
  ./tests/lib/test_filtered_threat_entity.py
64
+ ./tests/lib/test_my_account.py
62
65
  ./tests/lib/test_search_watch.py
63
66
  ./tests/lib/test_sightings.py
64
67
  ./tests/lib/test_tags.py
@@ -0,0 +1,51 @@
1
+ import pytest
2
+ import responses
3
+ from datalake import Datalake
4
+ from tests.common.fixture import TestData, datalake
5
+
6
+ mock_results_data = {
7
+ "email": "user@example.com",
8
+ "first_name": "string",
9
+ "force_password_change": True,
10
+ "full_name": "string",
11
+ "has_password": True,
12
+ "id": 0,
13
+ "is_active": True,
14
+ "is_deleted": False,
15
+ "last_login": "2025-10-15T08:43:33.390Z",
16
+ "last_name": "string",
17
+ "organization": {"id": 0, "name": "string", "path_names": ["string"]},
18
+ "previous_login": "2025-10-15T08:43:33.390Z",
19
+ "request_limit": 0,
20
+ "role": {
21
+ "administration_permissions": {"description": "string", "name": "string"},
22
+ "id": 0,
23
+ "name": "string",
24
+ },
25
+ "totp_enabled": True,
26
+ }
27
+
28
+
29
+ def mock_api_response():
30
+ responses.add(
31
+ responses.GET,
32
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
33
+ + TestData.TEST_CONFIG["api_version"]
34
+ + TestData.TEST_CONFIG["endpoints"]["users-me"],
35
+ json=mock_results_data, # Only return the "results" part
36
+ status=200,
37
+ )
38
+
39
+
40
+ @responses.activate
41
+ def test_my_account(datalake: Datalake):
42
+ mock_api_response()
43
+ response = datalake.MyAccount.me()
44
+ assert response == mock_results_data
45
+ assert isinstance(response, dict), "Results should be a dict"
46
+ assert responses.calls[0].request.url, (
47
+ TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
48
+ + TestData.TEST_CONFIG["api_version"]
49
+ + TestData.TEST_CONFIG["endpoints"]["users-me"]
50
+ )
51
+ assert responses.calls[0].request.method, "GET"
@@ -16,6 +16,7 @@ from datalake_scripts.scripts import (
16
16
  get_atom_values,
17
17
  get_filtered_threat_entity,
18
18
  search_watch,
19
+ get_my_user_info,
19
20
  )
20
21
 
21
22
 
@@ -36,6 +37,7 @@ from datalake_scripts.scripts import (
36
37
  get_atom_values,
37
38
  get_filtered_threat_entity,
38
39
  search_watch,
40
+ get_my_user_info,
39
41
  ],
40
42
  )
41
43
  def test_name_function(function_script):