datalake-scripts 2.6.9__tar.gz → 2.6.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/PKG-INFO +5 -1
  2. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/README.md +5 -0
  3. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/endpoint.py +1 -1
  4. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/threats.py +12 -0
  5. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/cli.py +1 -1
  6. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/engines/get_engine.py +21 -1
  7. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_atom_values.py +10 -0
  8. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/PKG-INFO +5 -1
  9. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/SOURCES.txt +3 -2
  10. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_threats.py +36 -0
  11. datalake-scripts-2.6.10/tests/scripts/test_cli.py +29 -0
  12. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/__init__.py +0 -0
  13. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/api_objects/__init__.py +0 -0
  14. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/api_objects/bulk_search_task.py +0 -0
  15. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/__init__.py +0 -0
  16. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/atom.py +0 -0
  17. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/atom_type.py +0 -0
  18. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/config.py +0 -0
  19. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/logger.py +0 -0
  20. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/ouput.py +0 -0
  21. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/throttler.py +0 -0
  22. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/token_manager.py +0 -0
  23. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/utils.py +0 -0
  24. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/common/warn.py +0 -0
  25. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/datalake.py +0 -0
  26. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/__init__.py +0 -0
  27. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/advanced_search.py +0 -0
  28. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/bulk_search.py +0 -0
  29. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/sightings.py +0 -0
  30. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake/endpoints/tags.py +0 -0
  31. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/__init__.py +0 -0
  32. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/common/__init__.py +0 -0
  33. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/common/base_engine.py +0 -0
  34. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/common/base_script.py +0 -0
  35. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/common/mixins.py +0 -0
  36. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/engines/__init__.py +0 -0
  37. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/engines/post_engine.py +0 -0
  38. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/__init__.py +0 -0
  39. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  40. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/utils.py +0 -0
  41. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/__init__.py +0 -0
  42. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_comments.py +0 -0
  43. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_tags.py +0 -0
  44. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_threats.py +0 -0
  45. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/advanced_search.py +0 -0
  46. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  47. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/edit_score.py +0 -0
  48. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_query_hash.py +0 -0
  49. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
  50. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  51. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts/scripts/lookup_threats.py +0 -0
  52. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  53. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/entry_points.txt +0 -0
  54. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/requires.txt +0 -0
  55. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/top_level.txt +0 -0
  56. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/setup.cfg +0 -0
  57. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/setup.py +0 -0
  58. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/__init__.py +0 -0
  59. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/common/__init__.py +0 -0
  60. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/common/fixture.py +0 -0
  61. {datalake-scripts-2.6.9/tests/lib → datalake-scripts-2.6.10/tests/common}/test_output.py +0 -0
  62. {datalake-scripts-2.6.9/tests/scripts → datalake-scripts-2.6.10/tests/common}/test_utils.py +0 -0
  63. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/conftest.py +0 -0
  64. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/__init__.py +0 -0
  65. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_advanced_search.py +0 -0
  66. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_atom_type.py +0 -0
  67. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_auth_via_tokens.py +0 -0
  68. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_bulk_search.py +0 -0
  69. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_sightings.py +0 -0
  70. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/lib/test_tags.py +0 -0
  71. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/scripts/__init__.py +0 -0
  72. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/scripts/test_add_new_threats.py +0 -0
  73. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/scripts/test_base_engine.py +0 -0
  74. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/scripts/test_csv_builder.py +0 -0
  75. {datalake-scripts-2.6.9 → datalake-scripts-2.6.10}/tests/scripts/test_endpoint_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.6.9
3
+ Version: 2.6.10
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
80
80
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
81
81
 
82
82
 
83
+ ### Contributing
84
+
85
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
86
+
@@ -58,3 +58,8 @@ Parameters common and optional for all commands:
58
58
  > -q will quiet the verbosity of the program (but still show errors / warnings)
59
59
 
60
60
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
61
+
62
+
63
+ ### Contributing
64
+
65
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
@@ -67,7 +67,7 @@ class Endpoint:
67
67
  while tries_left > 0:
68
68
  headers["Authorization"] = self.token_manager.access_token
69
69
  logger.debug(self._pretty_debug_request(url, method, post_body, headers))
70
-
70
+
71
71
  response = self._send_request(
72
72
  url, method, headers, post_body, stream=stream
73
73
  )
@@ -459,6 +459,18 @@ class Threats(Endpoint):
459
459
  Get all atom values based on given source id list and from time range.
460
460
  """
461
461
  self.check_atom_values_params(source_id, normalized_timestamp_since, normalized_timestamp_until, output, output_path)
462
+
463
+ url = self._build_url_for_endpoint("sources")
464
+ url = url + "?limit=1000&description_only=true"
465
+ for source in source_id:
466
+ url = url + "&source_ids=" + source
467
+ response = self.datalake_requests(url, "get", self._get_headers())
468
+ response = parse_response(response)
469
+ if response["count"] != len(source_id):
470
+ valid_sources = [response["results"][x]["id"] for x in range(0,len(response["results"]))]
471
+ invalid_sources = [source_id[y] for y in range(0,len(source_id)) if source_id[y] not in valid_sources]
472
+ raise ValueError(f"The following sources are invalid : {invalid_sources}")
473
+
462
474
  payload = {
463
475
  "source_id": source_id,
464
476
  "normalized_timestamp_since": normalized_timestamp_since,
@@ -9,7 +9,7 @@ from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_s
9
9
 
10
10
  class Cli:
11
11
  CLI_NAME = "ocd-dtl"
12
- VERSION = "2.6.9"
12
+ VERSION = "2.6.10"
13
13
 
14
14
  def __init__(self):
15
15
  parser = argparse.ArgumentParser(
@@ -1,5 +1,5 @@
1
1
  """All the engines that use a GET endpoint."""
2
-
2
+ from typing import List, Tuple
3
3
  from requests import PreparedRequest
4
4
 
5
5
  from datalake.common.logger import logger
@@ -77,3 +77,23 @@ class Threats(GetEngine):
77
77
  headers = {"Accept": response_format}
78
78
  response = self.datalake_requests(req.url, "get", headers=headers)
79
79
  return response
80
+
81
+
82
+ class Sources(GetEngine):
83
+ """ Checks if a source exists"""
84
+
85
+ def _build_url(self, endpoint_config: dict, environment: str):
86
+ return self._build_url_for_endpoint("sources")
87
+
88
+ def check_sources(self, sources_list: List[str]) -> Tuple[bool, List[str]]:
89
+ url = self.url + "?limit=1000&description_only=true"
90
+ for source in sources_list:
91
+ url = url + "&source_ids=" + source
92
+ response = self.datalake_requests(url, "get", self._get_headers())
93
+ if response["count"] == len(sources_list):
94
+ return True, []
95
+ else :
96
+ valid_sources = [response["results"][x]["id"] for x in range(0, len(response["results"]))]
97
+ invalid_sources = [sources_list[y] for y in range(0, len(sources_list)) if sources_list[y] not in valid_sources]
98
+ return False, invalid_sources
99
+
@@ -6,6 +6,7 @@ from datalake.common.ouput import Output
6
6
  from datalake.common.token_manager import TokenManager
7
7
  from datalake.common.utils import check_normalized_timestamp, convert_date_to_normalized_timestamp
8
8
  from datalake_scripts.common.base_script import BaseScripts
9
+ from datalake_scripts.engines.get_engine import Sources
9
10
  from datalake_scripts.engines.post_engine import AtomSearch
10
11
  from datalake_scripts.helper_scripts.utils import load_list, save_output
11
12
 
@@ -83,6 +84,15 @@ def main(override_args=None):
83
84
  # Load api_endpoints and tokens
84
85
  endpoint_config = Config().load_config()
85
86
  token_manager = TokenManager(endpoint_config, environment=args.env)
87
+
88
+ # Check list of sources are valid
89
+ engine_check_sources = Sources(endpoint_config, args.env, token_manager)
90
+ all_sources_are_valid, list_invalid_sources = engine_check_sources.check_sources(sources_list)
91
+ if not all_sources_are_valid:
92
+ parser.error(
93
+ f"The following sources are invalid : {list_invalid_sources}"
94
+ )
95
+
86
96
  search_engine_atom_values = AtomSearch(endpoint_config, args.env, token_manager)
87
97
  if not args.output_type or args.output_type=="json":
88
98
  list_atom_values = search_engine_atom_values.get_atoms(sources_list, since_ts, until_ts, Output.JSON)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.6.9
3
+ Version: 2.6.10
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
80
80
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
81
81
 
82
82
 
83
+ ### Contributing
84
+
85
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
86
+
@@ -50,21 +50,22 @@ setup.py
50
50
  ./tests/conftest.py
51
51
  ./tests/common/__init__.py
52
52
  ./tests/common/fixture.py
53
+ ./tests/common/test_output.py
54
+ ./tests/common/test_utils.py
53
55
  ./tests/lib/__init__.py
54
56
  ./tests/lib/test_advanced_search.py
55
57
  ./tests/lib/test_atom_type.py
56
58
  ./tests/lib/test_auth_via_tokens.py
57
59
  ./tests/lib/test_bulk_search.py
58
- ./tests/lib/test_output.py
59
60
  ./tests/lib/test_sightings.py
60
61
  ./tests/lib/test_tags.py
61
62
  ./tests/lib/test_threats.py
62
63
  ./tests/scripts/__init__.py
63
64
  ./tests/scripts/test_add_new_threats.py
64
65
  ./tests/scripts/test_base_engine.py
66
+ ./tests/scripts/test_cli.py
65
67
  ./tests/scripts/test_csv_builder.py
66
68
  ./tests/scripts/test_endpoint_config.py
67
- ./tests/scripts/test_utils.py
68
69
  datalake_scripts.egg-info/PKG-INFO
69
70
  datalake_scripts.egg-info/SOURCES.txt
70
71
  datalake_scripts.egg-info/dependency_links.txt
@@ -635,8 +635,38 @@ def test_atom_values_bad_output(datalake: Datalake):
635
635
  assert str(err.value) == "output needs to either be set to JSON or CSV"
636
636
 
637
637
 
638
+ @responses.activate
639
+ def test_atom_values_bad_source(datalake: Datalake):
640
+ url = (
641
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
642
+ )
643
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
644
+ responses.add(responses.GET, url, json=get_resp, status=200)
645
+
646
+ url = (
647
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
648
+ )
649
+ post_resp = [
650
+ {"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
651
+ {"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
652
+ ]
653
+ responses.add(responses.POST, url, json=post_resp, status=200)
654
+
655
+ with pytest.raises(ValueError) as err:
656
+ datalake.Threats.atom_values(
657
+ ["source_a", "source_b"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
658
+ )
659
+ assert str(err.value) == "The following sources are invalid : ['source_b']"
660
+
661
+
638
662
  @responses.activate
639
663
  def test_atom_values(datalake: Datalake):
664
+ url = (
665
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
666
+ )
667
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
668
+ responses.add(responses.GET, url, json=get_resp, status=200)
669
+
640
670
  url = (
641
671
  "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
642
672
  )
@@ -652,6 +682,12 @@ def test_atom_values(datalake: Datalake):
652
682
 
653
683
  @responses.activate
654
684
  def test_preprod_atom_values():
685
+ url = (
686
+ "https://ti.extranet.mrti-center.com/api/v2/mrti/sources/"
687
+ )
688
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
689
+ responses.add(responses.GET, url, json=get_resp, status=200)
690
+
655
691
  url = "https://ti.extranet.mrti-center.com/api/v2/auth/token/"
656
692
  auth_response = {"access_token": "access_token", "refresh_token": "refresh_token"}
657
693
  responses.add(responses.POST, url, json=auth_response, status=200)
@@ -0,0 +1,29 @@
1
+ import sys
2
+ import pytest
3
+ from unittest.mock import patch
4
+ from datalake_scripts.cli import Cli
5
+ from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
6
+ add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values
7
+
8
+
9
+ # Tests to make sure the commands call the right scripts (and recognizes the command)
10
+ @pytest.mark.parametrize("function_script", [add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
11
+ add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values])
12
+ def test_name_function(function_script):
13
+ command_name = function_script.__name__.split('.')[-1]
14
+
15
+ # Special cases in cli.py:
16
+ if command_name == "add_comments":
17
+ command_name = "add_comment"
18
+ elif command_name == "get_threats_by_hashkey":
19
+ command_name = "get_threats"
20
+
21
+ with patch.object(function_script, 'main', return_value='mocked response') as mock_function_script:
22
+ # Act: Change sys.argv to simulate the command line arguments
23
+ with patch('sys.argv', ['ocd-dtl', command_name, '--output', 'output.txt', '--env', 'preprod']):
24
+ cli_app = Cli()
25
+
26
+ # Assert: ensure the mock was called and the result is as expected
27
+ mock_function_script.assert_called_once_with(['--output', 'output.txt', '--env', 'preprod'])
28
+
29
+