datalake-scripts 2.6.8__tar.gz → 2.6.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/PKG-INFO +5 -1
  2. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/README.md +5 -0
  3. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/utils.py +47 -1
  4. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/endpoint.py +1 -1
  5. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/threats.py +53 -1
  6. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/cli.py +6 -2
  7. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/get_engine.py +21 -1
  8. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/post_engine.py +22 -2
  9. datalake-scripts-2.6.10/datalake_scripts/scripts/get_atom_values.py +109 -0
  10. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/PKG-INFO +5 -1
  11. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/SOURCES.txt +4 -2
  12. datalake-scripts-2.6.10/tests/common/test_utils.py +49 -0
  13. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/conftest.py +1 -1
  14. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_threats.py +135 -0
  15. datalake-scripts-2.6.10/tests/scripts/test_cli.py +29 -0
  16. datalake-scripts-2.6.8/tests/scripts/test_utils.py +0 -37
  17. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/__init__.py +0 -0
  18. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/api_objects/__init__.py +0 -0
  19. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/api_objects/bulk_search_task.py +0 -0
  20. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/__init__.py +0 -0
  21. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/atom.py +0 -0
  22. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/atom_type.py +0 -0
  23. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/config.py +0 -0
  24. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/logger.py +0 -0
  25. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/ouput.py +0 -0
  26. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/throttler.py +0 -0
  27. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/token_manager.py +0 -0
  28. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/warn.py +0 -0
  29. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/datalake.py +0 -0
  30. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/__init__.py +0 -0
  31. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/advanced_search.py +0 -0
  32. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/bulk_search.py +0 -0
  33. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/sightings.py +0 -0
  34. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/tags.py +0 -0
  35. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/__init__.py +0 -0
  36. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/__init__.py +0 -0
  37. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/base_engine.py +0 -0
  38. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/base_script.py +0 -0
  39. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/mixins.py +0 -0
  40. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/__init__.py +0 -0
  41. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/__init__.py +0 -0
  42. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  43. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/utils.py +0 -0
  44. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/__init__.py +0 -0
  45. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_comments.py +0 -0
  46. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_tags.py +0 -0
  47. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_threats.py +0 -0
  48. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/advanced_search.py +0 -0
  49. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  50. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/edit_score.py +0 -0
  51. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_query_hash.py +0 -0
  52. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
  53. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  54. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/lookup_threats.py +0 -0
  55. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  56. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/entry_points.txt +0 -0
  57. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/requires.txt +0 -0
  58. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/top_level.txt +0 -0
  59. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/setup.cfg +0 -0
  60. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/setup.py +0 -0
  61. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/__init__.py +0 -0
  62. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/common/__init__.py +0 -0
  63. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/common/fixture.py +0 -0
  64. {datalake-scripts-2.6.8/tests/lib → datalake-scripts-2.6.10/tests/common}/test_output.py +0 -0
  65. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/__init__.py +0 -0
  66. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_advanced_search.py +0 -0
  67. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_atom_type.py +0 -0
  68. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_auth_via_tokens.py +0 -0
  69. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_bulk_search.py +0 -0
  70. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_sightings.py +0 -0
  71. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_tags.py +0 -0
  72. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/__init__.py +0 -0
  73. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_add_new_threats.py +0 -0
  74. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_base_engine.py +0 -0
  75. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_csv_builder.py +0 -0
  76. {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_endpoint_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.6.8
3
+ Version: 2.6.10
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
80
80
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
81
81
 
82
82
 
83
+ ### Contributing
84
+
85
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
86
+
@@ -58,3 +58,8 @@ Parameters common and optional for all commands:
58
58
  > -q will quiet the verbosity of the program (but still show errors / warnings)
59
59
 
60
60
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
61
+
62
+
63
+ ### Contributing
64
+
65
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
@@ -1,4 +1,5 @@
1
- import datetime
1
+ import datetime
2
+ import json
2
3
  from collections import defaultdict
3
4
  from typing import Dict, Optional, Generator
4
5
 
@@ -51,3 +52,48 @@ def get_error_message(resp_body: dict):
51
52
  return resp_body.get("messages")
52
53
  else:
53
54
  raise ValueError("no error message in api response")
55
+
56
+ def save_output(file_name: str, data):
57
+ """
58
+ Save the data in a file.
59
+ If data is dict, file format will be JSON.
60
+ If data is a list, file format will be txt.
61
+ Else it will be saved as it comes.
62
+ """
63
+ with open(file_name, "w") as file_to_write:
64
+ if isinstance(data, dict):
65
+ file_to_write.write(json.dumps(data, sort_keys=True, indent=2))
66
+ elif isinstance(data, list):
67
+ for item in data:
68
+ file_to_write.write(f"{item}\n")
69
+ else:
70
+ file_to_write.write(data)
71
+
72
+ def check_normalized_timestamp(ts: str) -> bool:
73
+ """
74
+ Check if the provided timestamp is in a normalized format
75
+ """
76
+ try :
77
+ if ts != datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]+'Z':
78
+ raise ValueError
79
+ return True
80
+ except ValueError:
81
+ return False
82
+
83
+ def convert_date_to_normalized_timestamp(date_input : str, start : bool) -> str:
84
+ """
85
+ Convert a given date into a normalized timestamp format
86
+ Either at the begining of the day (start at True) or at the end (start at False)
87
+ """
88
+ try :
89
+ if start:
90
+ date_ts=date_input+'T00:00:00.000Z'
91
+ else:
92
+ date_ts=date_input+'T23:59:59.999Z'
93
+ if check_normalized_timestamp(date_ts):
94
+ return date_ts
95
+ else:
96
+ raise ValueError
97
+ except:
98
+ raise ValueError
99
+
@@ -67,7 +67,7 @@ class Endpoint:
67
67
  while tries_left > 0:
68
68
  headers["Authorization"] = self.token_manager.access_token
69
69
  logger.debug(self._pretty_debug_request(url, method, post_body, headers))
70
-
70
+
71
71
  response = self._send_request(
72
72
  url, method, headers, post_body, stream=stream
73
73
  )
@@ -7,7 +7,7 @@ from requests.sessions import PreparedRequest
7
7
  from datalake import AtomType, ThreatType, OverrideType, Atom
8
8
  from datalake.common.atom import ScoreMap
9
9
  from datalake.common.ouput import Output, output_supported, parse_response
10
- from datalake.common.utils import split_list, aggregate_csv_or_json_api_response
10
+ from datalake.common.utils import split_list, aggregate_csv_or_json_api_response, save_output, check_normalized_timestamp
11
11
  from datalake.endpoints.endpoint import Endpoint
12
12
 
13
13
 
@@ -432,3 +432,55 @@ class Threats(Endpoint):
432
432
  url = self._build_url_for_endpoint("threats-manual")
433
433
  response = self.datalake_requests(url, "post", self._post_headers(), payload)
434
434
  return parse_response(response)
435
+
436
+
437
+ @staticmethod
438
+ def check_atom_values_params(source_id, normalized_timestamp_since, normalized_timestamp_until, output, output_path):
439
+ if not source_id or len(source_id)<1:
440
+ raise ValueError("A list of minimum one source id required")
441
+ if not normalized_timestamp_since or not normalized_timestamp_until:
442
+ raise ValueError("Both timestamps parameters are required")
443
+ if not check_normalized_timestamp(normalized_timestamp_since):
444
+ raise ValueError("Invalid normalized_timestamp_since format")
445
+ if not check_normalized_timestamp(normalized_timestamp_until):
446
+ raise ValueError("Invalid normalized_timestamp_until format")
447
+ if output!=Output.JSON and output!=Output.CSV:
448
+ raise TypeError("output needs to either be set to JSON or CSV")
449
+
450
+ def atom_values(
451
+ self,
452
+ source_id: List = None,
453
+ normalized_timestamp_since: str = None,
454
+ normalized_timestamp_until: str = None,
455
+ output: Output = Output.JSON,
456
+ output_path: str = None
457
+ ):
458
+ """
459
+ Get all atom values based on given source id list and from time range.
460
+ """
461
+ self.check_atom_values_params(source_id, normalized_timestamp_since, normalized_timestamp_until, output, output_path)
462
+
463
+ url = self._build_url_for_endpoint("sources")
464
+ url = url + "?limit=1000&description_only=true"
465
+ for source in source_id:
466
+ url = url + "&source_ids=" + source
467
+ response = self.datalake_requests(url, "get", self._get_headers())
468
+ response = parse_response(response)
469
+ if response["count"] != len(source_id):
470
+ valid_sources = [response["results"][x]["id"] for x in range(0,len(response["results"]))]
471
+ invalid_sources = [source_id[y] for y in range(0,len(source_id)) if source_id[y] not in valid_sources]
472
+ raise ValueError(f"The following sources are invalid : {invalid_sources}")
473
+
474
+ payload = {
475
+ "source_id": source_id,
476
+ "normalized_timestamp_since": normalized_timestamp_since,
477
+ "normalized_timestamp_until": normalized_timestamp_until,
478
+ "ordering": ["atom_type"]
479
+ }
480
+ url = self._build_url_for_endpoint("threats-atom-values")
481
+ response = self.datalake_requests(url, "post", self._post_headers(output), payload)
482
+
483
+ if output_path:
484
+ save_output(output_path, parse_response(response))
485
+
486
+ return parse_response(response)
@@ -4,12 +4,12 @@ import argparse
4
4
  import sys
5
5
 
6
6
  from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
7
- add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search
7
+ add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values
8
8
 
9
9
 
10
10
  class Cli:
11
11
  CLI_NAME = "ocd-dtl"
12
- VERSION = "2.6.8"
12
+ VERSION = "2.6.10"
13
13
 
14
14
  def __init__(self):
15
15
  parser = argparse.ArgumentParser(
@@ -51,6 +51,10 @@ The most commonly used {self.CLI_NAME} commands are:
51
51
  def get_threats(self):
52
52
  args = sys.argv[2:]
53
53
  get_threats_by_hashkey.main(args)
54
+
55
+ def get_atom_values(self):
56
+ args = sys.argv[2:]
57
+ get_atom_values.main(args)
54
58
 
55
59
  def get_threats_from_query_hash(self):
56
60
  args = sys.argv[2:]
@@ -1,5 +1,5 @@
1
1
  """All the engines that use a GET endpoint."""
2
-
2
+ from typing import List, Tuple
3
3
  from requests import PreparedRequest
4
4
 
5
5
  from datalake.common.logger import logger
@@ -77,3 +77,23 @@ class Threats(GetEngine):
77
77
  headers = {"Accept": response_format}
78
78
  response = self.datalake_requests(req.url, "get", headers=headers)
79
79
  return response
80
+
81
+
82
+ class Sources(GetEngine):
83
+ """ Checks if a source exists"""
84
+
85
+ def _build_url(self, endpoint_config: dict, environment: str):
86
+ return self._build_url_for_endpoint("sources")
87
+
88
+ def check_sources(self, sources_list: List[str]) -> Tuple[bool, List[str]]:
89
+ url = self.url + "?limit=1000&description_only=true"
90
+ for source in sources_list:
91
+ url = url + "&source_ids=" + source
92
+ response = self.datalake_requests(url, "get", self._get_headers())
93
+ if response["count"] == len(sources_list):
94
+ return True, []
95
+ else :
96
+ valid_sources = [response["results"][x]["id"] for x in range(0, len(response["results"]))]
97
+ invalid_sources = [sources_list[y] for y in range(0, len(sources_list)) if sources_list[y] not in valid_sources]
98
+ return False, invalid_sources
99
+
@@ -5,6 +5,7 @@ from typing import Set, Dict, List, Union
5
5
  from requests import PreparedRequest
6
6
 
7
7
  from datalake.common.logger import logger
8
+ from datalake.common.ouput import Output, parse_response
8
9
  from datalake.endpoints import Endpoint
9
10
  from datalake_scripts.common.base_engine import BaseEngine
10
11
  from datalake_scripts.common.mixins import HandleBulkTaskMixin
@@ -58,8 +59,8 @@ class PostEngine(BaseEngine):
58
59
  if entry[key] == value:
59
60
  return entry
60
61
 
61
- def _post_headers(self) -> dict:
62
- return {"Accept": "application/json", "Content-Type": "application/json"}
62
+ def _post_headers(self, output=Output.JSON ) -> dict:
63
+ return {"Accept": output.value, "Content-Type": "application/json"}
63
64
 
64
65
  @staticmethod
65
66
  def build_full_query_body(query_body):
@@ -176,3 +177,22 @@ class AtomValuesExtractor(PostEngine):
176
177
  "treat_hashes_like": treat_hashes_like,
177
178
  }
178
179
  return self.datalake_requests(self.url, "post", self._post_headers(), payload)
180
+
181
+
182
+ class AtomSearch(PostEngine):
183
+ """Atom values by sources and time range"""
184
+ def _build_url(self, endpoint_config: dict, environment: str):
185
+ return self._build_url_for_endpoint("threats-atom-values")
186
+
187
+ def get_atoms(
188
+ self, sources_list: List[str], since_ts: str, until_ts: str, type: Output
189
+ ) -> dict:
190
+ payload = {
191
+ "source_id": sources_list,
192
+ "normalized_timestamp_since": since_ts,
193
+ "normalized_timestamp_until": until_ts,
194
+ "ordering": ["atom_type"]
195
+ }
196
+ return self.datalake_requests(self.url, "post", self._post_headers(output=type), payload)
197
+
198
+
@@ -0,0 +1,109 @@
1
+ import sys
2
+
3
+ from datalake.common.config import Config
4
+ from datalake.common.logger import logger, configure_logging
5
+ from datalake.common.ouput import Output
6
+ from datalake.common.token_manager import TokenManager
7
+ from datalake.common.utils import check_normalized_timestamp, convert_date_to_normalized_timestamp
8
+ from datalake_scripts.common.base_script import BaseScripts
9
+ from datalake_scripts.engines.get_engine import Sources
10
+ from datalake_scripts.engines.post_engine import AtomSearch
11
+ from datalake_scripts.helper_scripts.utils import load_list, save_output
12
+
13
+
14
+ def main(override_args=None):
15
+ """Method to start the script"""
16
+ logger.debug(f"START: get_atom_values.py")
17
+
18
+ # Load initial args
19
+ parser = BaseScripts.start(
20
+ "Retrieve atom values from a list of sources ids and a time range"
21
+ )
22
+ parser.add_argument(
23
+ "source_id",
24
+ help="sources ids of the atom values to retreive",
25
+ nargs="*",
26
+ )
27
+ parser.add_argument(
28
+ "-i",
29
+ "--input_file",
30
+ help="list of sources ids (hashkeys) that need to be retrieved",
31
+ )
32
+ parser.add_argument(
33
+ "--since",
34
+ help="normalized timestamp (or date) to set up the start of the search of atom values",
35
+ )
36
+ parser.add_argument(
37
+ "--until",
38
+ help="normalized timestamp (or date) to set up the end of the search of atom values",
39
+ )
40
+ parser.add_argument(
41
+ "-ot",
42
+ "--output_type",
43
+ default="json",
44
+ help="set to the output type desired {json,csv}. Default is json if not specified",
45
+ )
46
+ if override_args:
47
+ args = parser.parse_args(override_args)
48
+ else:
49
+ args = parser.parse_args()
50
+ configure_logging(args.loglevel)
51
+
52
+ if not args.source_id and not args.input_file:
53
+ parser.error("either a source_id or an input_file is required")
54
+ sources_list = load_list(args.input_file) if args.input_file else args.source_id
55
+ logger.debug(f"TOTAL: {len(sources_list)} sources found")
56
+
57
+ if not args.since or not args.until:
58
+ parser.error("Both timestamps since and until are required")
59
+
60
+ # Check list of sources is not empty
61
+ if len(sources_list) <1:
62
+ parser.error("At least one source id is required to perform the atom search")
63
+
64
+ # Check given timestamps are correct
65
+ if not check_normalized_timestamp(args.since):
66
+ try:
67
+ since_ts=convert_date_to_normalized_timestamp(args.since, True)
68
+ except:
69
+ parser.error(
70
+ f"Since Timestamp/Date provided is incorrect : {args.since}, expected format is '%Y-%m-%dT%H:%M:%S.%f[:3]Z' or '%Y-%m-%d' "
71
+ )
72
+ else:
73
+ since_ts=args.since
74
+ if not check_normalized_timestamp(args.until):
75
+ try:
76
+ until_ts=convert_date_to_normalized_timestamp(args.until, False)
77
+ except:
78
+ parser.error(
79
+ f"Until Timestamp/Date provided is incorrect : {args.until}, expected format is '%Y-%m-%dT%H:%M:%S.%f[:3]Z' or '%Y-%m-%d' "
80
+ )
81
+ else:
82
+ until_ts=args.until
83
+
84
+ # Load api_endpoints and tokens
85
+ endpoint_config = Config().load_config()
86
+ token_manager = TokenManager(endpoint_config, environment=args.env)
87
+
88
+ # Check list of sources are valid
89
+ engine_check_sources = Sources(endpoint_config, args.env, token_manager)
90
+ all_sources_are_valid, list_invalid_sources = engine_check_sources.check_sources(sources_list)
91
+ if not all_sources_are_valid:
92
+ parser.error(
93
+ f"The following sources are invalid : {list_invalid_sources}"
94
+ )
95
+
96
+ search_engine_atom_values = AtomSearch(endpoint_config, args.env, token_manager)
97
+ if not args.output_type or args.output_type=="json":
98
+ list_atom_values = search_engine_atom_values.get_atoms(sources_list, since_ts, until_ts, Output.JSON)
99
+ else:
100
+ list_atom_values = search_engine_atom_values.get_atoms(sources_list, since_ts, until_ts, Output.CSV)
101
+
102
+ if args.output:
103
+ save_output(args.output, list_atom_values)
104
+ logger.debug(f"Atom values saved in {args.output}\n")
105
+ logger.debug(f"END: get_atom_values.py")
106
+
107
+
108
+ if __name__ == "__main__":
109
+ sys.exit(main())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.6.8
3
+ Version: 2.6.10
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
80
80
  For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
81
81
 
82
82
 
83
+ ### Contributing
84
+
85
+ To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
86
+
@@ -41,6 +41,7 @@ setup.py
41
41
  ./datalake_scripts/scripts/advanced_search.py
42
42
  ./datalake_scripts/scripts/bulk_lookup_threats.py
43
43
  ./datalake_scripts/scripts/edit_score.py
44
+ ./datalake_scripts/scripts/get_atom_values.py
44
45
  ./datalake_scripts/scripts/get_query_hash.py
45
46
  ./datalake_scripts/scripts/get_threats_by_hashkey.py
46
47
  ./datalake_scripts/scripts/get_threats_from_query_hash.py
@@ -49,21 +50,22 @@ setup.py
49
50
  ./tests/conftest.py
50
51
  ./tests/common/__init__.py
51
52
  ./tests/common/fixture.py
53
+ ./tests/common/test_output.py
54
+ ./tests/common/test_utils.py
52
55
  ./tests/lib/__init__.py
53
56
  ./tests/lib/test_advanced_search.py
54
57
  ./tests/lib/test_atom_type.py
55
58
  ./tests/lib/test_auth_via_tokens.py
56
59
  ./tests/lib/test_bulk_search.py
57
- ./tests/lib/test_output.py
58
60
  ./tests/lib/test_sightings.py
59
61
  ./tests/lib/test_tags.py
60
62
  ./tests/lib/test_threats.py
61
63
  ./tests/scripts/__init__.py
62
64
  ./tests/scripts/test_add_new_threats.py
63
65
  ./tests/scripts/test_base_engine.py
66
+ ./tests/scripts/test_cli.py
64
67
  ./tests/scripts/test_csv_builder.py
65
68
  ./tests/scripts/test_endpoint_config.py
66
- ./tests/scripts/test_utils.py
67
69
  datalake_scripts.egg-info/PKG-INFO
68
70
  datalake_scripts.egg-info/SOURCES.txt
69
71
  datalake_scripts.egg-info/dependency_links.txt
@@ -0,0 +1,49 @@
1
+ from datalake.common.utils import join_dicts, check_normalized_timestamp, convert_date_to_normalized_timestamp
2
+ from datalake_scripts.helper_scripts.utils import split_list
3
+
4
+
5
+ def test_split_list():
6
+ list_to_split = [1, 2, 3, 4, 5]
7
+
8
+ generator = split_list(list_to_split, 3)
9
+ head = next(generator)
10
+ tail = next(generator)
11
+
12
+ assert head == [1, 2, 3]
13
+ assert tail == [4, 5]
14
+
15
+
16
+ def test_join_dicts():
17
+ d1 = {
18
+ "k1": ["k1", "k2", "k3"],
19
+ "a1": ["a1", "a2", "a3"],
20
+ "b1": ["b1", "b2"],
21
+ }
22
+ d2 = {
23
+ "k1": ["k4", "k5", "k6"],
24
+ "a1": ["a4", "a5", "a6"],
25
+ }
26
+ d3 = {
27
+ "k1": ["k7", "k8"],
28
+ "a1": ["a7", "a8"],
29
+ "p1": ["p1", "p2"],
30
+ }
31
+
32
+ assert join_dicts(d1, d2, d3) == {
33
+ "k1": ["k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8"],
34
+ "a1": ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8"],
35
+ "b1": ["b1", "b2"],
36
+ "p1": ["p1", "p2"],
37
+ }
38
+
39
+ def test_normalized_timestamp():
40
+ assert check_normalized_timestamp("2023-09-14T15:00:00.000Z") == True
41
+ assert check_normalized_timestamp("2023-09-15T15:12:13.825Z") == True
42
+ assert check_normalized_timestamp("2023-9-15T15:12:13.825Z") == False
43
+ assert check_normalized_timestamp("2023-09-15T15:12:13.825321Z") == False
44
+ assert check_normalized_timestamp("2023-09-15 15:12:13.825Z") == False
45
+ assert check_normalized_timestamp("2023-09-10T15:12:13.825") == False
46
+
47
+ def test_convert_date_to_normalized_timestamp():
48
+ assert convert_date_to_normalized_timestamp("2023-10-18", True) == "2023-10-18T00:00:00.000Z"
49
+ assert convert_date_to_normalized_timestamp("2023-07-18", False) == "2023-07-18T23:59:59.999Z"
@@ -5,4 +5,4 @@ def pytest_sessionstart(session):
5
5
  os.environ[
6
6
  "OCD_DTL_REQUESTS_PER_QUOTA_TIME"
7
7
  ] = "10000" # Disable the throttler during tests
8
- os.environ["OCD_DTL_MAX_BACK_OFF_TIME"] = "0.01" # waiting should be minimal
8
+ os.environ["OCD_DTL_MAX_BACK_OFF_TIME"] = "0.01" # waiting should be minimal
@@ -572,3 +572,138 @@ def test_add_threats_bulk(datalake: Datalake):
572
572
  "failed": [],
573
573
  }
574
574
  ]
575
+
576
+ def test_atom_values_empty_source_id(datalake: Datalake):
577
+ with pytest.raises(ValueError) as err:
578
+ source_id = None
579
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
580
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
581
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
582
+ assert str(err.value) == "A list of minimum one source id required"
583
+
584
+ with pytest.raises(ValueError) as err:
585
+ source_id = []
586
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
587
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
588
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
589
+ assert str(err.value) == "A list of minimum one source id required"
590
+
591
+
592
+ def test_atom_values_bad_timestamps(datalake: Datalake):
593
+ with pytest.raises(ValueError) as err:
594
+ source_id = ["source_a"]
595
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
596
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since)
597
+ assert str(err.value) == "Both timestamps parameters are required"
598
+
599
+ with pytest.raises(ValueError) as err:
600
+ source_id = ["source_a"]
601
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
602
+ datalake.Threats.atom_values(source_id=source_id, normalized_timestamp_until=normalized_timestamp_until)
603
+ assert str(err.value) == "Both timestamps parameters are required"
604
+
605
+ with pytest.raises(ValueError) as err:
606
+ source_id = ["source_a"]
607
+ normalized_timestamp_since = "2023-09-14 15:00:00.000Z"
608
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
609
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
610
+ assert str(err.value) == "Invalid normalized_timestamp_since format"
611
+
612
+ with pytest.raises(ValueError) as err:
613
+ source_id = ["source_a"]
614
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
615
+ normalized_timestamp_until = "2023-09-14T16:00:00.000"
616
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
617
+ assert str(err.value) == "Invalid normalized_timestamp_until format"
618
+
619
+
620
+ def test_atom_values_bad_output(datalake: Datalake):
621
+ with pytest.raises(TypeError) as err:
622
+ source_id = ["source_a"]
623
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
624
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
625
+ output=Output.JSON_ZIP
626
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until, output)
627
+ assert str(err.value) == "output needs to either be set to JSON or CSV"
628
+
629
+ with pytest.raises(TypeError) as err:
630
+ source_id = ["source_a"]
631
+ normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
632
+ normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
633
+ output="JSON"
634
+ datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until, output)
635
+ assert str(err.value) == "output needs to either be set to JSON or CSV"
636
+
637
+
638
+ @responses.activate
639
+ def test_atom_values_bad_source(datalake: Datalake):
640
+ url = (
641
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
642
+ )
643
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
644
+ responses.add(responses.GET, url, json=get_resp, status=200)
645
+
646
+ url = (
647
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
648
+ )
649
+ post_resp = [
650
+ {"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
651
+ {"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
652
+ ]
653
+ responses.add(responses.POST, url, json=post_resp, status=200)
654
+
655
+ with pytest.raises(ValueError) as err:
656
+ datalake.Threats.atom_values(
657
+ ["source_a", "source_b"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
658
+ )
659
+ assert str(err.value) == "The following sources are invalid : ['source_b']"
660
+
661
+
662
+ @responses.activate
663
+ def test_atom_values(datalake: Datalake):
664
+ url = (
665
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
666
+ )
667
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
668
+ responses.add(responses.GET, url, json=get_resp, status=200)
669
+
670
+ url = (
671
+ "https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
672
+ )
673
+ post_resp = [
674
+ {"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
675
+ {"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
676
+ ]
677
+ responses.add(responses.POST, url, json=post_resp, status=200)
678
+
679
+ assert datalake.Threats.atom_values(
680
+ ["source_a"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
681
+ ) == post_resp
682
+
683
+ @responses.activate
684
+ def test_preprod_atom_values():
685
+ url = (
686
+ "https://ti.extranet.mrti-center.com/api/v2/mrti/sources/"
687
+ )
688
+ get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
689
+ responses.add(responses.GET, url, json=get_resp, status=200)
690
+
691
+ url = "https://ti.extranet.mrti-center.com/api/v2/auth/token/"
692
+ auth_response = {"access_token": "access_token", "refresh_token": "refresh_token"}
693
+ responses.add(responses.POST, url, json=auth_response, status=200)
694
+
695
+ url = (
696
+ "https://ti.extranet.mrti-center.com/api/v2/mrti/atom-values/"
697
+ )
698
+ post_resp = [
699
+ {"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
700
+ {"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
701
+ ]
702
+ responses.add(responses.POST, url, json=post_resp, status=200)
703
+
704
+ datalake = Datalake(env='preprod', username="username", password="password")
705
+ assert datalake.Threats.atom_values(
706
+ ["source_a"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
707
+ ) == post_resp
708
+
709
+
@@ -0,0 +1,29 @@
1
+ import sys
2
+ import pytest
3
+ from unittest.mock import patch
4
+ from datalake_scripts.cli import Cli
5
+ from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
6
+ add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values
7
+
8
+
9
+ # Tests to make sure the commands call the right scripts (and recognizes the command)
10
+ @pytest.mark.parametrize("function_script", [add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
11
+ add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values])
12
+ def test_name_function(function_script):
13
+ command_name = function_script.__name__.split('.')[-1]
14
+
15
+ # Special cases in cli.py:
16
+ if command_name == "add_comments":
17
+ command_name = "add_comment"
18
+ elif command_name == "get_threats_by_hashkey":
19
+ command_name = "get_threats"
20
+
21
+ with patch.object(function_script, 'main', return_value='mocked response') as mock_function_script:
22
+ # Act: Change sys.argv to simulate the command line arguments
23
+ with patch('sys.argv', ['ocd-dtl', command_name, '--output', 'output.txt', '--env', 'preprod']):
24
+ cli_app = Cli()
25
+
26
+ # Assert: ensure the mock was called and the result is as expected
27
+ mock_function_script.assert_called_once_with(['--output', 'output.txt', '--env', 'preprod'])
28
+
29
+
@@ -1,37 +0,0 @@
1
- from datalake.common.utils import join_dicts
2
- from datalake_scripts.helper_scripts.utils import split_list
3
-
4
-
5
- def test_split_list():
6
- list_to_split = [1, 2, 3, 4, 5]
7
-
8
- generator = split_list(list_to_split, 3)
9
- head = next(generator)
10
- tail = next(generator)
11
-
12
- assert head == [1, 2, 3]
13
- assert tail == [4, 5]
14
-
15
-
16
- def test_join_dicts():
17
- d1 = {
18
- "k1": ["k1", "k2", "k3"],
19
- "a1": ["a1", "a2", "a3"],
20
- "b1": ["b1", "b2"],
21
- }
22
- d2 = {
23
- "k1": ["k4", "k5", "k6"],
24
- "a1": ["a4", "a5", "a6"],
25
- }
26
- d3 = {
27
- "k1": ["k7", "k8"],
28
- "a1": ["a7", "a8"],
29
- "p1": ["p1", "p2"],
30
- }
31
-
32
- assert join_dicts(d1, d2, d3) == {
33
- "k1": ["k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8"],
34
- "a1": ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8"],
35
- "b1": ["b1", "b2"],
36
- "p1": ["p1", "p2"],
37
- }