datalake-scripts 2.6.8__tar.gz → 2.6.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/PKG-INFO +5 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/README.md +5 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/utils.py +47 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/endpoint.py +1 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/threats.py +53 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/cli.py +6 -2
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/get_engine.py +21 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/post_engine.py +22 -2
- datalake-scripts-2.6.10/datalake_scripts/scripts/get_atom_values.py +109 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/PKG-INFO +5 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/SOURCES.txt +4 -2
- datalake-scripts-2.6.10/tests/common/test_utils.py +49 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/conftest.py +1 -1
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_threats.py +135 -0
- datalake-scripts-2.6.10/tests/scripts/test_cli.py +29 -0
- datalake-scripts-2.6.8/tests/scripts/test_utils.py +0 -37
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/api_objects/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/api_objects/bulk_search_task.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/atom.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/atom_type.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/config.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/logger.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/ouput.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/throttler.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/token_manager.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/common/warn.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/datalake.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/advanced_search.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/bulk_search.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/sightings.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake/endpoints/tags.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/base_engine.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/base_script.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/common/mixins.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/engines/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/entry_points.txt +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/setup.cfg +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/setup.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/common/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/common/fixture.py +0 -0
- {datalake-scripts-2.6.8/tests/lib → datalake-scripts-2.6.10/tests/common}/test_output.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_advanced_search.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_atom_type.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_bulk_search.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_sightings.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/lib/test_tags.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/__init__.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_base_engine.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_csv_builder.py +0 -0
- {datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/tests/scripts/test_endpoint_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.6.
|
|
3
|
+
Version: 2.6.10
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
|
|
|
80
80
|
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
81
81
|
|
|
82
82
|
|
|
83
|
+
### Contributing
|
|
84
|
+
|
|
85
|
+
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
86
|
+
|
|
@@ -58,3 +58,8 @@ Parameters common and optional for all commands:
|
|
|
58
58
|
> -q will quiet the verbosity of the program (but still show errors / warnings)
|
|
59
59
|
|
|
60
60
|
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
### Contributing
|
|
64
|
+
|
|
65
|
+
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import datetime
|
|
1
|
+
import datetime
|
|
2
|
+
import json
|
|
2
3
|
from collections import defaultdict
|
|
3
4
|
from typing import Dict, Optional, Generator
|
|
4
5
|
|
|
@@ -51,3 +52,48 @@ def get_error_message(resp_body: dict):
|
|
|
51
52
|
return resp_body.get("messages")
|
|
52
53
|
else:
|
|
53
54
|
raise ValueError("no error message in api response")
|
|
55
|
+
|
|
56
|
+
def save_output(file_name: str, data):
|
|
57
|
+
"""
|
|
58
|
+
Save the data in a file.
|
|
59
|
+
If data is dict, file format will be JSON.
|
|
60
|
+
If data is a list, file format will be txt.
|
|
61
|
+
Else it will be saved as it comes.
|
|
62
|
+
"""
|
|
63
|
+
with open(file_name, "w") as file_to_write:
|
|
64
|
+
if isinstance(data, dict):
|
|
65
|
+
file_to_write.write(json.dumps(data, sort_keys=True, indent=2))
|
|
66
|
+
elif isinstance(data, list):
|
|
67
|
+
for item in data:
|
|
68
|
+
file_to_write.write(f"{item}\n")
|
|
69
|
+
else:
|
|
70
|
+
file_to_write.write(data)
|
|
71
|
+
|
|
72
|
+
def check_normalized_timestamp(ts: str) -> bool:
|
|
73
|
+
"""
|
|
74
|
+
Check if the provided timestamp is in a normalized format
|
|
75
|
+
"""
|
|
76
|
+
try :
|
|
77
|
+
if ts != datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]+'Z':
|
|
78
|
+
raise ValueError
|
|
79
|
+
return True
|
|
80
|
+
except ValueError:
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
def convert_date_to_normalized_timestamp(date_input : str, start : bool) -> str:
|
|
84
|
+
"""
|
|
85
|
+
Convert a given date into a normalized timestamp format
|
|
86
|
+
Either at the begining of the day (start at True) or at the end (start at False)
|
|
87
|
+
"""
|
|
88
|
+
try :
|
|
89
|
+
if start:
|
|
90
|
+
date_ts=date_input+'T00:00:00.000Z'
|
|
91
|
+
else:
|
|
92
|
+
date_ts=date_input+'T23:59:59.999Z'
|
|
93
|
+
if check_normalized_timestamp(date_ts):
|
|
94
|
+
return date_ts
|
|
95
|
+
else:
|
|
96
|
+
raise ValueError
|
|
97
|
+
except:
|
|
98
|
+
raise ValueError
|
|
99
|
+
|
|
@@ -67,7 +67,7 @@ class Endpoint:
|
|
|
67
67
|
while tries_left > 0:
|
|
68
68
|
headers["Authorization"] = self.token_manager.access_token
|
|
69
69
|
logger.debug(self._pretty_debug_request(url, method, post_body, headers))
|
|
70
|
-
|
|
70
|
+
|
|
71
71
|
response = self._send_request(
|
|
72
72
|
url, method, headers, post_body, stream=stream
|
|
73
73
|
)
|
|
@@ -7,7 +7,7 @@ from requests.sessions import PreparedRequest
|
|
|
7
7
|
from datalake import AtomType, ThreatType, OverrideType, Atom
|
|
8
8
|
from datalake.common.atom import ScoreMap
|
|
9
9
|
from datalake.common.ouput import Output, output_supported, parse_response
|
|
10
|
-
from datalake.common.utils import split_list, aggregate_csv_or_json_api_response
|
|
10
|
+
from datalake.common.utils import split_list, aggregate_csv_or_json_api_response, save_output, check_normalized_timestamp
|
|
11
11
|
from datalake.endpoints.endpoint import Endpoint
|
|
12
12
|
|
|
13
13
|
|
|
@@ -432,3 +432,55 @@ class Threats(Endpoint):
|
|
|
432
432
|
url = self._build_url_for_endpoint("threats-manual")
|
|
433
433
|
response = self.datalake_requests(url, "post", self._post_headers(), payload)
|
|
434
434
|
return parse_response(response)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
@staticmethod
|
|
438
|
+
def check_atom_values_params(source_id, normalized_timestamp_since, normalized_timestamp_until, output, output_path):
|
|
439
|
+
if not source_id or len(source_id)<1:
|
|
440
|
+
raise ValueError("A list of minimum one source id required")
|
|
441
|
+
if not normalized_timestamp_since or not normalized_timestamp_until:
|
|
442
|
+
raise ValueError("Both timestamps parameters are required")
|
|
443
|
+
if not check_normalized_timestamp(normalized_timestamp_since):
|
|
444
|
+
raise ValueError("Invalid normalized_timestamp_since format")
|
|
445
|
+
if not check_normalized_timestamp(normalized_timestamp_until):
|
|
446
|
+
raise ValueError("Invalid normalized_timestamp_until format")
|
|
447
|
+
if output!=Output.JSON and output!=Output.CSV:
|
|
448
|
+
raise TypeError("output needs to either be set to JSON or CSV")
|
|
449
|
+
|
|
450
|
+
def atom_values(
|
|
451
|
+
self,
|
|
452
|
+
source_id: List = None,
|
|
453
|
+
normalized_timestamp_since: str = None,
|
|
454
|
+
normalized_timestamp_until: str = None,
|
|
455
|
+
output: Output = Output.JSON,
|
|
456
|
+
output_path: str = None
|
|
457
|
+
):
|
|
458
|
+
"""
|
|
459
|
+
Get all atom values based on given source id list and from time range.
|
|
460
|
+
"""
|
|
461
|
+
self.check_atom_values_params(source_id, normalized_timestamp_since, normalized_timestamp_until, output, output_path)
|
|
462
|
+
|
|
463
|
+
url = self._build_url_for_endpoint("sources")
|
|
464
|
+
url = url + "?limit=1000&description_only=true"
|
|
465
|
+
for source in source_id:
|
|
466
|
+
url = url + "&source_ids=" + source
|
|
467
|
+
response = self.datalake_requests(url, "get", self._get_headers())
|
|
468
|
+
response = parse_response(response)
|
|
469
|
+
if response["count"] != len(source_id):
|
|
470
|
+
valid_sources = [response["results"][x]["id"] for x in range(0,len(response["results"]))]
|
|
471
|
+
invalid_sources = [source_id[y] for y in range(0,len(source_id)) if source_id[y] not in valid_sources]
|
|
472
|
+
raise ValueError(f"The following sources are invalid : {invalid_sources}")
|
|
473
|
+
|
|
474
|
+
payload = {
|
|
475
|
+
"source_id": source_id,
|
|
476
|
+
"normalized_timestamp_since": normalized_timestamp_since,
|
|
477
|
+
"normalized_timestamp_until": normalized_timestamp_until,
|
|
478
|
+
"ordering": ["atom_type"]
|
|
479
|
+
}
|
|
480
|
+
url = self._build_url_for_endpoint("threats-atom-values")
|
|
481
|
+
response = self.datalake_requests(url, "post", self._post_headers(output), payload)
|
|
482
|
+
|
|
483
|
+
if output_path:
|
|
484
|
+
save_output(output_path, parse_response(response))
|
|
485
|
+
|
|
486
|
+
return parse_response(response)
|
|
@@ -4,12 +4,12 @@ import argparse
|
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
6
|
from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
|
|
7
|
-
add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search
|
|
7
|
+
add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class Cli:
|
|
11
11
|
CLI_NAME = "ocd-dtl"
|
|
12
|
-
VERSION = "2.6.
|
|
12
|
+
VERSION = "2.6.10"
|
|
13
13
|
|
|
14
14
|
def __init__(self):
|
|
15
15
|
parser = argparse.ArgumentParser(
|
|
@@ -51,6 +51,10 @@ The most commonly used {self.CLI_NAME} commands are:
|
|
|
51
51
|
def get_threats(self):
|
|
52
52
|
args = sys.argv[2:]
|
|
53
53
|
get_threats_by_hashkey.main(args)
|
|
54
|
+
|
|
55
|
+
def get_atom_values(self):
|
|
56
|
+
args = sys.argv[2:]
|
|
57
|
+
get_atom_values.main(args)
|
|
54
58
|
|
|
55
59
|
def get_threats_from_query_hash(self):
|
|
56
60
|
args = sys.argv[2:]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""All the engines that use a GET endpoint."""
|
|
2
|
-
|
|
2
|
+
from typing import List, Tuple
|
|
3
3
|
from requests import PreparedRequest
|
|
4
4
|
|
|
5
5
|
from datalake.common.logger import logger
|
|
@@ -77,3 +77,23 @@ class Threats(GetEngine):
|
|
|
77
77
|
headers = {"Accept": response_format}
|
|
78
78
|
response = self.datalake_requests(req.url, "get", headers=headers)
|
|
79
79
|
return response
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class Sources(GetEngine):
|
|
83
|
+
""" Checks if a source exists"""
|
|
84
|
+
|
|
85
|
+
def _build_url(self, endpoint_config: dict, environment: str):
|
|
86
|
+
return self._build_url_for_endpoint("sources")
|
|
87
|
+
|
|
88
|
+
def check_sources(self, sources_list: List[str]) -> Tuple[bool, List[str]]:
|
|
89
|
+
url = self.url + "?limit=1000&description_only=true"
|
|
90
|
+
for source in sources_list:
|
|
91
|
+
url = url + "&source_ids=" + source
|
|
92
|
+
response = self.datalake_requests(url, "get", self._get_headers())
|
|
93
|
+
if response["count"] == len(sources_list):
|
|
94
|
+
return True, []
|
|
95
|
+
else :
|
|
96
|
+
valid_sources = [response["results"][x]["id"] for x in range(0, len(response["results"]))]
|
|
97
|
+
invalid_sources = [sources_list[y] for y in range(0, len(sources_list)) if sources_list[y] not in valid_sources]
|
|
98
|
+
return False, invalid_sources
|
|
99
|
+
|
|
@@ -5,6 +5,7 @@ from typing import Set, Dict, List, Union
|
|
|
5
5
|
from requests import PreparedRequest
|
|
6
6
|
|
|
7
7
|
from datalake.common.logger import logger
|
|
8
|
+
from datalake.common.ouput import Output, parse_response
|
|
8
9
|
from datalake.endpoints import Endpoint
|
|
9
10
|
from datalake_scripts.common.base_engine import BaseEngine
|
|
10
11
|
from datalake_scripts.common.mixins import HandleBulkTaskMixin
|
|
@@ -58,8 +59,8 @@ class PostEngine(BaseEngine):
|
|
|
58
59
|
if entry[key] == value:
|
|
59
60
|
return entry
|
|
60
61
|
|
|
61
|
-
def _post_headers(self) -> dict:
|
|
62
|
-
return {"Accept":
|
|
62
|
+
def _post_headers(self, output=Output.JSON ) -> dict:
|
|
63
|
+
return {"Accept": output.value, "Content-Type": "application/json"}
|
|
63
64
|
|
|
64
65
|
@staticmethod
|
|
65
66
|
def build_full_query_body(query_body):
|
|
@@ -176,3 +177,22 @@ class AtomValuesExtractor(PostEngine):
|
|
|
176
177
|
"treat_hashes_like": treat_hashes_like,
|
|
177
178
|
}
|
|
178
179
|
return self.datalake_requests(self.url, "post", self._post_headers(), payload)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class AtomSearch(PostEngine):
|
|
183
|
+
"""Atom values by sources and time range"""
|
|
184
|
+
def _build_url(self, endpoint_config: dict, environment: str):
|
|
185
|
+
return self._build_url_for_endpoint("threats-atom-values")
|
|
186
|
+
|
|
187
|
+
def get_atoms(
|
|
188
|
+
self, sources_list: List[str], since_ts: str, until_ts: str, type: Output
|
|
189
|
+
) -> dict:
|
|
190
|
+
payload = {
|
|
191
|
+
"source_id": sources_list,
|
|
192
|
+
"normalized_timestamp_since": since_ts,
|
|
193
|
+
"normalized_timestamp_until": until_ts,
|
|
194
|
+
"ordering": ["atom_type"]
|
|
195
|
+
}
|
|
196
|
+
return self.datalake_requests(self.url, "post", self._post_headers(output=type), payload)
|
|
197
|
+
|
|
198
|
+
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from datalake.common.config import Config
|
|
4
|
+
from datalake.common.logger import logger, configure_logging
|
|
5
|
+
from datalake.common.ouput import Output
|
|
6
|
+
from datalake.common.token_manager import TokenManager
|
|
7
|
+
from datalake.common.utils import check_normalized_timestamp, convert_date_to_normalized_timestamp
|
|
8
|
+
from datalake_scripts.common.base_script import BaseScripts
|
|
9
|
+
from datalake_scripts.engines.get_engine import Sources
|
|
10
|
+
from datalake_scripts.engines.post_engine import AtomSearch
|
|
11
|
+
from datalake_scripts.helper_scripts.utils import load_list, save_output
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(override_args=None):
|
|
15
|
+
"""Method to start the script"""
|
|
16
|
+
logger.debug(f"START: get_atom_values.py")
|
|
17
|
+
|
|
18
|
+
# Load initial args
|
|
19
|
+
parser = BaseScripts.start(
|
|
20
|
+
"Retrieve atom values from a list of sources ids and a time range"
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
"source_id",
|
|
24
|
+
help="sources ids of the atom values to retreive",
|
|
25
|
+
nargs="*",
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"-i",
|
|
29
|
+
"--input_file",
|
|
30
|
+
help="list of sources ids (hashkeys) that need to be retrieved",
|
|
31
|
+
)
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"--since",
|
|
34
|
+
help="normalized timestamp (or date) to set up the start of the search of atom values",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--until",
|
|
38
|
+
help="normalized timestamp (or date) to set up the end of the search of atom values",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"-ot",
|
|
42
|
+
"--output_type",
|
|
43
|
+
default="json",
|
|
44
|
+
help="set to the output type desired {json,csv}. Default is json if not specified",
|
|
45
|
+
)
|
|
46
|
+
if override_args:
|
|
47
|
+
args = parser.parse_args(override_args)
|
|
48
|
+
else:
|
|
49
|
+
args = parser.parse_args()
|
|
50
|
+
configure_logging(args.loglevel)
|
|
51
|
+
|
|
52
|
+
if not args.source_id and not args.input_file:
|
|
53
|
+
parser.error("either a source_id or an input_file is required")
|
|
54
|
+
sources_list = load_list(args.input_file) if args.input_file else args.source_id
|
|
55
|
+
logger.debug(f"TOTAL: {len(sources_list)} sources found")
|
|
56
|
+
|
|
57
|
+
if not args.since or not args.until:
|
|
58
|
+
parser.error("Both timestamps since and until are required")
|
|
59
|
+
|
|
60
|
+
# Check list of sources is not empty
|
|
61
|
+
if len(sources_list) <1:
|
|
62
|
+
parser.error("At least one source id is required to perform the atom search")
|
|
63
|
+
|
|
64
|
+
# Check given timestamps are correct
|
|
65
|
+
if not check_normalized_timestamp(args.since):
|
|
66
|
+
try:
|
|
67
|
+
since_ts=convert_date_to_normalized_timestamp(args.since, True)
|
|
68
|
+
except:
|
|
69
|
+
parser.error(
|
|
70
|
+
f"Since Timestamp/Date provided is incorrect : {args.since}, expected format is '%Y-%m-%dT%H:%M:%S.%f[:3]Z' or '%Y-%m-%d' "
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
since_ts=args.since
|
|
74
|
+
if not check_normalized_timestamp(args.until):
|
|
75
|
+
try:
|
|
76
|
+
until_ts=convert_date_to_normalized_timestamp(args.until, False)
|
|
77
|
+
except:
|
|
78
|
+
parser.error(
|
|
79
|
+
f"Until Timestamp/Date provided is incorrect : {args.until}, expected format is '%Y-%m-%dT%H:%M:%S.%f[:3]Z' or '%Y-%m-%d' "
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
until_ts=args.until
|
|
83
|
+
|
|
84
|
+
# Load api_endpoints and tokens
|
|
85
|
+
endpoint_config = Config().load_config()
|
|
86
|
+
token_manager = TokenManager(endpoint_config, environment=args.env)
|
|
87
|
+
|
|
88
|
+
# Check list of sources are valid
|
|
89
|
+
engine_check_sources = Sources(endpoint_config, args.env, token_manager)
|
|
90
|
+
all_sources_are_valid, list_invalid_sources = engine_check_sources.check_sources(sources_list)
|
|
91
|
+
if not all_sources_are_valid:
|
|
92
|
+
parser.error(
|
|
93
|
+
f"The following sources are invalid : {list_invalid_sources}"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
search_engine_atom_values = AtomSearch(endpoint_config, args.env, token_manager)
|
|
97
|
+
if not args.output_type or args.output_type=="json":
|
|
98
|
+
list_atom_values = search_engine_atom_values.get_atoms(sources_list, since_ts, until_ts, Output.JSON)
|
|
99
|
+
else:
|
|
100
|
+
list_atom_values = search_engine_atom_values.get_atoms(sources_list, since_ts, until_ts, Output.CSV)
|
|
101
|
+
|
|
102
|
+
if args.output:
|
|
103
|
+
save_output(args.output, list_atom_values)
|
|
104
|
+
logger.debug(f"Atom values saved in {args.output}\n")
|
|
105
|
+
logger.debug(f"END: get_atom_values.py")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
if __name__ == "__main__":
|
|
109
|
+
sys.exit(main())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: datalake-scripts
|
|
3
|
-
Version: 2.6.
|
|
3
|
+
Version: 2.6.10
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
5
|
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
6
|
Author: OCD
|
|
@@ -80,3 +80,7 @@ Parameters common and optional for all commands:
|
|
|
80
80
|
For information about each command and more, please check [the documentation directory](https://github.com/cert-orangecyberdefense/datalake/tree/master/docs)
|
|
81
81
|
|
|
82
82
|
|
|
83
|
+
### Contributing
|
|
84
|
+
|
|
85
|
+
To develop on this repository, please refer to [this file](https://github.com/cert-orangecyberdefense/datalake/tree/master/CONTRIBUTING.md)
|
|
86
|
+
|
|
@@ -41,6 +41,7 @@ setup.py
|
|
|
41
41
|
./datalake_scripts/scripts/advanced_search.py
|
|
42
42
|
./datalake_scripts/scripts/bulk_lookup_threats.py
|
|
43
43
|
./datalake_scripts/scripts/edit_score.py
|
|
44
|
+
./datalake_scripts/scripts/get_atom_values.py
|
|
44
45
|
./datalake_scripts/scripts/get_query_hash.py
|
|
45
46
|
./datalake_scripts/scripts/get_threats_by_hashkey.py
|
|
46
47
|
./datalake_scripts/scripts/get_threats_from_query_hash.py
|
|
@@ -49,21 +50,22 @@ setup.py
|
|
|
49
50
|
./tests/conftest.py
|
|
50
51
|
./tests/common/__init__.py
|
|
51
52
|
./tests/common/fixture.py
|
|
53
|
+
./tests/common/test_output.py
|
|
54
|
+
./tests/common/test_utils.py
|
|
52
55
|
./tests/lib/__init__.py
|
|
53
56
|
./tests/lib/test_advanced_search.py
|
|
54
57
|
./tests/lib/test_atom_type.py
|
|
55
58
|
./tests/lib/test_auth_via_tokens.py
|
|
56
59
|
./tests/lib/test_bulk_search.py
|
|
57
|
-
./tests/lib/test_output.py
|
|
58
60
|
./tests/lib/test_sightings.py
|
|
59
61
|
./tests/lib/test_tags.py
|
|
60
62
|
./tests/lib/test_threats.py
|
|
61
63
|
./tests/scripts/__init__.py
|
|
62
64
|
./tests/scripts/test_add_new_threats.py
|
|
63
65
|
./tests/scripts/test_base_engine.py
|
|
66
|
+
./tests/scripts/test_cli.py
|
|
64
67
|
./tests/scripts/test_csv_builder.py
|
|
65
68
|
./tests/scripts/test_endpoint_config.py
|
|
66
|
-
./tests/scripts/test_utils.py
|
|
67
69
|
datalake_scripts.egg-info/PKG-INFO
|
|
68
70
|
datalake_scripts.egg-info/SOURCES.txt
|
|
69
71
|
datalake_scripts.egg-info/dependency_links.txt
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from datalake.common.utils import join_dicts, check_normalized_timestamp, convert_date_to_normalized_timestamp
|
|
2
|
+
from datalake_scripts.helper_scripts.utils import split_list
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_split_list():
|
|
6
|
+
list_to_split = [1, 2, 3, 4, 5]
|
|
7
|
+
|
|
8
|
+
generator = split_list(list_to_split, 3)
|
|
9
|
+
head = next(generator)
|
|
10
|
+
tail = next(generator)
|
|
11
|
+
|
|
12
|
+
assert head == [1, 2, 3]
|
|
13
|
+
assert tail == [4, 5]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_join_dicts():
|
|
17
|
+
d1 = {
|
|
18
|
+
"k1": ["k1", "k2", "k3"],
|
|
19
|
+
"a1": ["a1", "a2", "a3"],
|
|
20
|
+
"b1": ["b1", "b2"],
|
|
21
|
+
}
|
|
22
|
+
d2 = {
|
|
23
|
+
"k1": ["k4", "k5", "k6"],
|
|
24
|
+
"a1": ["a4", "a5", "a6"],
|
|
25
|
+
}
|
|
26
|
+
d3 = {
|
|
27
|
+
"k1": ["k7", "k8"],
|
|
28
|
+
"a1": ["a7", "a8"],
|
|
29
|
+
"p1": ["p1", "p2"],
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
assert join_dicts(d1, d2, d3) == {
|
|
33
|
+
"k1": ["k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8"],
|
|
34
|
+
"a1": ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8"],
|
|
35
|
+
"b1": ["b1", "b2"],
|
|
36
|
+
"p1": ["p1", "p2"],
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
def test_normalized_timestamp():
|
|
40
|
+
assert check_normalized_timestamp("2023-09-14T15:00:00.000Z") == True
|
|
41
|
+
assert check_normalized_timestamp("2023-09-15T15:12:13.825Z") == True
|
|
42
|
+
assert check_normalized_timestamp("2023-9-15T15:12:13.825Z") == False
|
|
43
|
+
assert check_normalized_timestamp("2023-09-15T15:12:13.825321Z") == False
|
|
44
|
+
assert check_normalized_timestamp("2023-09-15 15:12:13.825Z") == False
|
|
45
|
+
assert check_normalized_timestamp("2023-09-10T15:12:13.825") == False
|
|
46
|
+
|
|
47
|
+
def test_convert_date_to_normalized_timestamp():
|
|
48
|
+
assert convert_date_to_normalized_timestamp("2023-10-18", True) == "2023-10-18T00:00:00.000Z"
|
|
49
|
+
assert convert_date_to_normalized_timestamp("2023-07-18", False) == "2023-07-18T23:59:59.999Z"
|
|
@@ -5,4 +5,4 @@ def pytest_sessionstart(session):
|
|
|
5
5
|
os.environ[
|
|
6
6
|
"OCD_DTL_REQUESTS_PER_QUOTA_TIME"
|
|
7
7
|
] = "10000" # Disable the throttler during tests
|
|
8
|
-
os.environ["OCD_DTL_MAX_BACK_OFF_TIME"] = "0.01" # waiting should be minimal
|
|
8
|
+
os.environ["OCD_DTL_MAX_BACK_OFF_TIME"] = "0.01" # waiting should be minimal
|
|
@@ -572,3 +572,138 @@ def test_add_threats_bulk(datalake: Datalake):
|
|
|
572
572
|
"failed": [],
|
|
573
573
|
}
|
|
574
574
|
]
|
|
575
|
+
|
|
576
|
+
def test_atom_values_empty_source_id(datalake: Datalake):
|
|
577
|
+
with pytest.raises(ValueError) as err:
|
|
578
|
+
source_id = None
|
|
579
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
580
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
581
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
|
|
582
|
+
assert str(err.value) == "A list of minimum one source id required"
|
|
583
|
+
|
|
584
|
+
with pytest.raises(ValueError) as err:
|
|
585
|
+
source_id = []
|
|
586
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
587
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
588
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
|
|
589
|
+
assert str(err.value) == "A list of minimum one source id required"
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def test_atom_values_bad_timestamps(datalake: Datalake):
|
|
593
|
+
with pytest.raises(ValueError) as err:
|
|
594
|
+
source_id = ["source_a"]
|
|
595
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
596
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since)
|
|
597
|
+
assert str(err.value) == "Both timestamps parameters are required"
|
|
598
|
+
|
|
599
|
+
with pytest.raises(ValueError) as err:
|
|
600
|
+
source_id = ["source_a"]
|
|
601
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
602
|
+
datalake.Threats.atom_values(source_id=source_id, normalized_timestamp_until=normalized_timestamp_until)
|
|
603
|
+
assert str(err.value) == "Both timestamps parameters are required"
|
|
604
|
+
|
|
605
|
+
with pytest.raises(ValueError) as err:
|
|
606
|
+
source_id = ["source_a"]
|
|
607
|
+
normalized_timestamp_since = "2023-09-14 15:00:00.000Z"
|
|
608
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
609
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
|
|
610
|
+
assert str(err.value) == "Invalid normalized_timestamp_since format"
|
|
611
|
+
|
|
612
|
+
with pytest.raises(ValueError) as err:
|
|
613
|
+
source_id = ["source_a"]
|
|
614
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
615
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000"
|
|
616
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until)
|
|
617
|
+
assert str(err.value) == "Invalid normalized_timestamp_until format"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def test_atom_values_bad_output(datalake: Datalake):
|
|
621
|
+
with pytest.raises(TypeError) as err:
|
|
622
|
+
source_id = ["source_a"]
|
|
623
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
624
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
625
|
+
output=Output.JSON_ZIP
|
|
626
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until, output)
|
|
627
|
+
assert str(err.value) == "output needs to either be set to JSON or CSV"
|
|
628
|
+
|
|
629
|
+
with pytest.raises(TypeError) as err:
|
|
630
|
+
source_id = ["source_a"]
|
|
631
|
+
normalized_timestamp_since = "2023-09-14T15:00:00.000Z"
|
|
632
|
+
normalized_timestamp_until = "2023-09-14T16:00:00.000Z"
|
|
633
|
+
output="JSON"
|
|
634
|
+
datalake.Threats.atom_values(source_id, normalized_timestamp_since, normalized_timestamp_until, output)
|
|
635
|
+
assert str(err.value) == "output needs to either be set to JSON or CSV"
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
@responses.activate
|
|
639
|
+
def test_atom_values_bad_source(datalake: Datalake):
|
|
640
|
+
url = (
|
|
641
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
|
|
642
|
+
)
|
|
643
|
+
get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
|
|
644
|
+
responses.add(responses.GET, url, json=get_resp, status=200)
|
|
645
|
+
|
|
646
|
+
url = (
|
|
647
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
|
|
648
|
+
)
|
|
649
|
+
post_resp = [
|
|
650
|
+
{"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
|
|
651
|
+
{"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
|
|
652
|
+
]
|
|
653
|
+
responses.add(responses.POST, url, json=post_resp, status=200)
|
|
654
|
+
|
|
655
|
+
with pytest.raises(ValueError) as err:
|
|
656
|
+
datalake.Threats.atom_values(
|
|
657
|
+
["source_a", "source_b"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
|
|
658
|
+
)
|
|
659
|
+
assert str(err.value) == "The following sources are invalid : ['source_b']"
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
@responses.activate
|
|
663
|
+
def test_atom_values(datalake: Datalake):
|
|
664
|
+
url = (
|
|
665
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/sources/"
|
|
666
|
+
)
|
|
667
|
+
get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
|
|
668
|
+
responses.add(responses.GET, url, json=get_resp, status=200)
|
|
669
|
+
|
|
670
|
+
url = (
|
|
671
|
+
"https://datalake.cert.orangecyberdefense.com/api/v2/mrti/atom-values/"
|
|
672
|
+
)
|
|
673
|
+
post_resp = [
|
|
674
|
+
{"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
|
|
675
|
+
{"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
|
|
676
|
+
]
|
|
677
|
+
responses.add(responses.POST, url, json=post_resp, status=200)
|
|
678
|
+
|
|
679
|
+
assert datalake.Threats.atom_values(
|
|
680
|
+
["source_a"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
|
|
681
|
+
) == post_resp
|
|
682
|
+
|
|
683
|
+
@responses.activate
|
|
684
|
+
def test_preprod_atom_values():
|
|
685
|
+
url = (
|
|
686
|
+
"https://ti.extranet.mrti-center.com/api/v2/mrti/sources/"
|
|
687
|
+
)
|
|
688
|
+
get_resp = {"count": 1, "results": [{"description": "source_a description", "id": "source_a"}]}
|
|
689
|
+
responses.add(responses.GET, url, json=get_resp, status=200)
|
|
690
|
+
|
|
691
|
+
url = "https://ti.extranet.mrti-center.com/api/v2/auth/token/"
|
|
692
|
+
auth_response = {"access_token": "access_token", "refresh_token": "refresh_token"}
|
|
693
|
+
responses.add(responses.POST, url, json=auth_response, status=200)
|
|
694
|
+
|
|
695
|
+
url = (
|
|
696
|
+
"https://ti.extranet.mrti-center.com/api/v2/mrti/atom-values/"
|
|
697
|
+
)
|
|
698
|
+
post_resp = [
|
|
699
|
+
{"atom_value": "atom_value_1", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T08:08:23Z"},
|
|
700
|
+
{"atom_value": "atom_value_2", "atom_type": "domain", "source_id": "source_a", "normalized_timestamp": "2023-09-17T07:16:26Z"}
|
|
701
|
+
]
|
|
702
|
+
responses.add(responses.POST, url, json=post_resp, status=200)
|
|
703
|
+
|
|
704
|
+
datalake = Datalake(env='preprod', username="username", password="password")
|
|
705
|
+
assert datalake.Threats.atom_values(
|
|
706
|
+
["source_a"], "2023-09-17T07:16:26.345Z", "2023-09-17T08:08:23.345Z"
|
|
707
|
+
) == post_resp
|
|
708
|
+
|
|
709
|
+
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import pytest
|
|
3
|
+
from unittest.mock import patch
|
|
4
|
+
from datalake_scripts.cli import Cli
|
|
5
|
+
from datalake_scripts.scripts import add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
|
|
6
|
+
add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# Tests to make sure the commands call the right scripts (and recognizes the command)
|
|
10
|
+
@pytest.mark.parametrize("function_script", [add_threats, get_threats_by_hashkey, edit_score, get_threats_from_query_hash, \
|
|
11
|
+
add_comments, lookup_threats, add_tags, get_query_hash, bulk_lookup_threats, advanced_search, get_atom_values])
|
|
12
|
+
def test_name_function(function_script):
|
|
13
|
+
command_name = function_script.__name__.split('.')[-1]
|
|
14
|
+
|
|
15
|
+
# Special cases in cli.py:
|
|
16
|
+
if command_name == "add_comments":
|
|
17
|
+
command_name = "add_comment"
|
|
18
|
+
elif command_name == "get_threats_by_hashkey":
|
|
19
|
+
command_name = "get_threats"
|
|
20
|
+
|
|
21
|
+
with patch.object(function_script, 'main', return_value='mocked response') as mock_function_script:
|
|
22
|
+
# Act: Change sys.argv to simulate the command line arguments
|
|
23
|
+
with patch('sys.argv', ['ocd-dtl', command_name, '--output', 'output.txt', '--env', 'preprod']):
|
|
24
|
+
cli_app = Cli()
|
|
25
|
+
|
|
26
|
+
# Assert: ensure the mock was called and the result is as expected
|
|
27
|
+
mock_function_script.assert_called_once_with(['--output', 'output.txt', '--env', 'preprod'])
|
|
28
|
+
|
|
29
|
+
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
from datalake.common.utils import join_dicts
|
|
2
|
-
from datalake_scripts.helper_scripts.utils import split_list
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
def test_split_list():
|
|
6
|
-
list_to_split = [1, 2, 3, 4, 5]
|
|
7
|
-
|
|
8
|
-
generator = split_list(list_to_split, 3)
|
|
9
|
-
head = next(generator)
|
|
10
|
-
tail = next(generator)
|
|
11
|
-
|
|
12
|
-
assert head == [1, 2, 3]
|
|
13
|
-
assert tail == [4, 5]
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def test_join_dicts():
|
|
17
|
-
d1 = {
|
|
18
|
-
"k1": ["k1", "k2", "k3"],
|
|
19
|
-
"a1": ["a1", "a2", "a3"],
|
|
20
|
-
"b1": ["b1", "b2"],
|
|
21
|
-
}
|
|
22
|
-
d2 = {
|
|
23
|
-
"k1": ["k4", "k5", "k6"],
|
|
24
|
-
"a1": ["a4", "a5", "a6"],
|
|
25
|
-
}
|
|
26
|
-
d3 = {
|
|
27
|
-
"k1": ["k7", "k8"],
|
|
28
|
-
"a1": ["a7", "a8"],
|
|
29
|
-
"p1": ["p1", "p2"],
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
assert join_dicts(d1, d2, d3) == {
|
|
33
|
-
"k1": ["k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8"],
|
|
34
|
-
"a1": ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8"],
|
|
35
|
-
"b1": ["b1", "b2"],
|
|
36
|
-
"p1": ["p1", "p2"],
|
|
37
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/helper_scripts/output_builder.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/bulk_lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{datalake-scripts-2.6.8 → datalake-scripts-2.6.10}/datalake_scripts.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|