datalake-scripts 3.0.1__tar.gz → 3.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/PKG-INFO +9 -17
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/README.md +1 -1
- datalake_scripts-3.1.1/datalake/common/logger.py +97 -0
- datalake_scripts-3.1.1/datalake/common/output_mode.py +29 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/utils.py +2 -1
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/comments.py +4 -6
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/threats.py +2 -2
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/miscellaneous/search_watch.py +5 -13
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/cli.py +9 -1
- datalake_scripts-3.1.1/datalake_scripts/common/rendering/__init__.py +14 -0
- datalake_scripts-3.1.1/datalake_scripts/common/rendering/base.py +92 -0
- datalake_scripts-3.1.1/datalake_scripts/common/rendering/rich_renderer.py +121 -0
- datalake_scripts-3.1.1/datalake_scripts/common/rendering/simple_renderer.py +107 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/output_builder.py +14 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/utils.py +4 -2
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_tags.py +3 -5
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_threats.py +18 -28
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/advanced_search.py +1 -3
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/bulk_lookup_threats.py +57 -100
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/edit_score.py +3 -7
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_filtered_threat_entity.py +2 -1
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_my_user_info.py +2 -1
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_threats_from_query_hash.py +14 -32
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/lookup_threats.py +4 -5
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/search_watch.py +5 -36
- datalake_scripts-3.1.1/pyproject.toml +59 -0
- datalake_scripts-3.0.1/datalake/common/logger.py +0 -22
- datalake_scripts-3.0.1/datalake_scripts.egg-info/PKG-INFO +0 -140
- datalake_scripts-3.0.1/datalake_scripts.egg-info/SOURCES.txt +0 -156
- datalake_scripts-3.0.1/datalake_scripts.egg-info/dependency_links.txt +0 -1
- datalake_scripts-3.0.1/datalake_scripts.egg-info/entry_points.txt +0 -2
- datalake_scripts-3.0.1/datalake_scripts.egg-info/requires.txt +0 -3
- datalake_scripts-3.0.1/datalake_scripts.egg-info/top_level.txt +0 -3
- datalake_scripts-3.0.1/pyproject.toml +0 -3
- datalake_scripts-3.0.1/setup.cfg +0 -11
- datalake_scripts-3.0.1/setup.py +0 -56
- datalake_scripts-3.0.1/tests/__init__.py +0 -0
- datalake_scripts-3.0.1/tests/common/__init__.py +0 -0
- datalake_scripts-3.0.1/tests/common/fixture.py +0 -96
- datalake_scripts-3.0.1/tests/common/test_output.py +0 -33
- datalake_scripts-3.0.1/tests/common/test_user_agent.py +0 -36
- datalake_scripts-3.0.1/tests/common/test_utils.py +0 -61
- datalake_scripts-3.0.1/tests/conftest.py +0 -8
- datalake_scripts-3.0.1/tests/lib/__init__.py +0 -0
- datalake_scripts-3.0.1/tests/lib/test_advanced_search.py +0 -138
- datalake_scripts-3.0.1/tests/lib/test_atom_type.py +0 -126
- datalake_scripts-3.0.1/tests/lib/test_auth_via_tokens.py +0 -371
- datalake_scripts-3.0.1/tests/lib/test_bulk_search.py +0 -596
- datalake_scripts-3.0.1/tests/lib/test_filtered_threat_entity.py +0 -156
- datalake_scripts-3.0.1/tests/lib/test_my_account.py +0 -51
- datalake_scripts-3.0.1/tests/lib/test_search_watch.py +0 -301
- datalake_scripts-3.0.1/tests/lib/test_sightings.py +0 -759
- datalake_scripts-3.0.1/tests/lib/test_tags.py +0 -81
- datalake_scripts-3.0.1/tests/lib/test_threats.py +0 -789
- datalake_scripts-3.0.1/tests/scripts/__init__.py +0 -0
- datalake_scripts-3.0.1/tests/scripts/test_add_new_threats.py +0 -56
- datalake_scripts-3.0.1/tests/scripts/test_base_scripts.py +0 -19
- datalake_scripts-3.0.1/tests/scripts/test_cli.py +0 -59
- datalake_scripts-3.0.1/tests/scripts/test_csv_builder.py +0 -183
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/atom.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/atom_type.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/bulk_search_task.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/config.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/output.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/throttler.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/token_manager.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/user_agent.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/warn.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/config/endpoints.json +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/datalake.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/advanced_search.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/bulk_search.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/endpoint.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/my_account.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/sightings.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/sources.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/tags.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/miscellaneous/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/common/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/common/base_script.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_threats.py +0 -0
|
@@ -1,32 +1,24 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datalake_scripts
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.1.1
|
|
4
4
|
Summary: A collection of scripts to easily use the API of OCD Datalake
|
|
5
|
-
Home-page: https://github.com/cert-orangecyberdefense/datalake/
|
|
6
5
|
Author: OCD
|
|
7
6
|
Author-email: cert-contact.ocd@orange.com
|
|
8
|
-
|
|
7
|
+
Requires-Python: >=3.8,<4.0
|
|
9
8
|
Classifier: Programming Language :: Python :: 3.8
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
13
|
Classifier: Operating System :: POSIX :: Linux
|
|
13
14
|
Classifier: Environment :: Console
|
|
14
15
|
Classifier: Topic :: Security
|
|
15
16
|
Classifier: Natural Language :: English
|
|
16
|
-
Requires-
|
|
17
|
+
Requires-Dist: requests (>=2.32.3,<3)
|
|
18
|
+
Requires-Dist: rich (>=13.9.4,<14)
|
|
19
|
+
Project-URL: Homepage, https://github.com/cert-orangecyberdefense/datalake/
|
|
20
|
+
Project-URL: Repository, https://github.com/cert-orangecyberdefense/datalake/
|
|
17
21
|
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: requests
|
|
19
|
-
Requires-Dist: halo
|
|
20
|
-
Requires-Dist: prettytable
|
|
21
|
-
Dynamic: author
|
|
22
|
-
Dynamic: author-email
|
|
23
|
-
Dynamic: classifier
|
|
24
|
-
Dynamic: description
|
|
25
|
-
Dynamic: description-content-type
|
|
26
|
-
Dynamic: home-page
|
|
27
|
-
Dynamic: requires-dist
|
|
28
|
-
Dynamic: requires-python
|
|
29
|
-
Dynamic: summary
|
|
30
22
|
|
|
31
23
|
|
|
32
24
|
____ _ _ _ ____ _ _
|
|
@@ -45,7 +37,7 @@ You can use this repository either as a library or as a CLI
|
|
|
45
37
|
|
|
46
38
|
### Installation
|
|
47
39
|
|
|
48
|
-
With Python 3.
|
|
40
|
+
With Python 3.8+:
|
|
49
41
|
```
|
|
50
42
|
$ pip install datalake-scripts
|
|
51
43
|
$ pip3 install datalake-scripts
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
from rich.highlighter import ReprHighlighter
|
|
7
|
+
from rich.logging import RichHandler
|
|
8
|
+
from rich.text import Text
|
|
9
|
+
from rich.theme import Theme
|
|
10
|
+
|
|
11
|
+
from datalake.common.output_mode import rich_output_enabled
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("OCD_DTL")
|
|
14
|
+
|
|
15
|
+
DEBUG_FORMAT = "%(name)s:%(levelname)-4s %(message)s"
|
|
16
|
+
INFO_FORMAT = "%(message)s"
|
|
17
|
+
|
|
18
|
+
THEME = Theme(
|
|
19
|
+
{
|
|
20
|
+
"repr.status_ok": "bold green",
|
|
21
|
+
"repr.status_ko": "bold red",
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# Console used for everything that is not the actual result of a command
|
|
26
|
+
# (logs, spinners, progress). It writes on stderr like the plain logging handler.
|
|
27
|
+
# It must be shared by the log handler and the spinners so they don't overlap.
|
|
28
|
+
console = Console(stderr=True, theme=THEME)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DatalakeHighlighter(ReprHighlighter):
|
|
32
|
+
"""Color the status keywords used by the library and the CLI (OK, FAILED, ...)"""
|
|
33
|
+
|
|
34
|
+
highlights = ReprHighlighter.highlights + [
|
|
35
|
+
r"(?P<status_ok>\b(OK|FOUND)\b)",
|
|
36
|
+
r"(?P<status_ko>\b(KO|FAILED|NOT_FOUND|ERROR)\b)",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class DatalakeRichHandler(RichHandler):
|
|
41
|
+
LEVEL_STYLES = {
|
|
42
|
+
logging.WARNING: "yellow",
|
|
43
|
+
logging.ERROR: "bold red",
|
|
44
|
+
logging.CRITICAL: "bold white on red",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
def __init__(self, loglevel: int):
|
|
48
|
+
is_debug = loglevel == logging.DEBUG
|
|
49
|
+
super().__init__(
|
|
50
|
+
console=console,
|
|
51
|
+
highlighter=DatalakeHighlighter(),
|
|
52
|
+
show_time=is_debug,
|
|
53
|
+
show_level=is_debug,
|
|
54
|
+
show_path=is_debug,
|
|
55
|
+
markup=False,
|
|
56
|
+
rich_tracebacks=True,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def render_message(self, record: logging.LogRecord, message: str) -> Text:
|
|
60
|
+
text = super().render_message(record, message)
|
|
61
|
+
style = self.LEVEL_STYLES.get(record.levelno)
|
|
62
|
+
if style:
|
|
63
|
+
text.stylize_before(style)
|
|
64
|
+
return text
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SimpleHandler(logging.StreamHandler):
|
|
68
|
+
"""Plain, non wrapped output on stderr, used for scripted usages"""
|
|
69
|
+
|
|
70
|
+
def __init__(self, loglevel: int):
|
|
71
|
+
super().__init__(sys.stderr)
|
|
72
|
+
self.setFormatter(
|
|
73
|
+
logging.Formatter(
|
|
74
|
+
DEBUG_FORMAT if loglevel == logging.DEBUG else INFO_FORMAT
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
_handler: Optional[logging.Handler] = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _build_handler(loglevel: int) -> logging.Handler:
|
|
83
|
+
handler_class = DatalakeRichHandler if rich_output_enabled() else SimpleHandler
|
|
84
|
+
return handler_class(loglevel)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def configure_logging(loglevel: int):
|
|
88
|
+
global _handler
|
|
89
|
+
if _handler is not None:
|
|
90
|
+
logger.removeHandler(_handler)
|
|
91
|
+
_handler = _build_handler(loglevel)
|
|
92
|
+
logger.addHandler(_handler)
|
|
93
|
+
logger.setLevel(loglevel)
|
|
94
|
+
|
|
95
|
+
logger.debug(f"Logging configured with level {str(loglevel)}")
|
|
96
|
+
|
|
97
|
+
return logger
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Single source of truth for the output mode (rich or simple).
|
|
3
|
+
|
|
4
|
+
Rich output (colors, spinners, tables) is only used when the program runs
|
|
5
|
+
interactively, i.e. both stdout and stderr are attached to a terminal.
|
|
6
|
+
As soon as one of them is redirected (pipe, file, cron, CI...), the simple
|
|
7
|
+
output is used so that scripted usages keep a stable, plain text output.
|
|
8
|
+
|
|
9
|
+
Setting OCD_DTL_OUTPUT_MODE=simple forces the simple output.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
OUTPUT_MODE_ENV = "OCD_DTL_OUTPUT_MODE"
|
|
16
|
+
SIMPLE_MODE = "simple"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _isatty(stream) -> bool:
|
|
20
|
+
try:
|
|
21
|
+
return stream is not None and stream.isatty()
|
|
22
|
+
except (AttributeError, ValueError): # missing or closed stream
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def rich_output_enabled() -> bool:
|
|
27
|
+
if os.getenv(OUTPUT_MODE_ENV, "auto").strip().lower() == SIMPLE_MODE:
|
|
28
|
+
return False
|
|
29
|
+
return _isatty(sys.stdout) and _isatty(sys.stderr)
|
|
@@ -108,7 +108,8 @@ def convert_date_to_normalized_timestamp(date_input: str, start: bool) -> str:
|
|
|
108
108
|
|
|
109
109
|
|
|
110
110
|
def load_json(file_name: str) -> dict:
|
|
111
|
-
|
|
111
|
+
with open(file_name) as file:
|
|
112
|
+
return json.load(file)
|
|
112
113
|
|
|
113
114
|
|
|
114
115
|
class SetEncoder(json.JSONEncoder):
|
|
@@ -33,16 +33,14 @@ class Comments(Endpoint):
|
|
|
33
33
|
try:
|
|
34
34
|
response = self._post_comment(hashkey, comment, visibility)
|
|
35
35
|
except ValueError as e:
|
|
36
|
-
self.logger.warning("
|
|
37
|
-
self.logger.debug(
|
|
38
|
-
"\x1b[6;30;41m" + hashkey + ": FAILED : " + str(e) + "\x1b[0m"
|
|
39
|
-
)
|
|
36
|
+
self.logger.warning(f"{hashkey}: FAILED")
|
|
37
|
+
self.logger.debug(f"{hashkey}: FAILED : {e}")
|
|
40
38
|
list_haskeys_no_comment_added.append(hashkey)
|
|
41
39
|
continue
|
|
42
40
|
if not response:
|
|
43
|
-
self.logger.warning(f"
|
|
41
|
+
self.logger.warning(f"{hashkey}: KO")
|
|
44
42
|
list_haskeys_no_comment_added.append(hashkey)
|
|
45
43
|
else:
|
|
46
44
|
list_haskeys_comment_added.append(hashkey)
|
|
47
|
-
self.logger.debug(f"
|
|
45
|
+
self.logger.debug(f"{hashkey}: OK")
|
|
48
46
|
return list_haskeys_comment_added, list_haskeys_no_comment_added
|
|
@@ -571,11 +571,11 @@ class Threats(Endpoint):
|
|
|
571
571
|
if not threat_dict.get("hashkey"):
|
|
572
572
|
list_of_not_found_threats_hashkeys.append(hashkey)
|
|
573
573
|
self.logger.debug(
|
|
574
|
-
f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 11)}
|
|
574
|
+
f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 11)}ERROR"
|
|
575
575
|
)
|
|
576
576
|
else:
|
|
577
577
|
self.logger.debug(
|
|
578
|
-
f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 10)}
|
|
578
|
+
f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 10)} OK "
|
|
579
579
|
)
|
|
580
580
|
dict_threats["count"] += 1
|
|
581
581
|
dict_threats["results"].append(threat_dict)
|
|
@@ -128,9 +128,7 @@ class SearchWatch:
|
|
|
128
128
|
f"Reference file not found: {reference_file}"
|
|
129
129
|
) from e
|
|
130
130
|
|
|
131
|
-
self.logger.info(
|
|
132
|
-
f"\x1b[0;30;47m File to compare with {reference_file} \x1b[0m"
|
|
133
|
-
)
|
|
131
|
+
self.logger.info(f"File to compare with {reference_file}")
|
|
134
132
|
previous_datetime = self._extract_timestamp(reference_file)
|
|
135
133
|
diff_threats = self.threats_diff(
|
|
136
134
|
file_to_compare_with_data,
|
|
@@ -152,7 +150,7 @@ class SearchWatch:
|
|
|
152
150
|
)
|
|
153
151
|
file_to_compare_with_data = load_json(file_to_compare_with_json_path)
|
|
154
152
|
self.logger.info(
|
|
155
|
-
f"
|
|
153
|
+
f"File to compare with {file_to_compare_with_json_path}"
|
|
156
154
|
)
|
|
157
155
|
previous_datetime = self._extract_timestamp(file_to_compare_with_json)
|
|
158
156
|
diff_threats = self.threats_diff(
|
|
@@ -162,9 +160,7 @@ class SearchWatch:
|
|
|
162
160
|
actual_datetime,
|
|
163
161
|
)
|
|
164
162
|
else:
|
|
165
|
-
self.logger.info(
|
|
166
|
-
f"\x1b[0;30;43m No file to compare with {filepath} \x1b[0m"
|
|
167
|
-
)
|
|
163
|
+
self.logger.info(f"No file to compare with {filepath}")
|
|
168
164
|
|
|
169
165
|
if save_diff_threats:
|
|
170
166
|
diff_threats_path = (
|
|
@@ -178,13 +174,9 @@ class SearchWatch:
|
|
|
178
174
|
+ ".json"
|
|
179
175
|
)
|
|
180
176
|
save_output(diff_threats_path, diff_threats, cls=SetEncoder)
|
|
181
|
-
self.logger.info(
|
|
182
|
-
f"\x1b[0;37;42m OK: DIFF THREATS SAVED IN {diff_threats_path} \x1b[0m"
|
|
183
|
-
)
|
|
177
|
+
self.logger.info(f"OK: DIFF THREATS SAVED IN {diff_threats_path}")
|
|
184
178
|
|
|
185
179
|
save_output(filepath, bulk_search_result_json)
|
|
186
|
-
self.logger.info(
|
|
187
|
-
f"\x1b[0;37;42m OK: MATCHING THREATS SAVED IN {filepath} \x1b[0m"
|
|
188
|
-
)
|
|
180
|
+
self.logger.info(f"OK: MATCHING THREATS SAVED IN {filepath}")
|
|
189
181
|
|
|
190
182
|
return diff_threats
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
3
|
import sys, os
|
|
4
|
+
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
|
4
5
|
from datalake_scripts.scripts import (
|
|
5
6
|
add_threats,
|
|
6
7
|
get_threats,
|
|
@@ -19,9 +20,16 @@ from datalake_scripts.scripts import (
|
|
|
19
20
|
)
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
def _read_version():
|
|
24
|
+
try:
|
|
25
|
+
return _pkg_version("datalake_scripts")
|
|
26
|
+
except PackageNotFoundError:
|
|
27
|
+
return "0.0.0+unknown"
|
|
28
|
+
|
|
29
|
+
|
|
22
30
|
class Cli:
|
|
23
31
|
CLI_NAME = "ocd-dtl"
|
|
24
|
-
VERSION =
|
|
32
|
+
VERSION = _read_version()
|
|
25
33
|
|
|
26
34
|
def __init__(self):
|
|
27
35
|
parser = argparse.ArgumentParser(
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from datalake.common.output_mode import rich_output_enabled
|
|
2
|
+
from datalake_scripts.common.rendering.base import Progress, Renderer
|
|
3
|
+
from datalake_scripts.common.rendering.rich_renderer import RichRenderer
|
|
4
|
+
from datalake_scripts.common.rendering.simple_renderer import SimpleRenderer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def get_renderer(dtl) -> Renderer:
|
|
8
|
+
"""Factory: the only place choosing between the rich and the simple output"""
|
|
9
|
+
if rich_output_enabled():
|
|
10
|
+
return RichRenderer(dtl.logger)
|
|
11
|
+
return SimpleRenderer(dtl.logger, terminal_size=dtl.Threats.terminal_size)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
__all__ = ["get_renderer", "Renderer", "Progress", "RichRenderer", "SimpleRenderer"]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Presentation layer of the CLI.
|
|
3
|
+
|
|
4
|
+
Scripts describe *what* has to be shown (a lookup report, a diff, a JSON
|
|
5
|
+
result...). A `Renderer` decides *how* it is shown. Scripts never know which
|
|
6
|
+
renderer they use: it is picked once by `get_renderer()`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from typing import List, Tuple
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Progress:
|
|
15
|
+
"""
|
|
16
|
+
Context manager wrapping a long running step.
|
|
17
|
+
The default implementation displays nothing.
|
|
18
|
+
|
|
19
|
+
with renderer.progress("Creating bulk task") as progress:
|
|
20
|
+
...
|
|
21
|
+
progress.update("Waiting for the bulk task")
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __enter__(self) -> "Progress":
|
|
25
|
+
return self
|
|
26
|
+
|
|
27
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
def update(self, text: str):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Renderer(ABC):
|
|
35
|
+
MAX_SEARCH_LINKS = 10
|
|
36
|
+
|
|
37
|
+
def __init__(self, logger: logging.Logger):
|
|
38
|
+
self.logger = logger
|
|
39
|
+
|
|
40
|
+
def progress(self, text: str) -> Progress:
|
|
41
|
+
"""Display the progress of a long running step"""
|
|
42
|
+
return Progress()
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
def item_status(self, value: str, status: str, prefix: str = ""):
|
|
46
|
+
"""Report the status (OK, KO...) of a single processed item"""
|
|
47
|
+
|
|
48
|
+
@abstractmethod
|
|
49
|
+
def json(self, data):
|
|
50
|
+
"""Display a JSON result"""
|
|
51
|
+
|
|
52
|
+
@abstractmethod
|
|
53
|
+
def user_info(self, data: dict):
|
|
54
|
+
"""Display the information of the current user"""
|
|
55
|
+
|
|
56
|
+
@abstractmethod
|
|
57
|
+
def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
|
|
58
|
+
"""Display the result of a bulk search"""
|
|
59
|
+
|
|
60
|
+
@abstractmethod
|
|
61
|
+
def bulk_lookup(self, response, output_format: str, search_url: str):
|
|
62
|
+
"""
|
|
63
|
+
Display the result of a bulk lookup.
|
|
64
|
+
:param response: aggregated API response
|
|
65
|
+
:param output_format: json, csv or anything else for a human readable report
|
|
66
|
+
:param search_url: GUI url of a search, to be completed with a search hashkey
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
@abstractmethod
|
|
70
|
+
def search_watch_diff(self, diff: dict):
|
|
71
|
+
"""Display the added / removed threats of a search watch"""
|
|
72
|
+
|
|
73
|
+
@staticmethod
|
|
74
|
+
def _split_lookup_response(response: dict) -> Tuple[dict, List[str]]:
|
|
75
|
+
"""Separate the atoms (grouped by atom type) from the search hashkeys"""
|
|
76
|
+
results = dict(response)
|
|
77
|
+
search_hashkeys = results.pop("search_hashkey", None) or []
|
|
78
|
+
return results, search_hashkeys
|
|
79
|
+
|
|
80
|
+
def _search_links_lines(
|
|
81
|
+
self, search_url: str, search_hashkeys: List[str]
|
|
82
|
+
) -> List[str]:
|
|
83
|
+
if not search_hashkeys:
|
|
84
|
+
return ["No search hashkeys available for this lookup."]
|
|
85
|
+
if len(search_hashkeys) > self.MAX_SEARCH_LINKS:
|
|
86
|
+
return [
|
|
87
|
+
"Too many search hashkeys to display, check the output for the full list."
|
|
88
|
+
]
|
|
89
|
+
return [
|
|
90
|
+
f"Results available here : {search_url}{search_hashkey}"
|
|
91
|
+
for search_hashkey in search_hashkeys
|
|
92
|
+
]
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from itertools import zip_longest
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
from rich.console import Console
|
|
6
|
+
from rich.markup import escape
|
|
7
|
+
from rich.table import Table
|
|
8
|
+
from rich.text import Text
|
|
9
|
+
|
|
10
|
+
from datalake.common.logger import THEME, console as err_console
|
|
11
|
+
from datalake_scripts.common.rendering.base import Progress, Renderer
|
|
12
|
+
from datalake_scripts.helper_scripts.output_builder import format_threats_output
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Spinner(Progress):
|
|
16
|
+
"""Animated spinner on stderr, persisting a ✔ / ✖ line once the step is over"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, text: str):
|
|
19
|
+
self.text = text
|
|
20
|
+
self._status = err_console.status(escape(text), spinner="dots")
|
|
21
|
+
|
|
22
|
+
def __enter__(self) -> "Spinner":
|
|
23
|
+
self._status.start()
|
|
24
|
+
return self
|
|
25
|
+
|
|
26
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
27
|
+
self._status.stop()
|
|
28
|
+
mark = "[bold green]✔[/]" if exc_type is None else "[bold red]✖[/]"
|
|
29
|
+
err_console.print(f"{mark} {escape(self.text)}")
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
def update(self, text: str):
|
|
33
|
+
self.text = text
|
|
34
|
+
self._status.update(escape(text))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RichRenderer(Renderer):
|
|
38
|
+
"""Colored output with spinners and tables, used when running in a terminal"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, logger: logging.Logger):
|
|
41
|
+
super().__init__(logger)
|
|
42
|
+
self.out = Console(theme=THEME) # results go to stdout
|
|
43
|
+
|
|
44
|
+
def progress(self, text: str) -> Progress:
|
|
45
|
+
if not self.logger.isEnabledFor(logging.INFO): # --quiet
|
|
46
|
+
return super().progress(text)
|
|
47
|
+
return Spinner(text)
|
|
48
|
+
|
|
49
|
+
def item_status(self, value: str, status: str, prefix: str = ""):
|
|
50
|
+
self.logger.info(f"{prefix}{value}: {status}")
|
|
51
|
+
|
|
52
|
+
def json(self, data, sort_keys: bool = False):
|
|
53
|
+
self.out.print_json(data=data, indent=4, sort_keys=sort_keys)
|
|
54
|
+
|
|
55
|
+
def user_info(self, data: dict):
|
|
56
|
+
self.json(data)
|
|
57
|
+
|
|
58
|
+
def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
|
|
59
|
+
if one_threat_per_line:
|
|
60
|
+
print(format_threats_output(response, one_threat_per_line=True))
|
|
61
|
+
else:
|
|
62
|
+
self.json(response, sort_keys=True)
|
|
63
|
+
|
|
64
|
+
def bulk_lookup(self, response, output_format: str, search_url: str):
|
|
65
|
+
if output_format == "json":
|
|
66
|
+
self.json(response, sort_keys=True)
|
|
67
|
+
return
|
|
68
|
+
if output_format == "csv":
|
|
69
|
+
print(response)
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
results, search_hashkeys = self._split_lookup_response(response)
|
|
73
|
+
for atom_type, atoms in results.items():
|
|
74
|
+
table = self._table(title=f"[bold]{escape(atom_type.upper())}[/]")
|
|
75
|
+
table.add_column("Atom value", overflow="fold")
|
|
76
|
+
table.add_column("Hashkey", no_wrap=True)
|
|
77
|
+
table.add_column("Status", justify="center", no_wrap=True)
|
|
78
|
+
for atom in atoms:
|
|
79
|
+
found = atom.get("threat_found", False)
|
|
80
|
+
table.add_row(
|
|
81
|
+
Text(atom["atom_value"]),
|
|
82
|
+
Text(atom["hashkey"]),
|
|
83
|
+
self._status_badge(found, "FOUND", "NOT_FOUND"),
|
|
84
|
+
)
|
|
85
|
+
self.out.print(table)
|
|
86
|
+
self.out.print()
|
|
87
|
+
for line in self._search_links_lines(search_url, search_hashkeys):
|
|
88
|
+
self.out.print(Text(line))
|
|
89
|
+
|
|
90
|
+
def search_watch_diff(self, diff: dict):
|
|
91
|
+
added = sorted(diff.get("added", []), key=lambda x: x[0])
|
|
92
|
+
removed = sorted(diff.get("removed", []), key=lambda x: x[0])
|
|
93
|
+
|
|
94
|
+
self.out.print()
|
|
95
|
+
self.out.print(
|
|
96
|
+
f"[bold]Threats summary from {escape(str(diff['from']))} to {escape(str(diff['to']))}[/]\n"
|
|
97
|
+
f"Number of added items: [bold green]{len(added)}[/]\n"
|
|
98
|
+
f"Number of removed items: [bold red]{len(removed)}[/]\n"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
table = self._table()
|
|
102
|
+
table.add_column(
|
|
103
|
+
"Added (atom value - hash key)", style="green", overflow="fold"
|
|
104
|
+
)
|
|
105
|
+
table.add_column(
|
|
106
|
+
"Removed (atom value - hash key)", style="red", overflow="fold"
|
|
107
|
+
)
|
|
108
|
+
for add, remove in zip_longest(added, removed, fillvalue=None):
|
|
109
|
+
table.add_row(
|
|
110
|
+
Text(f"{add[0]} - {add[1]}" if add else ""),
|
|
111
|
+
Text(f"{remove[0]} - {remove[1]}" if remove else ""),
|
|
112
|
+
)
|
|
113
|
+
self.out.print(table)
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def _table(title: str = None) -> Table:
|
|
117
|
+
return Table(title=title, title_justify="left", header_style="bold")
|
|
118
|
+
|
|
119
|
+
@staticmethod
|
|
120
|
+
def _status_badge(ok: bool, ok_label: str, ko_label: str) -> str:
|
|
121
|
+
return f"[bold green]{ok_label}[/]" if ok else f"[bold red]{ko_label}[/]"
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from itertools import zip_longest
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
from datalake_scripts.common.rendering.base import Renderer
|
|
7
|
+
from datalake_scripts.helper_scripts.output_builder import format_threats_output
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SimpleRenderer(Renderer):
|
|
11
|
+
"""
|
|
12
|
+
Plain text output used for scripted usages (no tty).
|
|
13
|
+
The layouts are kept identical to the historical output of the CLI so that
|
|
14
|
+
customers parsing it are not disturbed.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
DIFF_HEADERS = [
|
|
18
|
+
"Added (atom value - hash key) ",
|
|
19
|
+
"Removed (atom value - hash key)",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
def __init__(self, logger: logging.Logger, terminal_size: int):
|
|
23
|
+
super().__init__(logger)
|
|
24
|
+
self.terminal_size = terminal_size
|
|
25
|
+
|
|
26
|
+
def item_status(self, value: str, status: str, prefix: str = ""):
|
|
27
|
+
self.logger.info(
|
|
28
|
+
f'{prefix}{value.ljust(self.terminal_size - 6, " ")} {status} '
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def json(self, data):
|
|
32
|
+
print(json.dumps(data, indent=4))
|
|
33
|
+
|
|
34
|
+
def user_info(self, data: dict):
|
|
35
|
+
print(data)
|
|
36
|
+
|
|
37
|
+
def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
|
|
38
|
+
self.logger.info(format_threats_output(response, one_threat_per_line))
|
|
39
|
+
self.logger.info("Done")
|
|
40
|
+
|
|
41
|
+
def bulk_lookup(self, response, output_format: str, search_url: str):
|
|
42
|
+
if output_format == "json":
|
|
43
|
+
self.logger.info(json.dumps(response, indent=4, sort_keys=True))
|
|
44
|
+
return
|
|
45
|
+
if output_format == "csv":
|
|
46
|
+
self.logger.info(response)
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
results, search_hashkeys = self._split_lookup_response(response)
|
|
50
|
+
for atom_type, atoms in results.items():
|
|
51
|
+
self.logger.info(
|
|
52
|
+
f'{"#" * 50} {atom_type.upper()} {"#" * (50 - len(atom_type))}'
|
|
53
|
+
)
|
|
54
|
+
for atom in atoms:
|
|
55
|
+
status = "FOUND" if atom.get("threat_found", False) else "NOT_FOUND"
|
|
56
|
+
self.logger.info(
|
|
57
|
+
f'{atom_type : <11} {atom["atom_value"][:29] : <30} hashkey: {atom["hashkey"]} {status :^15} '
|
|
58
|
+
)
|
|
59
|
+
self.logger.info("")
|
|
60
|
+
for line in self._search_links_lines(search_url, search_hashkeys):
|
|
61
|
+
self.logger.info(line)
|
|
62
|
+
|
|
63
|
+
def search_watch_diff(self, diff: dict):
|
|
64
|
+
rows = []
|
|
65
|
+
for add, remove in zip_longest(
|
|
66
|
+
sorted(diff.get("added", []), key=lambda x: x[0]),
|
|
67
|
+
sorted(diff.get("removed", []), key=lambda x: x[0]),
|
|
68
|
+
fillvalue=["", ""],
|
|
69
|
+
):
|
|
70
|
+
rows.append(
|
|
71
|
+
[
|
|
72
|
+
f"{add[0]} - {add[1]}" if add else "",
|
|
73
|
+
f"{remove[0]} - {remove[1]}" if remove else "",
|
|
74
|
+
]
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
summary = f"\nThreats summary from {diff['from']} to {diff['to']}:\n"
|
|
78
|
+
summary += f"Number of added items: {len(diff['added'])}\n"
|
|
79
|
+
summary += f"Number of removed items: {len(diff['removed'])}\n"
|
|
80
|
+
|
|
81
|
+
print(summary)
|
|
82
|
+
print(ascii_table(self.DIFF_HEADERS, rows))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _center(text: str, width: int) -> str:
|
|
86
|
+
"""Center a cell the same way the previous table library (prettytable) did"""
|
|
87
|
+
excess = width - len(text)
|
|
88
|
+
if excess % 2 and len(text) % 2 == 0:
|
|
89
|
+
return " " * (excess // 2 + 1) + text + " " * (excess // 2)
|
|
90
|
+
return " " * (excess // 2) + text + " " * (excess - excess // 2)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def ascii_table(headers: List[str], rows: List[List[str]]) -> str:
|
|
94
|
+
widths = [
|
|
95
|
+
max([len(headers[i])] + [len(row[i]) for row in rows])
|
|
96
|
+
for i in range(len(headers))
|
|
97
|
+
]
|
|
98
|
+
separator = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
|
|
99
|
+
|
|
100
|
+
def format_row(cells):
|
|
101
|
+
cells = (f" {_center(cell, width)} " for cell, width in zip(cells, widths))
|
|
102
|
+
return "|" + "|".join(cells) + "|"
|
|
103
|
+
|
|
104
|
+
lines = [separator, format_row(headers), separator]
|
|
105
|
+
lines += [format_row(row) for row in rows]
|
|
106
|
+
lines.append(separator)
|
|
107
|
+
return "\n".join(lines)
|
{datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/output_builder.py
RENAMED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
|
|
1
4
|
class CsvBuilder:
|
|
2
5
|
@staticmethod
|
|
3
6
|
def create_look_up_csv(
|
|
@@ -82,3 +85,14 @@ class CsvBuilder:
|
|
|
82
85
|
return None
|
|
83
86
|
tags = ",".join([tag["name"] for tag in tags]).replace('"', '""')
|
|
84
87
|
return f'"{tags}"'
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def format_threats_output(response: dict, one_threat_per_line=False) -> str:
|
|
91
|
+
"""Format a bulk search response as JSON, or as one threat per line"""
|
|
92
|
+
if one_threat_per_line:
|
|
93
|
+
threat_list = []
|
|
94
|
+
for result in response.get("results", []):
|
|
95
|
+
if result:
|
|
96
|
+
threat_list += result
|
|
97
|
+
return "\n".join(threat_list)
|
|
98
|
+
return json.dumps(response, sort_keys=True, indent=4)
|