datalake-scripts 3.0.1__tar.gz → 3.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/PKG-INFO +9 -17
  2. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/README.md +1 -1
  3. datalake_scripts-3.1.1/datalake/common/logger.py +97 -0
  4. datalake_scripts-3.1.1/datalake/common/output_mode.py +29 -0
  5. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/utils.py +2 -1
  6. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/comments.py +4 -6
  7. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/threats.py +2 -2
  8. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/miscellaneous/search_watch.py +5 -13
  9. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/cli.py +9 -1
  10. datalake_scripts-3.1.1/datalake_scripts/common/rendering/__init__.py +14 -0
  11. datalake_scripts-3.1.1/datalake_scripts/common/rendering/base.py +92 -0
  12. datalake_scripts-3.1.1/datalake_scripts/common/rendering/rich_renderer.py +121 -0
  13. datalake_scripts-3.1.1/datalake_scripts/common/rendering/simple_renderer.py +107 -0
  14. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/output_builder.py +14 -0
  15. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/utils.py +4 -2
  16. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_tags.py +3 -5
  17. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_threats.py +18 -28
  18. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/advanced_search.py +1 -3
  19. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/bulk_lookup_threats.py +57 -100
  20. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/edit_score.py +3 -7
  21. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_filtered_threat_entity.py +2 -1
  22. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_my_user_info.py +2 -1
  23. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_threats_from_query_hash.py +14 -32
  24. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/lookup_threats.py +4 -5
  25. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/search_watch.py +5 -36
  26. datalake_scripts-3.1.1/pyproject.toml +59 -0
  27. datalake_scripts-3.0.1/datalake/common/logger.py +0 -22
  28. datalake_scripts-3.0.1/datalake_scripts.egg-info/PKG-INFO +0 -140
  29. datalake_scripts-3.0.1/datalake_scripts.egg-info/SOURCES.txt +0 -156
  30. datalake_scripts-3.0.1/datalake_scripts.egg-info/dependency_links.txt +0 -1
  31. datalake_scripts-3.0.1/datalake_scripts.egg-info/entry_points.txt +0 -2
  32. datalake_scripts-3.0.1/datalake_scripts.egg-info/requires.txt +0 -3
  33. datalake_scripts-3.0.1/datalake_scripts.egg-info/top_level.txt +0 -3
  34. datalake_scripts-3.0.1/pyproject.toml +0 -3
  35. datalake_scripts-3.0.1/setup.cfg +0 -11
  36. datalake_scripts-3.0.1/setup.py +0 -56
  37. datalake_scripts-3.0.1/tests/__init__.py +0 -0
  38. datalake_scripts-3.0.1/tests/common/__init__.py +0 -0
  39. datalake_scripts-3.0.1/tests/common/fixture.py +0 -96
  40. datalake_scripts-3.0.1/tests/common/test_output.py +0 -33
  41. datalake_scripts-3.0.1/tests/common/test_user_agent.py +0 -36
  42. datalake_scripts-3.0.1/tests/common/test_utils.py +0 -61
  43. datalake_scripts-3.0.1/tests/conftest.py +0 -8
  44. datalake_scripts-3.0.1/tests/lib/__init__.py +0 -0
  45. datalake_scripts-3.0.1/tests/lib/test_advanced_search.py +0 -138
  46. datalake_scripts-3.0.1/tests/lib/test_atom_type.py +0 -126
  47. datalake_scripts-3.0.1/tests/lib/test_auth_via_tokens.py +0 -371
  48. datalake_scripts-3.0.1/tests/lib/test_bulk_search.py +0 -596
  49. datalake_scripts-3.0.1/tests/lib/test_filtered_threat_entity.py +0 -156
  50. datalake_scripts-3.0.1/tests/lib/test_my_account.py +0 -51
  51. datalake_scripts-3.0.1/tests/lib/test_search_watch.py +0 -301
  52. datalake_scripts-3.0.1/tests/lib/test_sightings.py +0 -759
  53. datalake_scripts-3.0.1/tests/lib/test_tags.py +0 -81
  54. datalake_scripts-3.0.1/tests/lib/test_threats.py +0 -789
  55. datalake_scripts-3.0.1/tests/scripts/__init__.py +0 -0
  56. datalake_scripts-3.0.1/tests/scripts/test_add_new_threats.py +0 -56
  57. datalake_scripts-3.0.1/tests/scripts/test_base_scripts.py +0 -19
  58. datalake_scripts-3.0.1/tests/scripts/test_cli.py +0 -59
  59. datalake_scripts-3.0.1/tests/scripts/test_csv_builder.py +0 -183
  60. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/__init__.py +0 -0
  61. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/__init__.py +0 -0
  62. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/atom.py +0 -0
  63. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/atom_type.py +0 -0
  64. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/bulk_search_task.py +0 -0
  65. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/config.py +0 -0
  66. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/output.py +0 -0
  67. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/throttler.py +0 -0
  68. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/token_manager.py +0 -0
  69. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/user_agent.py +0 -0
  70. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/common/warn.py +0 -0
  71. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/config/endpoints.json +0 -0
  72. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/datalake.py +0 -0
  73. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/__init__.py +0 -0
  74. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/advanced_search.py +0 -0
  75. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/bulk_search.py +0 -0
  76. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/endpoint.py +0 -0
  77. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/filtered_threat_entity.py +0 -0
  78. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/my_account.py +0 -0
  79. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/sightings.py +0 -0
  80. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/sources.py +0 -0
  81. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/endpoints/tags.py +0 -0
  82. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake/miscellaneous/__init__.py +0 -0
  83. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/__init__.py +0 -0
  84. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/common/__init__.py +0 -0
  85. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/common/base_script.py +0 -0
  86. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/helper_scripts/__init__.py +0 -0
  87. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/__init__.py +0 -0
  88. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/add_comments.py +0 -0
  89. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_atom_values.py +0 -0
  90. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_query_hash.py +0 -0
  91. {datalake_scripts-3.0.1 → datalake_scripts-3.1.1}/datalake_scripts/scripts/get_threats.py +0 -0
@@ -1,32 +1,24 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datalake_scripts
3
- Version: 3.0.1
3
+ Version: 3.1.1
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
- Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
5
  Author: OCD
7
6
  Author-email: cert-contact.ocd@orange.com
8
- Classifier: Programming Language :: Python :: 3.7
7
+ Requires-Python: >=3.8,<4.0
9
8
  Classifier: Programming Language :: Python :: 3.8
10
9
  Classifier: Programming Language :: Python :: 3.9
11
10
  Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
12
13
  Classifier: Operating System :: POSIX :: Linux
13
14
  Classifier: Environment :: Console
14
15
  Classifier: Topic :: Security
15
16
  Classifier: Natural Language :: English
16
- Requires-Python: >=3.6
17
+ Requires-Dist: requests (>=2.32.3,<3)
18
+ Requires-Dist: rich (>=13.9.4,<14)
19
+ Project-URL: Homepage, https://github.com/cert-orangecyberdefense/datalake/
20
+ Project-URL: Repository, https://github.com/cert-orangecyberdefense/datalake/
17
21
  Description-Content-Type: text/markdown
18
- Requires-Dist: requests
19
- Requires-Dist: halo
20
- Requires-Dist: prettytable
21
- Dynamic: author
22
- Dynamic: author-email
23
- Dynamic: classifier
24
- Dynamic: description
25
- Dynamic: description-content-type
26
- Dynamic: home-page
27
- Dynamic: requires-dist
28
- Dynamic: requires-python
29
- Dynamic: summary
30
22
 
31
23
 
32
24
  ____ _ _ _ ____ _ _
@@ -45,7 +37,7 @@ You can use this repository either as a library or as a CLI
45
37
 
46
38
  ### Installation
47
39
 
48
- With Python 3.6+:
40
+ With Python 3.8+:
49
41
  ```
50
42
  $ pip install datalake-scripts
51
43
  $ pip3 install datalake-scripts
@@ -15,7 +15,7 @@ You can use this repository either as a library or as a CLI
15
15
 
16
16
  ### Installation
17
17
 
18
- With Python 3.6+:
18
+ With Python 3.8+:
19
19
  ```
20
20
  $ pip install datalake-scripts
21
21
  $ pip3 install datalake-scripts
@@ -0,0 +1,97 @@
1
+ import logging
2
+ import sys
3
+ from typing import Optional
4
+
5
+ from rich.console import Console
6
+ from rich.highlighter import ReprHighlighter
7
+ from rich.logging import RichHandler
8
+ from rich.text import Text
9
+ from rich.theme import Theme
10
+
11
+ from datalake.common.output_mode import rich_output_enabled
12
+
13
+ logger = logging.getLogger("OCD_DTL")
14
+
15
+ DEBUG_FORMAT = "%(name)s:%(levelname)-4s %(message)s"
16
+ INFO_FORMAT = "%(message)s"
17
+
18
+ THEME = Theme(
19
+ {
20
+ "repr.status_ok": "bold green",
21
+ "repr.status_ko": "bold red",
22
+ }
23
+ )
24
+
25
+ # Console used for everything that is not the actual result of a command
26
+ # (logs, spinners, progress). It writes on stderr like the plain logging handler.
27
+ # It must be shared by the log handler and the spinners so they don't overlap.
28
+ console = Console(stderr=True, theme=THEME)
29
+
30
+
31
+ class DatalakeHighlighter(ReprHighlighter):
32
+ """Color the status keywords used by the library and the CLI (OK, FAILED, ...)"""
33
+
34
+ highlights = ReprHighlighter.highlights + [
35
+ r"(?P<status_ok>\b(OK|FOUND)\b)",
36
+ r"(?P<status_ko>\b(KO|FAILED|NOT_FOUND|ERROR)\b)",
37
+ ]
38
+
39
+
40
+ class DatalakeRichHandler(RichHandler):
41
+ LEVEL_STYLES = {
42
+ logging.WARNING: "yellow",
43
+ logging.ERROR: "bold red",
44
+ logging.CRITICAL: "bold white on red",
45
+ }
46
+
47
+ def __init__(self, loglevel: int):
48
+ is_debug = loglevel == logging.DEBUG
49
+ super().__init__(
50
+ console=console,
51
+ highlighter=DatalakeHighlighter(),
52
+ show_time=is_debug,
53
+ show_level=is_debug,
54
+ show_path=is_debug,
55
+ markup=False,
56
+ rich_tracebacks=True,
57
+ )
58
+
59
+ def render_message(self, record: logging.LogRecord, message: str) -> Text:
60
+ text = super().render_message(record, message)
61
+ style = self.LEVEL_STYLES.get(record.levelno)
62
+ if style:
63
+ text.stylize_before(style)
64
+ return text
65
+
66
+
67
+ class SimpleHandler(logging.StreamHandler):
68
+ """Plain, non wrapped output on stderr, used for scripted usages"""
69
+
70
+ def __init__(self, loglevel: int):
71
+ super().__init__(sys.stderr)
72
+ self.setFormatter(
73
+ logging.Formatter(
74
+ DEBUG_FORMAT if loglevel == logging.DEBUG else INFO_FORMAT
75
+ )
76
+ )
77
+
78
+
79
+ _handler: Optional[logging.Handler] = None
80
+
81
+
82
+ def _build_handler(loglevel: int) -> logging.Handler:
83
+ handler_class = DatalakeRichHandler if rich_output_enabled() else SimpleHandler
84
+ return handler_class(loglevel)
85
+
86
+
87
+ def configure_logging(loglevel: int):
88
+ global _handler
89
+ if _handler is not None:
90
+ logger.removeHandler(_handler)
91
+ _handler = _build_handler(loglevel)
92
+ logger.addHandler(_handler)
93
+ logger.setLevel(loglevel)
94
+
95
+ logger.debug(f"Logging configured with level {str(loglevel)}")
96
+
97
+ return logger
@@ -0,0 +1,29 @@
1
+ """
2
+ Single source of truth for the output mode (rich or simple).
3
+
4
+ Rich output (colors, spinners, tables) is only used when the program runs
5
+ interactively, i.e. both stdout and stderr are attached to a terminal.
6
+ As soon as one of them is redirected (pipe, file, cron, CI...), the simple
7
+ output is used so that scripted usages keep a stable, plain text output.
8
+
9
+ Setting OCD_DTL_OUTPUT_MODE=simple forces the simple output.
10
+ """
11
+
12
+ import os
13
+ import sys
14
+
15
+ OUTPUT_MODE_ENV = "OCD_DTL_OUTPUT_MODE"
16
+ SIMPLE_MODE = "simple"
17
+
18
+
19
+ def _isatty(stream) -> bool:
20
+ try:
21
+ return stream is not None and stream.isatty()
22
+ except (AttributeError, ValueError): # missing or closed stream
23
+ return False
24
+
25
+
26
+ def rich_output_enabled() -> bool:
27
+ if os.getenv(OUTPUT_MODE_ENV, "auto").strip().lower() == SIMPLE_MODE:
28
+ return False
29
+ return _isatty(sys.stdout) and _isatty(sys.stderr)
@@ -108,7 +108,8 @@ def convert_date_to_normalized_timestamp(date_input: str, start: bool) -> str:
108
108
 
109
109
 
110
110
  def load_json(file_name: str) -> dict:
111
- return json.load(open(file_name))
111
+ with open(file_name) as file:
112
+ return json.load(file)
112
113
 
113
114
 
114
115
  class SetEncoder(json.JSONEncoder):
@@ -33,16 +33,14 @@ class Comments(Endpoint):
33
33
  try:
34
34
  response = self._post_comment(hashkey, comment, visibility)
35
35
  except ValueError as e:
36
- self.logger.warning("\x1b[6;30;41m" + hashkey + ": FAILED\x1b[0m")
37
- self.logger.debug(
38
- "\x1b[6;30;41m" + hashkey + ": FAILED : " + str(e) + "\x1b[0m"
39
- )
36
+ self.logger.warning(f"{hashkey}: FAILED")
37
+ self.logger.debug(f"{hashkey}: FAILED : {e}")
40
38
  list_haskeys_no_comment_added.append(hashkey)
41
39
  continue
42
40
  if not response:
43
- self.logger.warning(f"\x1b[6;30;41m{hashkey}: KO\x1b[0m")
41
+ self.logger.warning(f"{hashkey}: KO")
44
42
  list_haskeys_no_comment_added.append(hashkey)
45
43
  else:
46
44
  list_haskeys_comment_added.append(hashkey)
47
- self.logger.debug(f"\x1b[6;30;42m{hashkey}: OK\x1b[0m")
45
+ self.logger.debug(f"{hashkey}: OK")
48
46
  return list_haskeys_comment_added, list_haskeys_no_comment_added
@@ -571,11 +571,11 @@ class Threats(Endpoint):
571
571
  if not threat_dict.get("hashkey"):
572
572
  list_of_not_found_threats_hashkeys.append(hashkey)
573
573
  self.logger.debug(
574
- f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 11)}\x1b[0;30;41mERROR\x1b[0m"
574
+ f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 11)}ERROR"
575
575
  )
576
576
  else:
577
577
  self.logger.debug(
578
- f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 10)}\x1b[0;30;42m OK \x1b[0m"
578
+ f"{str(index).ljust(5)}:{hashkey.ljust(terminal_size - 10)} OK "
579
579
  )
580
580
  dict_threats["count"] += 1
581
581
  dict_threats["results"].append(threat_dict)
@@ -128,9 +128,7 @@ class SearchWatch:
128
128
  f"Reference file not found: {reference_file}"
129
129
  ) from e
130
130
 
131
- self.logger.info(
132
- f"\x1b[0;30;47m File to compare with {reference_file} \x1b[0m"
133
- )
131
+ self.logger.info(f"File to compare with {reference_file}")
134
132
  previous_datetime = self._extract_timestamp(reference_file)
135
133
  diff_threats = self.threats_diff(
136
134
  file_to_compare_with_data,
@@ -152,7 +150,7 @@ class SearchWatch:
152
150
  )
153
151
  file_to_compare_with_data = load_json(file_to_compare_with_json_path)
154
152
  self.logger.info(
155
- f"\x1b[0;30;47m File to compare with {file_to_compare_with_json_path} \x1b[0m"
153
+ f"File to compare with {file_to_compare_with_json_path}"
156
154
  )
157
155
  previous_datetime = self._extract_timestamp(file_to_compare_with_json)
158
156
  diff_threats = self.threats_diff(
@@ -162,9 +160,7 @@ class SearchWatch:
162
160
  actual_datetime,
163
161
  )
164
162
  else:
165
- self.logger.info(
166
- f"\x1b[0;30;43m No file to compare with {filepath} \x1b[0m"
167
- )
163
+ self.logger.info(f"No file to compare with {filepath}")
168
164
 
169
165
  if save_diff_threats:
170
166
  diff_threats_path = (
@@ -178,13 +174,9 @@ class SearchWatch:
178
174
  + ".json"
179
175
  )
180
176
  save_output(diff_threats_path, diff_threats, cls=SetEncoder)
181
- self.logger.info(
182
- f"\x1b[0;37;42m OK: DIFF THREATS SAVED IN {diff_threats_path} \x1b[0m"
183
- )
177
+ self.logger.info(f"OK: DIFF THREATS SAVED IN {diff_threats_path}")
184
178
 
185
179
  save_output(filepath, bulk_search_result_json)
186
- self.logger.info(
187
- f"\x1b[0;37;42m OK: MATCHING THREATS SAVED IN {filepath} \x1b[0m"
188
- )
180
+ self.logger.info(f"OK: MATCHING THREATS SAVED IN {filepath}")
189
181
 
190
182
  return diff_threats
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env python3
2
2
  import argparse
3
3
  import sys, os
4
+ from importlib.metadata import version as _pkg_version, PackageNotFoundError
4
5
  from datalake_scripts.scripts import (
5
6
  add_threats,
6
7
  get_threats,
@@ -19,9 +20,16 @@ from datalake_scripts.scripts import (
19
20
  )
20
21
 
21
22
 
23
+ def _read_version():
24
+ try:
25
+ return _pkg_version("datalake_scripts")
26
+ except PackageNotFoundError:
27
+ return "0.0.0+unknown"
28
+
29
+
22
30
  class Cli:
23
31
  CLI_NAME = "ocd-dtl"
24
- VERSION = "3.0.1"
32
+ VERSION = _read_version()
25
33
 
26
34
  def __init__(self):
27
35
  parser = argparse.ArgumentParser(
@@ -0,0 +1,14 @@
1
+ from datalake.common.output_mode import rich_output_enabled
2
+ from datalake_scripts.common.rendering.base import Progress, Renderer
3
+ from datalake_scripts.common.rendering.rich_renderer import RichRenderer
4
+ from datalake_scripts.common.rendering.simple_renderer import SimpleRenderer
5
+
6
+
7
+ def get_renderer(dtl) -> Renderer:
8
+ """Factory: the only place choosing between the rich and the simple output"""
9
+ if rich_output_enabled():
10
+ return RichRenderer(dtl.logger)
11
+ return SimpleRenderer(dtl.logger, terminal_size=dtl.Threats.terminal_size)
12
+
13
+
14
+ __all__ = ["get_renderer", "Renderer", "Progress", "RichRenderer", "SimpleRenderer"]
@@ -0,0 +1,92 @@
1
+ """
2
+ Presentation layer of the CLI.
3
+
4
+ Scripts describe *what* has to be shown (a lookup report, a diff, a JSON
5
+ result...). A `Renderer` decides *how* it is shown. Scripts never know which
6
+ renderer they use: it is picked once by `get_renderer()`.
7
+ """
8
+
9
+ import logging
10
+ from abc import ABC, abstractmethod
11
+ from typing import List, Tuple
12
+
13
+
14
+ class Progress:
15
+ """
16
+ Context manager wrapping a long running step.
17
+ The default implementation displays nothing.
18
+
19
+ with renderer.progress("Creating bulk task") as progress:
20
+ ...
21
+ progress.update("Waiting for the bulk task")
22
+ """
23
+
24
+ def __enter__(self) -> "Progress":
25
+ return self
26
+
27
+ def __exit__(self, exc_type, exc_value, traceback):
28
+ return False
29
+
30
+ def update(self, text: str):
31
+ pass
32
+
33
+
34
+ class Renderer(ABC):
35
+ MAX_SEARCH_LINKS = 10
36
+
37
+ def __init__(self, logger: logging.Logger):
38
+ self.logger = logger
39
+
40
+ def progress(self, text: str) -> Progress:
41
+ """Display the progress of a long running step"""
42
+ return Progress()
43
+
44
+ @abstractmethod
45
+ def item_status(self, value: str, status: str, prefix: str = ""):
46
+ """Report the status (OK, KO...) of a single processed item"""
47
+
48
+ @abstractmethod
49
+ def json(self, data):
50
+ """Display a JSON result"""
51
+
52
+ @abstractmethod
53
+ def user_info(self, data: dict):
54
+ """Display the information of the current user"""
55
+
56
+ @abstractmethod
57
+ def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
58
+ """Display the result of a bulk search"""
59
+
60
+ @abstractmethod
61
+ def bulk_lookup(self, response, output_format: str, search_url: str):
62
+ """
63
+ Display the result of a bulk lookup.
64
+ :param response: aggregated API response
65
+ :param output_format: json, csv or anything else for a human readable report
66
+ :param search_url: GUI url of a search, to be completed with a search hashkey
67
+ """
68
+
69
+ @abstractmethod
70
+ def search_watch_diff(self, diff: dict):
71
+ """Display the added / removed threats of a search watch"""
72
+
73
+ @staticmethod
74
+ def _split_lookup_response(response: dict) -> Tuple[dict, List[str]]:
75
+ """Separate the atoms (grouped by atom type) from the search hashkeys"""
76
+ results = dict(response)
77
+ search_hashkeys = results.pop("search_hashkey", None) or []
78
+ return results, search_hashkeys
79
+
80
+ def _search_links_lines(
81
+ self, search_url: str, search_hashkeys: List[str]
82
+ ) -> List[str]:
83
+ if not search_hashkeys:
84
+ return ["No search hashkeys available for this lookup."]
85
+ if len(search_hashkeys) > self.MAX_SEARCH_LINKS:
86
+ return [
87
+ "Too many search hashkeys to display, check the output for the full list."
88
+ ]
89
+ return [
90
+ f"Results available here : {search_url}{search_hashkey}"
91
+ for search_hashkey in search_hashkeys
92
+ ]
@@ -0,0 +1,121 @@
1
+ import logging
2
+ from itertools import zip_longest
3
+ from typing import List
4
+
5
+ from rich.console import Console
6
+ from rich.markup import escape
7
+ from rich.table import Table
8
+ from rich.text import Text
9
+
10
+ from datalake.common.logger import THEME, console as err_console
11
+ from datalake_scripts.common.rendering.base import Progress, Renderer
12
+ from datalake_scripts.helper_scripts.output_builder import format_threats_output
13
+
14
+
15
+ class Spinner(Progress):
16
+ """Animated spinner on stderr, persisting a ✔ / ✖ line once the step is over"""
17
+
18
+ def __init__(self, text: str):
19
+ self.text = text
20
+ self._status = err_console.status(escape(text), spinner="dots")
21
+
22
+ def __enter__(self) -> "Spinner":
23
+ self._status.start()
24
+ return self
25
+
26
+ def __exit__(self, exc_type, exc_value, traceback):
27
+ self._status.stop()
28
+ mark = "[bold green]✔[/]" if exc_type is None else "[bold red]✖[/]"
29
+ err_console.print(f"{mark} {escape(self.text)}")
30
+ return False
31
+
32
+ def update(self, text: str):
33
+ self.text = text
34
+ self._status.update(escape(text))
35
+
36
+
37
+ class RichRenderer(Renderer):
38
+ """Colored output with spinners and tables, used when running in a terminal"""
39
+
40
+ def __init__(self, logger: logging.Logger):
41
+ super().__init__(logger)
42
+ self.out = Console(theme=THEME) # results go to stdout
43
+
44
+ def progress(self, text: str) -> Progress:
45
+ if not self.logger.isEnabledFor(logging.INFO): # --quiet
46
+ return super().progress(text)
47
+ return Spinner(text)
48
+
49
+ def item_status(self, value: str, status: str, prefix: str = ""):
50
+ self.logger.info(f"{prefix}{value}: {status}")
51
+
52
+ def json(self, data, sort_keys: bool = False):
53
+ self.out.print_json(data=data, indent=4, sort_keys=sort_keys)
54
+
55
+ def user_info(self, data: dict):
56
+ self.json(data)
57
+
58
+ def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
59
+ if one_threat_per_line:
60
+ print(format_threats_output(response, one_threat_per_line=True))
61
+ else:
62
+ self.json(response, sort_keys=True)
63
+
64
+ def bulk_lookup(self, response, output_format: str, search_url: str):
65
+ if output_format == "json":
66
+ self.json(response, sort_keys=True)
67
+ return
68
+ if output_format == "csv":
69
+ print(response)
70
+ return
71
+
72
+ results, search_hashkeys = self._split_lookup_response(response)
73
+ for atom_type, atoms in results.items():
74
+ table = self._table(title=f"[bold]{escape(atom_type.upper())}[/]")
75
+ table.add_column("Atom value", overflow="fold")
76
+ table.add_column("Hashkey", no_wrap=True)
77
+ table.add_column("Status", justify="center", no_wrap=True)
78
+ for atom in atoms:
79
+ found = atom.get("threat_found", False)
80
+ table.add_row(
81
+ Text(atom["atom_value"]),
82
+ Text(atom["hashkey"]),
83
+ self._status_badge(found, "FOUND", "NOT_FOUND"),
84
+ )
85
+ self.out.print(table)
86
+ self.out.print()
87
+ for line in self._search_links_lines(search_url, search_hashkeys):
88
+ self.out.print(Text(line))
89
+
90
+ def search_watch_diff(self, diff: dict):
91
+ added = sorted(diff.get("added", []), key=lambda x: x[0])
92
+ removed = sorted(diff.get("removed", []), key=lambda x: x[0])
93
+
94
+ self.out.print()
95
+ self.out.print(
96
+ f"[bold]Threats summary from {escape(str(diff['from']))} to {escape(str(diff['to']))}[/]\n"
97
+ f"Number of added items: [bold green]{len(added)}[/]\n"
98
+ f"Number of removed items: [bold red]{len(removed)}[/]\n"
99
+ )
100
+
101
+ table = self._table()
102
+ table.add_column(
103
+ "Added (atom value - hash key)", style="green", overflow="fold"
104
+ )
105
+ table.add_column(
106
+ "Removed (atom value - hash key)", style="red", overflow="fold"
107
+ )
108
+ for add, remove in zip_longest(added, removed, fillvalue=None):
109
+ table.add_row(
110
+ Text(f"{add[0]} - {add[1]}" if add else ""),
111
+ Text(f"{remove[0]} - {remove[1]}" if remove else ""),
112
+ )
113
+ self.out.print(table)
114
+
115
+ @staticmethod
116
+ def _table(title: str = None) -> Table:
117
+ return Table(title=title, title_justify="left", header_style="bold")
118
+
119
+ @staticmethod
120
+ def _status_badge(ok: bool, ok_label: str, ko_label: str) -> str:
121
+ return f"[bold green]{ok_label}[/]" if ok else f"[bold red]{ko_label}[/]"
@@ -0,0 +1,107 @@
1
+ import json
2
+ import logging
3
+ from itertools import zip_longest
4
+ from typing import List
5
+
6
+ from datalake_scripts.common.rendering.base import Renderer
7
+ from datalake_scripts.helper_scripts.output_builder import format_threats_output
8
+
9
+
10
+ class SimpleRenderer(Renderer):
11
+ """
12
+ Plain text output used for scripted usages (no tty).
13
+ The layouts are kept identical to the historical output of the CLI so that
14
+ customers parsing it are not disturbed.
15
+ """
16
+
17
+ DIFF_HEADERS = [
18
+ "Added (atom value - hash key) ",
19
+ "Removed (atom value - hash key)",
20
+ ]
21
+
22
+ def __init__(self, logger: logging.Logger, terminal_size: int):
23
+ super().__init__(logger)
24
+ self.terminal_size = terminal_size
25
+
26
+ def item_status(self, value: str, status: str, prefix: str = ""):
27
+ self.logger.info(
28
+ f'{prefix}{value.ljust(self.terminal_size - 6, " ")} {status} '
29
+ )
30
+
31
+ def json(self, data):
32
+ print(json.dumps(data, indent=4))
33
+
34
+ def user_info(self, data: dict):
35
+ print(data)
36
+
37
+ def threats_from_query_hash(self, response: dict, one_threat_per_line: bool):
38
+ self.logger.info(format_threats_output(response, one_threat_per_line))
39
+ self.logger.info("Done")
40
+
41
+ def bulk_lookup(self, response, output_format: str, search_url: str):
42
+ if output_format == "json":
43
+ self.logger.info(json.dumps(response, indent=4, sort_keys=True))
44
+ return
45
+ if output_format == "csv":
46
+ self.logger.info(response)
47
+ return
48
+
49
+ results, search_hashkeys = self._split_lookup_response(response)
50
+ for atom_type, atoms in results.items():
51
+ self.logger.info(
52
+ f'{"#" * 50} {atom_type.upper()} {"#" * (50 - len(atom_type))}'
53
+ )
54
+ for atom in atoms:
55
+ status = "FOUND" if atom.get("threat_found", False) else "NOT_FOUND"
56
+ self.logger.info(
57
+ f'{atom_type : <11} {atom["atom_value"][:29] : <30} hashkey: {atom["hashkey"]} {status :^15} '
58
+ )
59
+ self.logger.info("")
60
+ for line in self._search_links_lines(search_url, search_hashkeys):
61
+ self.logger.info(line)
62
+
63
+ def search_watch_diff(self, diff: dict):
64
+ rows = []
65
+ for add, remove in zip_longest(
66
+ sorted(diff.get("added", []), key=lambda x: x[0]),
67
+ sorted(diff.get("removed", []), key=lambda x: x[0]),
68
+ fillvalue=["", ""],
69
+ ):
70
+ rows.append(
71
+ [
72
+ f"{add[0]} - {add[1]}" if add else "",
73
+ f"{remove[0]} - {remove[1]}" if remove else "",
74
+ ]
75
+ )
76
+
77
+ summary = f"\nThreats summary from {diff['from']} to {diff['to']}:\n"
78
+ summary += f"Number of added items: {len(diff['added'])}\n"
79
+ summary += f"Number of removed items: {len(diff['removed'])}\n"
80
+
81
+ print(summary)
82
+ print(ascii_table(self.DIFF_HEADERS, rows))
83
+
84
+
85
+ def _center(text: str, width: int) -> str:
86
+ """Center a cell the same way the previous table library (prettytable) did"""
87
+ excess = width - len(text)
88
+ if excess % 2 and len(text) % 2 == 0:
89
+ return " " * (excess // 2 + 1) + text + " " * (excess // 2)
90
+ return " " * (excess // 2) + text + " " * (excess - excess // 2)
91
+
92
+
93
+ def ascii_table(headers: List[str], rows: List[List[str]]) -> str:
94
+ widths = [
95
+ max([len(headers[i])] + [len(row[i]) for row in rows])
96
+ for i in range(len(headers))
97
+ ]
98
+ separator = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
99
+
100
+ def format_row(cells):
101
+ cells = (f" {_center(cell, width)} " for cell, width in zip(cells, widths))
102
+ return "|" + "|".join(cells) + "|"
103
+
104
+ lines = [separator, format_row(headers), separator]
105
+ lines += [format_row(row) for row in rows]
106
+ lines.append(separator)
107
+ return "\n".join(lines)
@@ -1,3 +1,6 @@
1
+ import json
2
+
3
+
1
4
  class CsvBuilder:
2
5
  @staticmethod
3
6
  def create_look_up_csv(
@@ -82,3 +85,14 @@ class CsvBuilder:
82
85
  return None
83
86
  tags = ",".join([tag["name"] for tag in tags]).replace('"', '""')
84
87
  return f'"{tags}"'
88
+
89
+
90
+ def format_threats_output(response: dict, one_threat_per_line=False) -> str:
91
+ """Format a bulk search response as JSON, or as one threat per line"""
92
+ if one_threat_per_line:
93
+ threat_list = []
94
+ for result in response.get("results", []):
95
+ if result:
96
+ threat_list += result
97
+ return "\n".join(threat_list)
98
+ return json.dumps(response, sort_keys=True, indent=4)