secator 0.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- secator/.gitignore +162 -0
- secator/__init__.py +0 -0
- secator/celery.py +453 -0
- secator/celery_signals.py +138 -0
- secator/celery_utils.py +320 -0
- secator/cli.py +2035 -0
- secator/cli_helper.py +395 -0
- secator/click.py +87 -0
- secator/config.py +670 -0
- secator/configs/__init__.py +0 -0
- secator/configs/profiles/__init__.py +0 -0
- secator/configs/profiles/aggressive.yaml +8 -0
- secator/configs/profiles/all_ports.yaml +7 -0
- secator/configs/profiles/full.yaml +31 -0
- secator/configs/profiles/http_headless.yaml +7 -0
- secator/configs/profiles/http_record.yaml +8 -0
- secator/configs/profiles/insane.yaml +8 -0
- secator/configs/profiles/paranoid.yaml +8 -0
- secator/configs/profiles/passive.yaml +11 -0
- secator/configs/profiles/polite.yaml +8 -0
- secator/configs/profiles/sneaky.yaml +8 -0
- secator/configs/profiles/tor.yaml +5 -0
- secator/configs/scans/__init__.py +0 -0
- secator/configs/scans/domain.yaml +31 -0
- secator/configs/scans/host.yaml +23 -0
- secator/configs/scans/network.yaml +30 -0
- secator/configs/scans/subdomain.yaml +27 -0
- secator/configs/scans/url.yaml +19 -0
- secator/configs/workflows/__init__.py +0 -0
- secator/configs/workflows/cidr_recon.yaml +48 -0
- secator/configs/workflows/code_scan.yaml +29 -0
- secator/configs/workflows/domain_recon.yaml +46 -0
- secator/configs/workflows/host_recon.yaml +95 -0
- secator/configs/workflows/subdomain_recon.yaml +120 -0
- secator/configs/workflows/url_bypass.yaml +15 -0
- secator/configs/workflows/url_crawl.yaml +98 -0
- secator/configs/workflows/url_dirsearch.yaml +62 -0
- secator/configs/workflows/url_fuzz.yaml +68 -0
- secator/configs/workflows/url_params_fuzz.yaml +66 -0
- secator/configs/workflows/url_secrets_hunt.yaml +23 -0
- secator/configs/workflows/url_vuln.yaml +91 -0
- secator/configs/workflows/user_hunt.yaml +29 -0
- secator/configs/workflows/wordpress.yaml +38 -0
- secator/cve.py +718 -0
- secator/decorators.py +7 -0
- secator/definitions.py +168 -0
- secator/exporters/__init__.py +14 -0
- secator/exporters/_base.py +3 -0
- secator/exporters/console.py +10 -0
- secator/exporters/csv.py +37 -0
- secator/exporters/gdrive.py +123 -0
- secator/exporters/json.py +16 -0
- secator/exporters/table.py +36 -0
- secator/exporters/txt.py +28 -0
- secator/hooks/__init__.py +0 -0
- secator/hooks/gcs.py +80 -0
- secator/hooks/mongodb.py +281 -0
- secator/installer.py +694 -0
- secator/loader.py +128 -0
- secator/output_types/__init__.py +49 -0
- secator/output_types/_base.py +108 -0
- secator/output_types/certificate.py +78 -0
- secator/output_types/domain.py +50 -0
- secator/output_types/error.py +42 -0
- secator/output_types/exploit.py +58 -0
- secator/output_types/info.py +24 -0
- secator/output_types/ip.py +47 -0
- secator/output_types/port.py +55 -0
- secator/output_types/progress.py +36 -0
- secator/output_types/record.py +36 -0
- secator/output_types/stat.py +41 -0
- secator/output_types/state.py +29 -0
- secator/output_types/subdomain.py +45 -0
- secator/output_types/tag.py +69 -0
- secator/output_types/target.py +38 -0
- secator/output_types/url.py +112 -0
- secator/output_types/user_account.py +41 -0
- secator/output_types/vulnerability.py +101 -0
- secator/output_types/warning.py +30 -0
- secator/report.py +140 -0
- secator/rich.py +130 -0
- secator/runners/__init__.py +14 -0
- secator/runners/_base.py +1240 -0
- secator/runners/_helpers.py +218 -0
- secator/runners/celery.py +18 -0
- secator/runners/command.py +1178 -0
- secator/runners/python.py +126 -0
- secator/runners/scan.py +87 -0
- secator/runners/task.py +81 -0
- secator/runners/workflow.py +168 -0
- secator/scans/__init__.py +29 -0
- secator/serializers/__init__.py +8 -0
- secator/serializers/dataclass.py +39 -0
- secator/serializers/json.py +45 -0
- secator/serializers/regex.py +25 -0
- secator/tasks/__init__.py +8 -0
- secator/tasks/_categories.py +487 -0
- secator/tasks/arjun.py +113 -0
- secator/tasks/arp.py +53 -0
- secator/tasks/arpscan.py +70 -0
- secator/tasks/bbot.py +372 -0
- secator/tasks/bup.py +118 -0
- secator/tasks/cariddi.py +193 -0
- secator/tasks/dalfox.py +87 -0
- secator/tasks/dirsearch.py +84 -0
- secator/tasks/dnsx.py +186 -0
- secator/tasks/feroxbuster.py +93 -0
- secator/tasks/ffuf.py +135 -0
- secator/tasks/fping.py +85 -0
- secator/tasks/gau.py +102 -0
- secator/tasks/getasn.py +60 -0
- secator/tasks/gf.py +36 -0
- secator/tasks/gitleaks.py +96 -0
- secator/tasks/gospider.py +84 -0
- secator/tasks/grype.py +109 -0
- secator/tasks/h8mail.py +75 -0
- secator/tasks/httpx.py +167 -0
- secator/tasks/jswhois.py +36 -0
- secator/tasks/katana.py +203 -0
- secator/tasks/maigret.py +87 -0
- secator/tasks/mapcidr.py +42 -0
- secator/tasks/msfconsole.py +179 -0
- secator/tasks/naabu.py +85 -0
- secator/tasks/nmap.py +487 -0
- secator/tasks/nuclei.py +151 -0
- secator/tasks/search_vulns.py +225 -0
- secator/tasks/searchsploit.py +109 -0
- secator/tasks/sshaudit.py +299 -0
- secator/tasks/subfinder.py +48 -0
- secator/tasks/testssl.py +283 -0
- secator/tasks/trivy.py +130 -0
- secator/tasks/trufflehog.py +240 -0
- secator/tasks/urlfinder.py +100 -0
- secator/tasks/wafw00f.py +106 -0
- secator/tasks/whois.py +34 -0
- secator/tasks/wpprobe.py +116 -0
- secator/tasks/wpscan.py +202 -0
- secator/tasks/x8.py +94 -0
- secator/tasks/xurlfind3r.py +83 -0
- secator/template.py +294 -0
- secator/thread.py +24 -0
- secator/tree.py +196 -0
- secator/utils.py +922 -0
- secator/utils_test.py +297 -0
- secator/workflows/__init__.py +29 -0
- secator-0.22.0.dist-info/METADATA +447 -0
- secator-0.22.0.dist-info/RECORD +150 -0
- secator-0.22.0.dist-info/WHEEL +4 -0
- secator-0.22.0.dist-info/entry_points.txt +2 -0
- secator-0.22.0.dist-info/licenses/LICENSE +60 -0
secator/cli.py
ADDED
|
@@ -0,0 +1,2035 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import shutil
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from stat import S_ISFIFO
|
|
9
|
+
|
|
10
|
+
import rich_click as click
|
|
11
|
+
from dotmap import DotMap
|
|
12
|
+
from fp.fp import FreeProxy
|
|
13
|
+
from jinja2 import Template
|
|
14
|
+
from rich.live import Live
|
|
15
|
+
from rich.markdown import Markdown
|
|
16
|
+
from rich.rule import Rule
|
|
17
|
+
from rich.table import Table
|
|
18
|
+
|
|
19
|
+
from secator.config import CONFIG, ROOT_FOLDER, Config, default_config, config_path, download_files
|
|
20
|
+
from secator.click import OrderedGroup
|
|
21
|
+
from secator.cli_helper import register_runner
|
|
22
|
+
from secator.definitions import ADDONS_ENABLED, ASCII, DEV_PACKAGE, FORCE_TTY, VERSION, STATE_COLORS
|
|
23
|
+
from secator.installer import ToolInstaller, fmt_health_table_row, get_health_table, get_version_info, get_distro_config
|
|
24
|
+
from secator.output_types import FINDING_TYPES, Info, Warning, Error
|
|
25
|
+
from secator.report import Report
|
|
26
|
+
from secator.rich import console
|
|
27
|
+
from secator.runners import Command, Runner
|
|
28
|
+
from secator.serializers.dataclass import loads_dataclass
|
|
29
|
+
from secator.loader import get_configs_by_type, discover_tasks
|
|
30
|
+
from secator.utils import (
|
|
31
|
+
debug, detect_host, flatten, print_version, get_file_date, is_terminal_interactive,
|
|
32
|
+
sort_files_by_date, get_file_timestamp, list_reports, get_info_from_report_path, human_to_timedelta
|
|
33
|
+
)
|
|
34
|
+
from contextlib import nullcontext
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
click.rich_click.USE_RICH_MARKUP = True
|
|
38
|
+
click.rich_click.STYLE_ARGUMENT = ""
|
|
39
|
+
click.rich_click.STYLE_OPTION_HELP = ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
FINDING_TYPES_LOWER = [c.__name__.lower() for c in FINDING_TYPES]
|
|
43
|
+
CONTEXT_SETTINGS = dict(help_option_names=['-h', '-help', '--help'])
|
|
44
|
+
TASKS = get_configs_by_type('task')
|
|
45
|
+
WORKFLOWS = get_configs_by_type('workflow')
|
|
46
|
+
SCANS = get_configs_by_type('scan')
|
|
47
|
+
PROFILES = get_configs_by_type('profile')
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
#-----#
|
|
51
|
+
# CLI #
|
|
52
|
+
#-----#
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@click.group(cls=OrderedGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
|
|
56
|
+
@click.option('--version', '-version', '-v', is_flag=True, default=False)
|
|
57
|
+
@click.option('--quiet', '-quiet', '-q', is_flag=True, default=False)
|
|
58
|
+
@click.pass_context
|
|
59
|
+
def cli(ctx, version, quiet):
|
|
60
|
+
"""Secator CLI."""
|
|
61
|
+
ctx.obj = {
|
|
62
|
+
'piped_input': (is_terminal_interactive() or FORCE_TTY) and S_ISFIFO(os.fstat(0).st_mode),
|
|
63
|
+
'piped_output': not sys.stdout.isatty()
|
|
64
|
+
}
|
|
65
|
+
if not ctx.obj['piped_output'] and not quiet:
|
|
66
|
+
console.print(ASCII, highlight=False)
|
|
67
|
+
if ctx.invoked_subcommand is None:
|
|
68
|
+
if version:
|
|
69
|
+
print_version()
|
|
70
|
+
else:
|
|
71
|
+
ctx.get_help()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
#------#
|
|
75
|
+
# TASK #
|
|
76
|
+
#------#
|
|
77
|
+
|
|
78
|
+
@cli.group(aliases=['x', 't', 'tasks'], invoke_without_command=True)
|
|
79
|
+
@click.pass_context
|
|
80
|
+
def task(ctx):
|
|
81
|
+
"""Run a task."""
|
|
82
|
+
if ctx.invoked_subcommand is None:
|
|
83
|
+
ctx.get_help()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
for config in TASKS:
|
|
87
|
+
register_runner(task, config)
|
|
88
|
+
|
|
89
|
+
#----------#
|
|
90
|
+
# WORKFLOW #
|
|
91
|
+
#----------#
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@cli.group(cls=OrderedGroup, aliases=['w', 'workflows'], invoke_without_command=True)
|
|
95
|
+
@click.pass_context
|
|
96
|
+
def workflow(ctx):
|
|
97
|
+
"""Run a workflow."""
|
|
98
|
+
if ctx.invoked_subcommand is None:
|
|
99
|
+
ctx.get_help()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
for config in WORKFLOWS:
|
|
103
|
+
register_runner(workflow, config)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
#------#
|
|
107
|
+
# SCAN #
|
|
108
|
+
#------#
|
|
109
|
+
|
|
110
|
+
@cli.group(cls=OrderedGroup, aliases=['s', 'scans'], invoke_without_command=True)
|
|
111
|
+
@click.pass_context
|
|
112
|
+
def scan(ctx):
|
|
113
|
+
"""Run a scan."""
|
|
114
|
+
if ctx.invoked_subcommand is None:
|
|
115
|
+
ctx.get_help()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
for config in SCANS:
|
|
119
|
+
register_runner(scan, config)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
#--------#
|
|
123
|
+
# WORKER #
|
|
124
|
+
#--------#
|
|
125
|
+
|
|
126
|
+
@cli.command(name='worker', context_settings=dict(ignore_unknown_options=True), aliases=['wk'])
|
|
127
|
+
@click.option('-n', '--hostname', type=str, default='runner', help='Celery worker hostname (unique).')
|
|
128
|
+
@click.option('-c', '--concurrency', type=int, default=100, help='Number of child processes processing the queue.')
|
|
129
|
+
@click.option('-r', '--reload', is_flag=True, help='Autoreload Celery on code changes.')
|
|
130
|
+
@click.option('-Q', '--queue', type=str, default='', help='Listen to a specific queue.')
|
|
131
|
+
@click.option('-P', '--pool', type=str, default='eventlet', help='Pool implementation.')
|
|
132
|
+
@click.option('--quiet', is_flag=True, default=False, help='Quiet mode.')
|
|
133
|
+
@click.option('--loglevel', type=str, default='INFO', help='Log level.')
|
|
134
|
+
@click.option('--check', is_flag=True, help='Check if Celery worker is alive.')
|
|
135
|
+
@click.option('--dev', is_flag=True, help='Start a worker in dev mode (celery multi).')
|
|
136
|
+
@click.option('--stop', is_flag=True, help='Stop a worker in dev mode (celery multi).')
|
|
137
|
+
@click.option('--show', is_flag=True, help='Show command (celery multi).')
|
|
138
|
+
@click.option('--use-command-runner', is_flag=True, default=False, help='Use command runner to run the command.')
|
|
139
|
+
@click.option('--without-gossip', is_flag=True)
|
|
140
|
+
@click.option('--without-mingle', is_flag=True)
|
|
141
|
+
@click.option('--without-heartbeat', is_flag=True)
|
|
142
|
+
def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, dev, stop, show, use_command_runner, without_gossip, without_mingle, without_heartbeat): # noqa: E501
|
|
143
|
+
"""Run a worker."""
|
|
144
|
+
|
|
145
|
+
# Check Celery addon is installed
|
|
146
|
+
if not ADDONS_ENABLED['worker']:
|
|
147
|
+
console.print(Error(message='Missing worker addon: please run "secator install addons worker".'))
|
|
148
|
+
sys.exit(1)
|
|
149
|
+
|
|
150
|
+
# Check broken / backend addon is installed
|
|
151
|
+
broker_protocol = CONFIG.celery.broker_url.split('://')[0]
|
|
152
|
+
backend_protocol = CONFIG.celery.result_backend.split('://')[0]
|
|
153
|
+
if CONFIG.celery.broker_url and \
|
|
154
|
+
(broker_protocol == 'redis' or backend_protocol == 'redis') and \
|
|
155
|
+
not ADDONS_ENABLED['redis']:
|
|
156
|
+
console.print(Error(message='Missing redis addon: please run "secator install addons redis".'))
|
|
157
|
+
sys.exit(1)
|
|
158
|
+
|
|
159
|
+
# Debug Celery config
|
|
160
|
+
from secator.celery import app, is_celery_worker_alive
|
|
161
|
+
debug('conf', obj=dict(app.conf), obj_breaklines=True, sub='celery.app')
|
|
162
|
+
debug('registered tasks', obj=list(app.tasks.keys()), obj_breaklines=True, sub='celery.app')
|
|
163
|
+
|
|
164
|
+
if check:
|
|
165
|
+
is_celery_worker_alive()
|
|
166
|
+
return
|
|
167
|
+
|
|
168
|
+
if not queue:
|
|
169
|
+
queue = 'io,cpu,poll,' + ','.join(set([r['queue'] for r in app.conf.task_routes.values()]))
|
|
170
|
+
|
|
171
|
+
app_str = 'secator.celery.app'
|
|
172
|
+
celery = f'{sys.executable} -m celery'
|
|
173
|
+
if quiet:
|
|
174
|
+
celery += ' --quiet'
|
|
175
|
+
|
|
176
|
+
if dev:
|
|
177
|
+
subcmd = 'stop' if stop else 'show' if show else 'start'
|
|
178
|
+
logfile = '%n.log'
|
|
179
|
+
pidfile = '%n.pid'
|
|
180
|
+
queues = '-Q:1 celery -Q:2 io -Q:3 cpu'
|
|
181
|
+
concur = '-c:1 10 -c:2 100 -c:3 4'
|
|
182
|
+
pool = 'eventlet'
|
|
183
|
+
cmd = f'{celery} -A {app_str} multi {subcmd} 3 {queues} -P {pool} {concur} --logfile={logfile} --pidfile={pidfile}'
|
|
184
|
+
else:
|
|
185
|
+
cmd = f'{celery} -A {app_str} worker -n {hostname} -Q {queue}'
|
|
186
|
+
|
|
187
|
+
cmd += f' -P {pool}' if pool else ''
|
|
188
|
+
cmd += f' -c {concurrency}' if concurrency else ''
|
|
189
|
+
cmd += f' -l {loglevel}' if loglevel else ''
|
|
190
|
+
cmd += ' --without-mingle' if without_mingle else ''
|
|
191
|
+
cmd += ' --without-gossip' if without_gossip else ''
|
|
192
|
+
cmd += ' --without-heartbeat' if without_heartbeat else ''
|
|
193
|
+
|
|
194
|
+
if reload:
|
|
195
|
+
patterns = "celery.py;tasks/*.py;runners/*.py;serializers/*.py;output_types/*.py;hooks/*.py;exporters/*.py"
|
|
196
|
+
cmd = f'watchmedo auto-restart --directory=./ --patterns="{patterns}" --recursive -- {cmd}'
|
|
197
|
+
|
|
198
|
+
if use_command_runner:
|
|
199
|
+
ret = Command.execute(cmd, name='secator_worker')
|
|
200
|
+
sys.exit(ret.return_code)
|
|
201
|
+
else:
|
|
202
|
+
console.print(f'[bold red]{cmd}[/]')
|
|
203
|
+
ret = os.system(cmd)
|
|
204
|
+
sys.exit(os.waitstatus_to_exitcode(ret))
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
#-------#
|
|
208
|
+
# UTILS #
|
|
209
|
+
#-------#
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@cli.group(aliases=['u'])
|
|
213
|
+
def util():
|
|
214
|
+
"""Run a utility."""
|
|
215
|
+
pass
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@util.command()
|
|
219
|
+
@click.option('--timeout', type=float, default=0.2, help='Proxy timeout (in seconds)')
|
|
220
|
+
@click.option('--number', '-n', type=int, default=1, help='Number of proxies')
|
|
221
|
+
def proxy(timeout, number):
|
|
222
|
+
"""Get random proxies from FreeProxy."""
|
|
223
|
+
if CONFIG.offline_mode:
|
|
224
|
+
console.print(Error(message='Cannot run this command in offline mode.'))
|
|
225
|
+
sys.exit(1)
|
|
226
|
+
proxy = FreeProxy(timeout=timeout, rand=True, anonym=True)
|
|
227
|
+
for _ in range(number):
|
|
228
|
+
url = proxy.get()
|
|
229
|
+
console.print(url)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@util.command()
|
|
233
|
+
@click.argument('name', type=str, default=None, required=False)
|
|
234
|
+
@click.option('--host', '-h', type=str, default=None, help='Specify LHOST for revshell, otherwise autodetected.')
|
|
235
|
+
@click.option('--port', '-p', type=int, default=9001, show_default=True, help='Specify PORT for revshell')
|
|
236
|
+
@click.option('--interface', '-i', type=str, help='Interface to use to detect IP')
|
|
237
|
+
@click.option('--listen', '-l', is_flag=True, default=False, help='Spawn netcat listener on specified port')
|
|
238
|
+
@click.option('--force', is_flag=True)
|
|
239
|
+
def revshell(name, host, port, interface, listen, force):
|
|
240
|
+
"""Show reverse shell source codes and run netcat listener (-l)."""
|
|
241
|
+
if host is None: # detect host automatically
|
|
242
|
+
host = detect_host(interface)
|
|
243
|
+
if not host:
|
|
244
|
+
console.print(Error(message=f'Interface "{interface}" could not be found. Run "ifconfig" to see the list of available interfaces')) # noqa: E501
|
|
245
|
+
sys.exit(1)
|
|
246
|
+
else:
|
|
247
|
+
console.print(Info(message=f'Detected host IP: {host}'))
|
|
248
|
+
|
|
249
|
+
# Download reverse shells JSON from repo
|
|
250
|
+
revshells_json = f'{CONFIG.dirs.revshells}/revshells.json'
|
|
251
|
+
if not os.path.exists(revshells_json) or force:
|
|
252
|
+
if CONFIG.offline_mode:
|
|
253
|
+
console.print(Error(message='Cannot run this command in offline mode'))
|
|
254
|
+
sys.exit(1)
|
|
255
|
+
ret = Command.execute(
|
|
256
|
+
f'wget https://raw.githubusercontent.com/freelabz/secator/main/scripts/revshells.json && mv revshells.json {CONFIG.dirs.revshells}', # noqa: E501
|
|
257
|
+
cls_attributes={'shell': True}
|
|
258
|
+
)
|
|
259
|
+
if not ret.return_code == 0:
|
|
260
|
+
sys.exit(1)
|
|
261
|
+
|
|
262
|
+
# Parse JSON into shells
|
|
263
|
+
with open(revshells_json) as f:
|
|
264
|
+
shells = json.loads(f.read())
|
|
265
|
+
for sh in shells:
|
|
266
|
+
sh['alias'] = '_'.join(sh['name'].lower()
|
|
267
|
+
.replace('-c', '')
|
|
268
|
+
.replace('-e', '')
|
|
269
|
+
.replace('-i', '')
|
|
270
|
+
.replace('c#', 'cs')
|
|
271
|
+
.replace('#', '')
|
|
272
|
+
.replace('(', '')
|
|
273
|
+
.replace(')', '')
|
|
274
|
+
.strip()
|
|
275
|
+
.split(' ')).replace('_1', '')
|
|
276
|
+
cmd = re.sub(r"\s\s+", "", sh.get('command', ''), flags=re.UNICODE)
|
|
277
|
+
cmd = cmd.replace('\n', ' ')
|
|
278
|
+
sh['cmd_short'] = (cmd[:30] + '..') if len(cmd) > 30 else cmd
|
|
279
|
+
|
|
280
|
+
shell = [
|
|
281
|
+
shell for shell in shells if shell['name'] == name or shell['alias'] == name
|
|
282
|
+
]
|
|
283
|
+
if not shell:
|
|
284
|
+
console.print('Available shells:', style='bold yellow')
|
|
285
|
+
shells_str = [
|
|
286
|
+
'[bold magenta]{alias:<20}[/][dim white]{name:<20}[/][dim gold3]{cmd_short:<20}[/]'.format(**sh)
|
|
287
|
+
for sh in shells
|
|
288
|
+
]
|
|
289
|
+
console.print('\n'.join(shells_str))
|
|
290
|
+
else:
|
|
291
|
+
shell = shell[0]
|
|
292
|
+
command = shell['command'].replace('[', r'\[')
|
|
293
|
+
alias = shell['alias']
|
|
294
|
+
name = shell['name']
|
|
295
|
+
command_str = Template(command).render(ip=host, port=port, shell='bash')
|
|
296
|
+
console.print(Rule(f'[bold gold3]{alias}[/] - [bold red]{name} REMOTE SHELL', style='bold red', align='left'))
|
|
297
|
+
lang = shell.get('lang') or 'sh'
|
|
298
|
+
if len(command.splitlines()) == 1:
|
|
299
|
+
console.print(command_str, style='cyan', highlight=False, soft_wrap=True)
|
|
300
|
+
else:
|
|
301
|
+
md = Markdown(f'```{lang}\n{command_str}\n```')
|
|
302
|
+
console.print(md)
|
|
303
|
+
console.print(f'Save this script as rev.{lang} and run it on your target', style='dim italic')
|
|
304
|
+
console.print()
|
|
305
|
+
console.print(Rule(style='bold red'))
|
|
306
|
+
|
|
307
|
+
if listen:
|
|
308
|
+
console.print(Info(message=f'Starting netcat listener on port {port} ...'))
|
|
309
|
+
cmd = f'nc -lvnp {port}'
|
|
310
|
+
Command.execute(cmd)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
@util.command()
|
|
314
|
+
@click.option('--directory', '-d', type=str, default=CONFIG.dirs.payloads, help='HTTP server directory')
|
|
315
|
+
@click.option('--host', '-h', type=str, default=None, help='HTTP host')
|
|
316
|
+
@click.option('--port', '-p', type=int, default=9001, help='HTTP server port')
|
|
317
|
+
@click.option('--interface', '-i', type=str, default=None, help='Interface to use to auto-detect host IP')
|
|
318
|
+
def serve(directory, host, port, interface):
|
|
319
|
+
"""Run HTTP server to serve payloads."""
|
|
320
|
+
fnames = list(os.listdir(directory))
|
|
321
|
+
if not fnames:
|
|
322
|
+
console.print(Warning(message=f'No payloads found in {directory}.'))
|
|
323
|
+
download_files(CONFIG.payloads.templates, CONFIG.dirs.payloads, CONFIG.offline_mode, 'payload')
|
|
324
|
+
fnames = list(os.listdir(directory))
|
|
325
|
+
|
|
326
|
+
console.print(Rule())
|
|
327
|
+
console.print(f'Available payloads in {directory}: ', style='bold yellow')
|
|
328
|
+
fnames.sort()
|
|
329
|
+
for fname in fnames:
|
|
330
|
+
if not host:
|
|
331
|
+
host = detect_host(interface)
|
|
332
|
+
if not host:
|
|
333
|
+
console.print(Error(message=f'Interface "{interface}" could not be found. Run "ifconfig" to see the list of interfaces.')) # noqa: E501
|
|
334
|
+
return
|
|
335
|
+
console.print(f'{fname} [dim][/]', style='bold magenta')
|
|
336
|
+
console.print(f'wget http://{host}:{port}/{fname}', style='dim italic')
|
|
337
|
+
console.print('')
|
|
338
|
+
console.print(Rule())
|
|
339
|
+
console.print(Info(message=f'[bold yellow]Started HTTP server on port {port}, waiting for incoming connections ...[/]')) # noqa: E501
|
|
340
|
+
Command.execute(f'{sys.executable} -m http.server {port}', cwd=directory)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@util.command()
|
|
344
|
+
@click.argument('record_name', type=str, default=None)
|
|
345
|
+
@click.option('--script', '-s', type=str, default=None, help='Script to run. See scripts/stories/ for examples.')
|
|
346
|
+
@click.option('--interactive', '-i', is_flag=True, default=False, help='Interactive record.')
|
|
347
|
+
@click.option('--width', '-w', type=int, default=None, help='Recording width')
|
|
348
|
+
@click.option('--height', '-h', type=int, default=None, help='Recording height')
|
|
349
|
+
@click.option('--output-dir', type=str, default=f'{ROOT_FOLDER}/images')
|
|
350
|
+
def record(record_name, script, interactive, width, height, output_dir):
|
|
351
|
+
"""Record secator session using asciinema."""
|
|
352
|
+
# 120 x 30 is a good ratio for GitHub
|
|
353
|
+
width = width or console.size.width
|
|
354
|
+
height = height or console.size.height
|
|
355
|
+
attrs = {
|
|
356
|
+
'shell': False,
|
|
357
|
+
'env': {
|
|
358
|
+
'RECORD': '1',
|
|
359
|
+
'LINES': str(height),
|
|
360
|
+
'PS1': '$ ',
|
|
361
|
+
'COLUMNS': str(width),
|
|
362
|
+
'TERM': 'xterm-256color'
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
output_cast_path = f'{output_dir}/{record_name}.cast'
|
|
366
|
+
output_gif_path = f'{output_dir}/{record_name}.gif'
|
|
367
|
+
|
|
368
|
+
# Run automated 'story' script with asciinema-automation
|
|
369
|
+
if script:
|
|
370
|
+
# If existing cast file, remove it
|
|
371
|
+
if os.path.exists(output_cast_path):
|
|
372
|
+
os.unlink(output_cast_path)
|
|
373
|
+
console.print(Info(message=f'Removed existing {output_cast_path}'))
|
|
374
|
+
|
|
375
|
+
with console.status(Info(message='Recording with asciinema ...')):
|
|
376
|
+
Command.execute(
|
|
377
|
+
f'asciinema-automation -aa "-c /bin/sh" {script} {output_cast_path} --timeout 200',
|
|
378
|
+
cls_attributes=attrs,
|
|
379
|
+
raw=True,
|
|
380
|
+
)
|
|
381
|
+
console.print(f'Generated {output_cast_path}', style='bold green')
|
|
382
|
+
elif interactive:
|
|
383
|
+
os.environ.update(attrs['env'])
|
|
384
|
+
Command.execute(f'asciinema rec -c /bin/bash --stdin --overwrite {output_cast_path}')
|
|
385
|
+
|
|
386
|
+
# Resize cast file
|
|
387
|
+
if os.path.exists(output_cast_path):
|
|
388
|
+
with console.status('[bold gold3]Cleaning up .cast and set custom settings ...'):
|
|
389
|
+
with open(output_cast_path, 'r') as f:
|
|
390
|
+
lines = f.readlines()
|
|
391
|
+
updated_lines = []
|
|
392
|
+
for ix, line in enumerate(lines):
|
|
393
|
+
tmp_line = json.loads(line)
|
|
394
|
+
if ix == 0:
|
|
395
|
+
tmp_line['width'] = width
|
|
396
|
+
tmp_line['height'] = height
|
|
397
|
+
tmp_line['env']['SHELL'] = '/bin/sh'
|
|
398
|
+
lines[0] = json.dumps(tmp_line) + '\n'
|
|
399
|
+
updated_lines.append(json.dumps(tmp_line) + '\n')
|
|
400
|
+
elif tmp_line[2].endswith(' \r'):
|
|
401
|
+
tmp_line[2] = tmp_line[2].replace(' \r', '')
|
|
402
|
+
updated_lines.append(json.dumps(tmp_line) + '\n')
|
|
403
|
+
else:
|
|
404
|
+
updated_lines.append(line)
|
|
405
|
+
with open(output_cast_path, 'w') as f:
|
|
406
|
+
f.writelines(updated_lines)
|
|
407
|
+
console.print('')
|
|
408
|
+
|
|
409
|
+
# Edit cast file to reduce long timeouts
|
|
410
|
+
with console.status('[bold gold3] Editing cast file to reduce long commands ...'):
|
|
411
|
+
Command.execute(
|
|
412
|
+
f'asciinema-edit quantize --range 1 {output_cast_path} --out {output_cast_path}.tmp',
|
|
413
|
+
cls_attributes=attrs,
|
|
414
|
+
raw=True,
|
|
415
|
+
)
|
|
416
|
+
if os.path.exists(f'{output_cast_path}.tmp'):
|
|
417
|
+
os.replace(f'{output_cast_path}.tmp', output_cast_path)
|
|
418
|
+
console.print(f'Edited {output_cast_path}', style='bold green')
|
|
419
|
+
|
|
420
|
+
# Convert to GIF
|
|
421
|
+
with console.status(f'[bold gold3]Converting to {output_gif_path} ...[/]'):
|
|
422
|
+
Command.execute(
|
|
423
|
+
f'agg {output_cast_path} {output_gif_path}',
|
|
424
|
+
cls_attributes=attrs,
|
|
425
|
+
)
|
|
426
|
+
console.print(Info(message=f'Generated {output_gif_path}'))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
@util.command('build')
|
|
430
|
+
@click.option('--version', type=str, help='Override version specified in pyproject.toml')
|
|
431
|
+
def build(version):
|
|
432
|
+
"""Build secator PyPI package."""
|
|
433
|
+
if not DEV_PACKAGE:
|
|
434
|
+
console.print(Error(message='You MUST use a development version of secator to make builds'))
|
|
435
|
+
sys.exit(1)
|
|
436
|
+
if not ADDONS_ENABLED['build']:
|
|
437
|
+
console.print(Error(message='Missing build addon: please run "secator install addons build"'))
|
|
438
|
+
sys.exit(1)
|
|
439
|
+
|
|
440
|
+
# Update version in pyproject.toml if --version is explicitely passed
|
|
441
|
+
if version:
|
|
442
|
+
pyproject_toml_path = Path.cwd() / 'pyproject.toml'
|
|
443
|
+
if not pyproject_toml_path.exists():
|
|
444
|
+
console.print(Error(message='You must be in the secator root directory to make builds with --version'))
|
|
445
|
+
sys.exit(1)
|
|
446
|
+
console.print(Info(message=f'Updating version in pyproject.toml to {version}'))
|
|
447
|
+
with open(pyproject_toml_path, "r") as file:
|
|
448
|
+
content = file.read()
|
|
449
|
+
updated_content = re.sub(r'^\s*version\s*=\s*".*?"', f'version = "{version}"', content, flags=re.MULTILINE)
|
|
450
|
+
with open(pyproject_toml_path, "w") as file:
|
|
451
|
+
file.write(updated_content)
|
|
452
|
+
|
|
453
|
+
with console.status('[bold gold3]Building PyPI package...[/]'):
|
|
454
|
+
ret = Command.execute(f'{sys.executable} -m hatch build', name='hatch build', cwd=ROOT_FOLDER)
|
|
455
|
+
sys.exit(ret.return_code)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
@util.command('publish')
|
|
459
|
+
def publish():
|
|
460
|
+
"""Publish secator PyPI package."""
|
|
461
|
+
if not DEV_PACKAGE:
|
|
462
|
+
console.print(Error(message='You MUST use a development version of secator to publish builds.'))
|
|
463
|
+
sys.exit(1)
|
|
464
|
+
if not ADDONS_ENABLED['build']:
|
|
465
|
+
console.print(Error(message='Missing build addon: please run "secator install addons build"'))
|
|
466
|
+
sys.exit(1)
|
|
467
|
+
os.environ['HATCH_INDEX_USER'] = '__token__'
|
|
468
|
+
hatch_token = os.environ.get('HATCH_INDEX_AUTH')
|
|
469
|
+
if not hatch_token:
|
|
470
|
+
console.print(Error(message='Missing PyPI auth token (HATCH_INDEX_AUTH env variable).'))
|
|
471
|
+
sys.exit(1)
|
|
472
|
+
with console.status('[bold gold3]Publishing PyPI package...[/]'):
|
|
473
|
+
ret = Command.execute(f'{sys.executable} -m hatch publish', name='hatch publish', cwd=ROOT_FOLDER)
|
|
474
|
+
sys.exit(ret.return_code)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
#--------#
|
|
478
|
+
# CONFIG #
|
|
479
|
+
#--------#
|
|
480
|
+
|
|
481
|
+
@cli.group(aliases=['c'])
|
|
482
|
+
def config():
|
|
483
|
+
"""View or edit config."""
|
|
484
|
+
pass
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@config.command('get')
|
|
488
|
+
@click.option('--user/--full', is_flag=True, help='Show config (user/full)')
|
|
489
|
+
@click.argument('key', required=False)
|
|
490
|
+
def config_get(user, key=None):
|
|
491
|
+
"""Get config value."""
|
|
492
|
+
if key is None:
|
|
493
|
+
partial = user and default_config != CONFIG
|
|
494
|
+
CONFIG.print(partial=partial)
|
|
495
|
+
return
|
|
496
|
+
CONFIG.get(key)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
@config.command('set')
|
|
500
|
+
@click.argument('key')
|
|
501
|
+
@click.argument('value')
|
|
502
|
+
def config_set(key, value):
|
|
503
|
+
"""Set config value."""
|
|
504
|
+
CONFIG.set(key, value)
|
|
505
|
+
config = CONFIG.validate()
|
|
506
|
+
if config:
|
|
507
|
+
CONFIG.get(key)
|
|
508
|
+
saved = CONFIG.save()
|
|
509
|
+
if not saved:
|
|
510
|
+
return
|
|
511
|
+
console.print(f'[bold green]:tada: Saved config to [/]{CONFIG._path}')
|
|
512
|
+
else:
|
|
513
|
+
console.print(Error(message='Invalid config, not saving it.'))
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
@config.command('unset')
|
|
517
|
+
@click.argument('key')
|
|
518
|
+
def config_unset(key):
|
|
519
|
+
"""Unset a config value."""
|
|
520
|
+
CONFIG.unset(key)
|
|
521
|
+
config = CONFIG.validate()
|
|
522
|
+
if config:
|
|
523
|
+
saved = CONFIG.save()
|
|
524
|
+
if not saved:
|
|
525
|
+
return
|
|
526
|
+
console.print(f'[bold green]:tada: Saved config to [/]{CONFIG._path}')
|
|
527
|
+
else:
|
|
528
|
+
console.print(Error(message='Invalid config, not saving it.'))
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
@config.command('edit')
|
|
532
|
+
@click.option('--resume', is_flag=True)
|
|
533
|
+
def config_edit(resume):
|
|
534
|
+
"""Edit config."""
|
|
535
|
+
tmp_config = CONFIG.dirs.data / 'config.yml.patch'
|
|
536
|
+
if not tmp_config.exists() or not resume:
|
|
537
|
+
shutil.copyfile(config_path, tmp_config)
|
|
538
|
+
click.edit(filename=tmp_config)
|
|
539
|
+
config = Config.parse(path=tmp_config)
|
|
540
|
+
if config:
|
|
541
|
+
config.save(config_path)
|
|
542
|
+
console.print(f'\n[bold green]:tada: Saved config to [/]{config_path}.')
|
|
543
|
+
tmp_config.unlink()
|
|
544
|
+
else:
|
|
545
|
+
console.print('\n[bold green]Hint:[/] Run "secator config edit --resume" to edit your patch and fix issues.')
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
@config.command('default')
|
|
549
|
+
@click.option('--save', type=str, help='Save default config to file.')
|
|
550
|
+
def config_default(save):
|
|
551
|
+
"""Get default config."""
|
|
552
|
+
default_config.print(partial=False)
|
|
553
|
+
if save:
|
|
554
|
+
default_config.save(target_path=Path(save), partial=False)
|
|
555
|
+
console.print(f'\n[bold green]:tada: Saved default config to [/]{save}.')
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
# TODO: implement reset method
|
|
559
|
+
# @_config.command('reset')
|
|
560
|
+
# @click.argument('key')
|
|
561
|
+
# def config_reset(key):
|
|
562
|
+
# """Reset config value to default."""
|
|
563
|
+
# success = CONFIG.set(key, None)
|
|
564
|
+
# if success:
|
|
565
|
+
# CONFIG.print()
|
|
566
|
+
# CONFIG.save()
|
|
567
|
+
# console.print(f'\n[bold green]:tada: Saved config to [/]{CONFIG._path}')
|
|
568
|
+
|
|
569
|
+
#-----------#
|
|
570
|
+
# WORKSPACE #
|
|
571
|
+
#-----------#
|
|
572
|
+
@cli.group(aliases=['ws', 'workspaces'])
|
|
573
|
+
def workspace():
|
|
574
|
+
"""Workspaces."""
|
|
575
|
+
pass
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
@workspace.command('list')
|
|
579
|
+
def workspace_list():
|
|
580
|
+
"""List workspaces."""
|
|
581
|
+
workspaces = {}
|
|
582
|
+
json_reports = []
|
|
583
|
+
for root, _, files in os.walk(CONFIG.dirs.reports):
|
|
584
|
+
for file in files:
|
|
585
|
+
if file.endswith('report.json'):
|
|
586
|
+
path = Path(root) / file
|
|
587
|
+
json_reports.append(path)
|
|
588
|
+
json_reports = sorted(json_reports, key=lambda x: x.stat().st_mtime, reverse=False)
|
|
589
|
+
for path in json_reports:
|
|
590
|
+
ws, runner_type, number = str(path).split('/')[-4:-1]
|
|
591
|
+
if ws not in workspaces:
|
|
592
|
+
workspaces[ws] = {'count': 0, 'path': '/'.join(str(path).split('/')[:-3])}
|
|
593
|
+
workspaces[ws]['count'] += 1
|
|
594
|
+
|
|
595
|
+
# Build table
|
|
596
|
+
table = Table()
|
|
597
|
+
table.add_column("Workspace name", style="bold gold3")
|
|
598
|
+
table.add_column("Run count", overflow='fold')
|
|
599
|
+
table.add_column("Path")
|
|
600
|
+
for workspace, config in workspaces.items():
|
|
601
|
+
table.add_row(workspace, str(config['count']), config['path'])
|
|
602
|
+
console.print(table)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
#----------#
|
|
606
|
+
# PROFILES #
|
|
607
|
+
#----------#
|
|
608
|
+
|
|
609
|
+
@cli.group(aliases=['p', 'profiles'])
|
|
610
|
+
@click.pass_context
|
|
611
|
+
def profile(ctx):
|
|
612
|
+
"""Profiles"""
|
|
613
|
+
pass
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
@profile.command('list')
|
|
617
|
+
def profile_list():
|
|
618
|
+
table = Table()
|
|
619
|
+
table.add_column("Profile name", style="bold gold3")
|
|
620
|
+
table.add_column("Description", overflow='fold')
|
|
621
|
+
table.add_column("Options", overflow='fold')
|
|
622
|
+
for profile in PROFILES:
|
|
623
|
+
opts_str = ', '.join(f'[yellow3]{k}[/]=[dim yellow3]{v}[/]' for k, v in profile.opts.items())
|
|
624
|
+
table.add_row(profile.name, profile.description or '', opts_str)
|
|
625
|
+
console.print(table)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
#-------#
|
|
629
|
+
# ALIAS #
|
|
630
|
+
#-------#
|
|
631
|
+
|
|
632
|
+
@cli.group(aliases=['a', 'aliases'])
|
|
633
|
+
def alias():
|
|
634
|
+
"""Aliases."""
|
|
635
|
+
pass
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
@alias.command('enable')
|
|
639
|
+
@click.pass_context
|
|
640
|
+
def enable_aliases(ctx):
|
|
641
|
+
"""Enable aliases."""
|
|
642
|
+
fpath = f'{CONFIG.dirs.data}/.aliases'
|
|
643
|
+
aliases = ctx.invoke(list_aliases, silent=True)
|
|
644
|
+
aliases_str = '\n'.join(aliases)
|
|
645
|
+
with open(fpath, 'w') as f:
|
|
646
|
+
f.write(aliases_str)
|
|
647
|
+
console.print('')
|
|
648
|
+
console.print(f':file_cabinet: Alias file written to {fpath}', style='bold green')
|
|
649
|
+
console.print('To load the aliases, run:')
|
|
650
|
+
md = f"""
|
|
651
|
+
```sh
|
|
652
|
+
source {fpath} # load the aliases in the current shell
|
|
653
|
+
echo "source {fpath} >> ~/.bashrc" # or add this line to your ~/.bashrc to load them automatically
|
|
654
|
+
```
|
|
655
|
+
"""
|
|
656
|
+
console.print(Markdown(md))
|
|
657
|
+
console.print()
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
@alias.command('disable')
|
|
661
|
+
@click.pass_context
|
|
662
|
+
def disable_aliases(ctx):
|
|
663
|
+
"""Disable aliases."""
|
|
664
|
+
fpath = f'{CONFIG.dirs.data}/.unalias'
|
|
665
|
+
aliases = ctx.invoke(list_aliases, silent=True)
|
|
666
|
+
aliases_str = ''
|
|
667
|
+
for alias in aliases:
|
|
668
|
+
alias_name = alias.split('=')[0]
|
|
669
|
+
if alias.strip().startswith('alias'):
|
|
670
|
+
alias_name = 'un' + alias_name
|
|
671
|
+
aliases_str += alias_name + '\n'
|
|
672
|
+
console.print(f':file_cabinet: Unalias file written to {fpath}', style='bold green')
|
|
673
|
+
console.print('To unload the aliases, run:')
|
|
674
|
+
with open(fpath, 'w') as f:
|
|
675
|
+
f.write(aliases_str)
|
|
676
|
+
md = f"""
|
|
677
|
+
```sh
|
|
678
|
+
source {fpath}
|
|
679
|
+
```
|
|
680
|
+
"""
|
|
681
|
+
console.print(Markdown(md))
|
|
682
|
+
console.print()
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
@alias.command('list')
|
|
686
|
+
@click.option('--silent', is_flag=True, default=False, help='No print')
|
|
687
|
+
def list_aliases(silent):
|
|
688
|
+
"""List aliases"""
|
|
689
|
+
aliases = []
|
|
690
|
+
aliases.append('\n# Global commands')
|
|
691
|
+
aliases.append('alias x="secator tasks"')
|
|
692
|
+
aliases.append('alias w="secator workflows"')
|
|
693
|
+
aliases.append('alias s="secator scans"')
|
|
694
|
+
aliases.append('alias wk="secator worker"')
|
|
695
|
+
aliases.append('alias ut="secator util"')
|
|
696
|
+
aliases.append('alias c="secator config"')
|
|
697
|
+
aliases.append('alias ws="secator workspaces"')
|
|
698
|
+
aliases.append('alias p="secator profiles"')
|
|
699
|
+
aliases.append('alias a="secator alias"')
|
|
700
|
+
aliases.append('alias r="secator reports"')
|
|
701
|
+
aliases.append('alias h="secator health"')
|
|
702
|
+
aliases.append('alias i="secator install"')
|
|
703
|
+
aliases.append('alias update="secator update"')
|
|
704
|
+
aliases.append('alias t="secator test"')
|
|
705
|
+
aliases.append('alias cs="secator cheatsheet"')
|
|
706
|
+
aliases.append('\n# Tasks')
|
|
707
|
+
for task in [t for t in discover_tasks()]:
|
|
708
|
+
alias_str = f'alias {task.__name__}="secator task {task.__name__}"'
|
|
709
|
+
if task.__external__:
|
|
710
|
+
alias_str += ' # external'
|
|
711
|
+
aliases.append(alias_str)
|
|
712
|
+
|
|
713
|
+
if silent:
|
|
714
|
+
return aliases
|
|
715
|
+
console.print('[bold gold3]:wrench: Aliases:[/]')
|
|
716
|
+
for alias in aliases:
|
|
717
|
+
alias_split = alias.split('=')
|
|
718
|
+
if len(alias_split) != 2:
|
|
719
|
+
console.print(f'[bold magenta]{alias}')
|
|
720
|
+
continue
|
|
721
|
+
alias_name, alias_cmd = alias_split[0].replace('alias ', ''), alias_split[1].replace('"', '')
|
|
722
|
+
if '# external' in alias_cmd:
|
|
723
|
+
alias_cmd = alias_cmd.replace('# external', ' [bold red]# external[/]')
|
|
724
|
+
console.print(f'[bold gold3]{alias_name:<15}[/] [dim]->[/] [bold green]{alias_cmd}[/]')
|
|
725
|
+
|
|
726
|
+
return aliases
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
#--------#
|
|
730
|
+
# REPORT #
|
|
731
|
+
#--------#
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
@cli.group(aliases=['r', 'reports'])
|
|
735
|
+
def report():
|
|
736
|
+
"""Reports."""
|
|
737
|
+
pass
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def process_query(query, fields=None):
|
|
741
|
+
if fields is None:
|
|
742
|
+
fields = []
|
|
743
|
+
otypes = [o.__name__.lower() for o in FINDING_TYPES]
|
|
744
|
+
extractors = []
|
|
745
|
+
|
|
746
|
+
# Process fields
|
|
747
|
+
fields_filter = {}
|
|
748
|
+
if fields:
|
|
749
|
+
for field in fields:
|
|
750
|
+
parts = field.split('.')
|
|
751
|
+
if len(parts) == 2:
|
|
752
|
+
_type, field = parts
|
|
753
|
+
else:
|
|
754
|
+
_type = parts[0]
|
|
755
|
+
field = None
|
|
756
|
+
if _type not in otypes:
|
|
757
|
+
console.print(Error(message='Invalid output type: ' + _type))
|
|
758
|
+
sys.exit(1)
|
|
759
|
+
fields_filter[_type] = field
|
|
760
|
+
|
|
761
|
+
# No query
|
|
762
|
+
if not query:
|
|
763
|
+
if fields:
|
|
764
|
+
extractors = [{'type': field_type, 'field': field, 'condition': 'True', 'op': 'or'} for field_type, field in fields_filter.items()] # noqa: E501
|
|
765
|
+
return extractors
|
|
766
|
+
|
|
767
|
+
# Get operator
|
|
768
|
+
operator = '||'
|
|
769
|
+
if '&&' in query and '||' in query:
|
|
770
|
+
console.print(Error(message='Cannot mix && and || in the same query'))
|
|
771
|
+
sys.exit(1)
|
|
772
|
+
elif '&&' in query:
|
|
773
|
+
operator = '&&'
|
|
774
|
+
elif '||' in query:
|
|
775
|
+
operator = '||'
|
|
776
|
+
|
|
777
|
+
# Process query
|
|
778
|
+
query = query.split(operator)
|
|
779
|
+
for part in query:
|
|
780
|
+
part = part.strip()
|
|
781
|
+
split_part = part.split('.')
|
|
782
|
+
_type = split_part[0]
|
|
783
|
+
if _type not in otypes:
|
|
784
|
+
console.print(Error(message='Invalid output type: ' + _type))
|
|
785
|
+
sys.exit(1)
|
|
786
|
+
if fields and _type not in fields_filter:
|
|
787
|
+
console.print(Warning(message='Type not allowed by --filter field: ' + _type + ' (allowed: ' + ', '.join(fields_filter.keys()) + '). Ignoring extractor.')) # noqa: E501
|
|
788
|
+
continue
|
|
789
|
+
extractor = {
|
|
790
|
+
'type': _type,
|
|
791
|
+
'condition': part or 'True',
|
|
792
|
+
'op': 'and' if operator == '&&' else 'or'
|
|
793
|
+
}
|
|
794
|
+
field = fields_filter.get(_type)
|
|
795
|
+
if field:
|
|
796
|
+
extractor['field'] = field
|
|
797
|
+
extractors.append(extractor)
|
|
798
|
+
return extractors
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
@report.command('show')
|
|
802
|
+
@click.argument('report_query', required=False)
|
|
803
|
+
@click.option('-o', '--output', type=str, default='console', help='Exporters')
|
|
804
|
+
@click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
|
|
805
|
+
@click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
|
|
806
|
+
@click.option('-f', '--format', "_format", type=str, default='', help=f'Format output, comma-separated of: <output_type> or <output_type>.<field>. [bold]Allowed output types[/]: {", ".join(FINDING_TYPES_LOWER)}') # noqa: E501
|
|
807
|
+
@click.option('-q', '--query', type=str, default=None, help='Query results using a Python expression')
|
|
808
|
+
@click.option('-w', '-ws', '--workspace', type=str, default=None, help='Filter by workspace name')
|
|
809
|
+
@click.option('-u', '--unified', is_flag=True, default=False, help='Show unified results (merge reports and de-duplicates results)') # noqa: E501
|
|
810
|
+
@click.pass_context
|
|
811
|
+
def report_show(ctx, report_query, output, runner_type, time_delta, _format, query, workspace, unified):
|
|
812
|
+
"""Show report results and filter on them."""
|
|
813
|
+
|
|
814
|
+
# Get report query from piped input
|
|
815
|
+
if ctx.obj['piped_input']:
|
|
816
|
+
report_query = ','.join(sys.stdin.read().splitlines())
|
|
817
|
+
unified = True
|
|
818
|
+
|
|
819
|
+
# Get extractors
|
|
820
|
+
extractors = process_query(query, fields=_format.split(',') if _format else [])
|
|
821
|
+
if extractors:
|
|
822
|
+
console.print(':wrench: [bold gold3]Showing query summary[/]')
|
|
823
|
+
op = extractors[0]['op']
|
|
824
|
+
console.print(f':carousel_horse: [bold blue]Op[/] [bold orange3]->[/] [bold green]{op.upper()}[/]')
|
|
825
|
+
for extractor in extractors:
|
|
826
|
+
console.print(f':zap: [bold blue]{extractor["type"].title()}[/] [bold orange3]->[/] [bold green]{extractor["condition"]}[/]', highlight=False) # noqa: E501
|
|
827
|
+
|
|
828
|
+
# Build runner instance
|
|
829
|
+
current = get_file_timestamp()
|
|
830
|
+
runner = DotMap({
|
|
831
|
+
"config": {
|
|
832
|
+
"name": f"consolidated_report_{current}"
|
|
833
|
+
},
|
|
834
|
+
"name": "runner",
|
|
835
|
+
"workspace_name": "_consolidated",
|
|
836
|
+
"reports_folder": Path.cwd(),
|
|
837
|
+
})
|
|
838
|
+
exporters = Runner.resolve_exporters(output)
|
|
839
|
+
|
|
840
|
+
# Build report queries from fuzzy input
|
|
841
|
+
paths = []
|
|
842
|
+
report_query = report_query.split(',') if report_query else []
|
|
843
|
+
load_all_reports = not report_query or any([not Path(p).exists() for p in report_query]) # fuzzy query, need to load all reports # noqa: E501
|
|
844
|
+
all_reports = []
|
|
845
|
+
if load_all_reports or workspace:
|
|
846
|
+
all_reports = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
|
|
847
|
+
if not report_query:
|
|
848
|
+
report_query = all_reports
|
|
849
|
+
for query in report_query:
|
|
850
|
+
query = str(query)
|
|
851
|
+
if not query.endswith('/'):
|
|
852
|
+
query += '/'
|
|
853
|
+
path = Path(query)
|
|
854
|
+
if not path.exists():
|
|
855
|
+
matches = []
|
|
856
|
+
for path in all_reports:
|
|
857
|
+
if query in str(path):
|
|
858
|
+
matches.append(path)
|
|
859
|
+
if not matches:
|
|
860
|
+
console.print(
|
|
861
|
+
f'[bold orange3]Query {query} did not return any matches. [/][bold green]Ignoring.[/]')
|
|
862
|
+
paths.extend(matches)
|
|
863
|
+
else:
|
|
864
|
+
paths.append(path)
|
|
865
|
+
paths = sort_files_by_date(paths)
|
|
866
|
+
|
|
867
|
+
# Load reports, extract results
|
|
868
|
+
all_results = []
|
|
869
|
+
for ix, path in enumerate(paths):
|
|
870
|
+
if unified:
|
|
871
|
+
if ix == 0:
|
|
872
|
+
console.print(f'\n:wrench: [bold gold3]Loading {len(paths)} reports ...[/]')
|
|
873
|
+
console.print(rf':file_cabinet: Loading {path} \[[bold yellow4]{ix + 1}[/]/[bold yellow4]{len(paths)}[/]] \[results={len(all_results)}]...') # noqa: E501
|
|
874
|
+
with open(path, 'r') as f:
|
|
875
|
+
try:
|
|
876
|
+
data = loads_dataclass(f.read())
|
|
877
|
+
info = get_info_from_report_path(path)
|
|
878
|
+
runner_type = info.get('type', 'unknowns')[:-1]
|
|
879
|
+
runner.results = flatten(list(data['results'].values()))
|
|
880
|
+
if unified:
|
|
881
|
+
all_results.extend(runner.results)
|
|
882
|
+
continue
|
|
883
|
+
report = Report(runner, title=f"Consolidated report - {current}", exporters=exporters)
|
|
884
|
+
report.build(extractors=extractors if not unified else [], dedupe=unified)
|
|
885
|
+
file_date = get_file_date(path)
|
|
886
|
+
runner_name = data['info']['name']
|
|
887
|
+
if not report.is_empty():
|
|
888
|
+
console.print(
|
|
889
|
+
f'\n{path} ([bold blue]{runner_name}[/] [dim]{runner_type}[/]) ([dim]{file_date}[/]):')
|
|
890
|
+
if report.is_empty():
|
|
891
|
+
if len(paths) == 1:
|
|
892
|
+
console.print(Warning(message='No results in report.'))
|
|
893
|
+
continue
|
|
894
|
+
report.send()
|
|
895
|
+
except json.decoder.JSONDecodeError as e:
|
|
896
|
+
console.print(Error(message=f'Could not load {path}: {str(e)}'))
|
|
897
|
+
|
|
898
|
+
if unified:
|
|
899
|
+
console.print(f'\n:wrench: [bold gold3]Building report by crunching {len(all_results)} results ...[/]', end='')
|
|
900
|
+
console.print(' (:coffee: [dim]this can take a while ...[/])')
|
|
901
|
+
runner.results = all_results
|
|
902
|
+
report = Report(runner, title=f"Consolidated report - {current}", exporters=exporters)
|
|
903
|
+
report.build(extractors=extractors, dedupe=True)
|
|
904
|
+
report.send()
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
@report.command('list')
|
|
908
|
+
@click.option('-ws', '-w', '--workspace', type=str)
|
|
909
|
+
@click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
|
|
910
|
+
@click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
|
|
911
|
+
@click.pass_context
|
|
912
|
+
def report_list(ctx, workspace, runner_type, time_delta):
|
|
913
|
+
"""List all secator reports."""
|
|
914
|
+
paths = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
|
|
915
|
+
paths = sorted(paths, key=lambda x: x.stat().st_mtime, reverse=False)
|
|
916
|
+
|
|
917
|
+
# Build table
|
|
918
|
+
table = Table()
|
|
919
|
+
table.add_column("Workspace", style="bold gold3")
|
|
920
|
+
table.add_column("Path", overflow='fold')
|
|
921
|
+
table.add_column("Name")
|
|
922
|
+
table.add_column("Id")
|
|
923
|
+
table.add_column("Date")
|
|
924
|
+
table.add_column("Status", style="green")
|
|
925
|
+
|
|
926
|
+
# Print paths if piped
|
|
927
|
+
if ctx.obj['piped_output']:
|
|
928
|
+
if not paths:
|
|
929
|
+
console.print(Error(message='No reports found.'))
|
|
930
|
+
return
|
|
931
|
+
for path in paths:
|
|
932
|
+
print(path)
|
|
933
|
+
return
|
|
934
|
+
|
|
935
|
+
# Load each report
|
|
936
|
+
for path in paths:
|
|
937
|
+
try:
|
|
938
|
+
info = get_info_from_report_path(path)
|
|
939
|
+
with open(path, 'r') as f:
|
|
940
|
+
content = json.loads(f.read())
|
|
941
|
+
data = {
|
|
942
|
+
'workspace': info['workspace'],
|
|
943
|
+
'name': f"[bold blue]{content['info']['name']}[/]",
|
|
944
|
+
'status': content['info'].get('status', ''),
|
|
945
|
+
'id': info['type'] + '/' + info['id'],
|
|
946
|
+
'date': get_file_date(path), # Assuming get_file_date returns a readable date
|
|
947
|
+
}
|
|
948
|
+
status_color = STATE_COLORS[data['status']] if data['status'] in STATE_COLORS else 'white'
|
|
949
|
+
|
|
950
|
+
# Update table
|
|
951
|
+
table.add_row(
|
|
952
|
+
data['workspace'],
|
|
953
|
+
str(path),
|
|
954
|
+
data['name'],
|
|
955
|
+
data['id'],
|
|
956
|
+
data['date'],
|
|
957
|
+
f"[{status_color}]{data['status']}[/]"
|
|
958
|
+
)
|
|
959
|
+
except json.JSONDecodeError as e:
|
|
960
|
+
console.print(Error(message=f'Could not load {path}: {str(e)}'))
|
|
961
|
+
|
|
962
|
+
if len(paths) > 0:
|
|
963
|
+
console.print(table)
|
|
964
|
+
console.print(Info(message=f'Found {len(paths)} reports.'))
|
|
965
|
+
else:
|
|
966
|
+
console.print(Error(message='No reports found.'))
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
@report.command('export')
|
|
970
|
+
@click.argument('json_path', type=str)
|
|
971
|
+
@click.option('--output-folder', '-of', type=str)
|
|
972
|
+
@click.option('--output', '-o', type=str, required=True)
|
|
973
|
+
def report_export(json_path, output_folder, output):
|
|
974
|
+
with open(json_path, 'r') as f:
|
|
975
|
+
data = loads_dataclass(f.read())
|
|
976
|
+
|
|
977
|
+
split = json_path.split('/')
|
|
978
|
+
workspace_name = '/'.join(split[:-4]) if len(split) > 4 else '_default'
|
|
979
|
+
runner_instance = DotMap({
|
|
980
|
+
"config": {
|
|
981
|
+
"name": data['info']['name']
|
|
982
|
+
},
|
|
983
|
+
"workspace_name": workspace_name,
|
|
984
|
+
"reports_folder": output_folder or Path.cwd(),
|
|
985
|
+
"data": data,
|
|
986
|
+
"results": flatten(list(data['results'].values()))
|
|
987
|
+
})
|
|
988
|
+
exporters = Runner.resolve_exporters(output)
|
|
989
|
+
report = Report(runner_instance, title=data['info']['title'], exporters=exporters)
|
|
990
|
+
report.data = data
|
|
991
|
+
report.send()
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
#--------#
|
|
995
|
+
# DEPLOY #
|
|
996
|
+
#--------#
|
|
997
|
+
|
|
998
|
+
# TODO: work on this
|
|
999
|
+
# @cli.group(aliases=['d'])
|
|
1000
|
+
# def deploy():
|
|
1001
|
+
# """Deploy secator."""
|
|
1002
|
+
# pass
|
|
1003
|
+
|
|
1004
|
+
# @deploy.command()
|
|
1005
|
+
# def docker_compose():
|
|
1006
|
+
# """Deploy secator on docker-compose."""
|
|
1007
|
+
# pass
|
|
1008
|
+
|
|
1009
|
+
# @deploy.command()
|
|
1010
|
+
# @click.option('-t', '--target', type=str, default='minikube', help='Deployment target amongst minikube, gke')
|
|
1011
|
+
# def k8s():
|
|
1012
|
+
# """Deploy secator on Kubernetes."""
|
|
1013
|
+
# pass
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
#--------#
|
|
1017
|
+
# HEALTH #
|
|
1018
|
+
#--------#
|
|
1019
|
+
|
|
1020
|
+
@cli.command(name='health', aliases=['h'])
|
|
1021
|
+
@click.option('--json', '-json', 'json_', is_flag=True, default=False, help='JSON lines output')
|
|
1022
|
+
@click.option('--debug', '-debug', is_flag=True, default=False, help='Debug health output')
|
|
1023
|
+
@click.option('--strict', '-strict', is_flag=True, default=False, help='Fail if missing tools')
|
|
1024
|
+
@click.option('--bleeding', '-bleeding', is_flag=True, default=False, help='Check bleeding edge version of tools')
|
|
1025
|
+
def health(json_, debug, strict, bleeding):
|
|
1026
|
+
"""Get health status."""
|
|
1027
|
+
tools = discover_tasks()
|
|
1028
|
+
upgrade_cmd = ''
|
|
1029
|
+
results = []
|
|
1030
|
+
messages = []
|
|
1031
|
+
|
|
1032
|
+
# Abort if offline mode is enabled
|
|
1033
|
+
if CONFIG.offline_mode:
|
|
1034
|
+
console.print(Error(message='Cannot run this command in offline mode.'))
|
|
1035
|
+
sys.exit(1)
|
|
1036
|
+
|
|
1037
|
+
# Check secator
|
|
1038
|
+
console.print(':wrench: [bold gold3]Checking secator ...[/]') if not json_ else None
|
|
1039
|
+
info = get_version_info('secator', '-version', 'freelabz/secator')
|
|
1040
|
+
info['_type'] = 'core'
|
|
1041
|
+
if info['outdated']:
|
|
1042
|
+
messages.append(f'secator is outdated (latest:{info["latest_version"]}).')
|
|
1043
|
+
results.append(info)
|
|
1044
|
+
table = get_health_table()
|
|
1045
|
+
contextmanager = Live(table, console=console) if not json_ else nullcontext()
|
|
1046
|
+
with contextmanager:
|
|
1047
|
+
row = fmt_health_table_row(info)
|
|
1048
|
+
table.add_row(*row)
|
|
1049
|
+
|
|
1050
|
+
# Check addons
|
|
1051
|
+
console.print('\n:wrench: [bold gold3]Checking addons ...[/]') if not json_ else None
|
|
1052
|
+
table = get_health_table()
|
|
1053
|
+
contextmanager = Live(table, console=console) if not json_ else nullcontext()
|
|
1054
|
+
with contextmanager:
|
|
1055
|
+
for addon, installed in ADDONS_ENABLED.items():
|
|
1056
|
+
info = {
|
|
1057
|
+
'name': addon,
|
|
1058
|
+
'version': None,
|
|
1059
|
+
'status': 'ok' if installed else 'missing_ok',
|
|
1060
|
+
'latest_version': None,
|
|
1061
|
+
'installed': installed,
|
|
1062
|
+
'location': None
|
|
1063
|
+
}
|
|
1064
|
+
info['_type'] = 'addon'
|
|
1065
|
+
results.append(info)
|
|
1066
|
+
row = fmt_health_table_row(info, 'addons')
|
|
1067
|
+
table.add_row(*row)
|
|
1068
|
+
if json_:
|
|
1069
|
+
print(json.dumps(info))
|
|
1070
|
+
|
|
1071
|
+
# Check languages
|
|
1072
|
+
console.print('\n:wrench: [bold gold3]Checking languages ...[/]') if not json_ else None
|
|
1073
|
+
version_cmds = {'go': 'version', 'python3': '--version', 'ruby': '--version'}
|
|
1074
|
+
table = get_health_table()
|
|
1075
|
+
contextmanager = Live(table, console=console) if not json_ else nullcontext()
|
|
1076
|
+
with contextmanager:
|
|
1077
|
+
for lang, version_flag in version_cmds.items():
|
|
1078
|
+
info = get_version_info(lang, version_flag)
|
|
1079
|
+
row = fmt_health_table_row(info, 'langs')
|
|
1080
|
+
table.add_row(*row)
|
|
1081
|
+
info['_type'] = 'lang'
|
|
1082
|
+
results.append(info)
|
|
1083
|
+
if json_:
|
|
1084
|
+
print(json.dumps(info))
|
|
1085
|
+
|
|
1086
|
+
# Check tools
|
|
1087
|
+
console.print('\n:wrench: [bold gold3]Checking tools ...[/]') if not json_ else None
|
|
1088
|
+
table = get_health_table()
|
|
1089
|
+
error = False
|
|
1090
|
+
contextmanager = Live(table, console=console) if not json_ else nullcontext()
|
|
1091
|
+
upgrade_cmd = 'secator install tools'
|
|
1092
|
+
with contextmanager:
|
|
1093
|
+
for tool in tools:
|
|
1094
|
+
info = get_version_info(
|
|
1095
|
+
tool.cmd.split(' ')[0],
|
|
1096
|
+
tool.version_flag or f'{tool.opt_prefix}version',
|
|
1097
|
+
tool.github_handle,
|
|
1098
|
+
tool.install_github_version_prefix,
|
|
1099
|
+
tool.install_cmd,
|
|
1100
|
+
tool.install_version,
|
|
1101
|
+
bleeding=bleeding
|
|
1102
|
+
)
|
|
1103
|
+
info['_name'] = tool.__name__
|
|
1104
|
+
info['_type'] = 'tool'
|
|
1105
|
+
row = fmt_health_table_row(info, 'tools')
|
|
1106
|
+
table.add_row(*row)
|
|
1107
|
+
if not info['installed']:
|
|
1108
|
+
messages.append(f'{tool.__name__} is not installed.')
|
|
1109
|
+
info['next_version'] = tool.install_version
|
|
1110
|
+
error = True
|
|
1111
|
+
elif info['outdated']:
|
|
1112
|
+
msg = 'latest' if bleeding else 'supported'
|
|
1113
|
+
message = (
|
|
1114
|
+
f'{tool.__name__} is outdated (current:{info["version"]}, {msg}:{info["latest_version"]}).'
|
|
1115
|
+
)
|
|
1116
|
+
messages.append(message)
|
|
1117
|
+
info['upgrade'] = True
|
|
1118
|
+
info['next_version'] = info['latest_version']
|
|
1119
|
+
|
|
1120
|
+
elif info['bleeding']:
|
|
1121
|
+
msg = 'latest' if bleeding else 'supported'
|
|
1122
|
+
message = (
|
|
1123
|
+
f'{tool.__name__} is bleeding edge (current:{info["version"]}, {msg}:{info["latest_version"]}).'
|
|
1124
|
+
)
|
|
1125
|
+
messages.append(message)
|
|
1126
|
+
info['downgrade'] = True
|
|
1127
|
+
info['next_version'] = info['latest_version']
|
|
1128
|
+
results.append(info)
|
|
1129
|
+
if json_:
|
|
1130
|
+
print(json.dumps(info))
|
|
1131
|
+
console.print('') if not json_ else None
|
|
1132
|
+
|
|
1133
|
+
if not json_ and messages:
|
|
1134
|
+
console.print('\n[bold red]Issues found:[/]')
|
|
1135
|
+
for message in messages:
|
|
1136
|
+
console.print(Warning(message=message))
|
|
1137
|
+
|
|
1138
|
+
# Strict mode
|
|
1139
|
+
if strict:
|
|
1140
|
+
if error:
|
|
1141
|
+
sys.exit(1)
|
|
1142
|
+
console.print(Info(message='Strict healthcheck passed !')) if not json_ else None
|
|
1143
|
+
|
|
1144
|
+
# Build upgrade command
|
|
1145
|
+
cmds = []
|
|
1146
|
+
tool_cmd = ''
|
|
1147
|
+
for info in results:
|
|
1148
|
+
if info['_type'] == 'core' and info['outdated']:
|
|
1149
|
+
cmds.append('secator update')
|
|
1150
|
+
elif info['_type'] == 'tool' and info.get('next_version'):
|
|
1151
|
+
tool_cmd += f',{info["_name"]}=={info["next_version"]}'
|
|
1152
|
+
|
|
1153
|
+
if tool_cmd:
|
|
1154
|
+
tool_cmd = f'secator install tools {tool_cmd.lstrip(",")}'
|
|
1155
|
+
cmds.append(tool_cmd)
|
|
1156
|
+
upgrade_cmd = ' && '.join(cmds)
|
|
1157
|
+
console.print('') if not json_ else None
|
|
1158
|
+
if upgrade_cmd:
|
|
1159
|
+
console.print(Info(message='Run the following to upgrade secator and tools:')) if not json_ else None
|
|
1160
|
+
if json_:
|
|
1161
|
+
print(json.dumps({'upgrade_cmd': upgrade_cmd}))
|
|
1162
|
+
else:
|
|
1163
|
+
print(upgrade_cmd)
|
|
1164
|
+
else:
|
|
1165
|
+
console.print(Info(message='Everything is up to date !')) if not json_ else None
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
#------------#
|
|
1169
|
+
# CHEATSHEET #
|
|
1170
|
+
#------------#
|
|
1171
|
+
|
|
1172
|
+
@cli.command(name='cheatsheet', aliases=['cs'])
|
|
1173
|
+
def cheatsheet():
|
|
1174
|
+
"""Display a cheatsheet of secator commands."""
|
|
1175
|
+
from rich.panel import Panel
|
|
1176
|
+
from rich import box
|
|
1177
|
+
kwargs = {
|
|
1178
|
+
'box': box.ROUNDED,
|
|
1179
|
+
'title_align': 'left',
|
|
1180
|
+
# 'style': 'bold blue3',
|
|
1181
|
+
'border_style': 'green',
|
|
1182
|
+
'padding': (0, 1, 0, 1),
|
|
1183
|
+
'highlight': False,
|
|
1184
|
+
'expand': False,
|
|
1185
|
+
}
|
|
1186
|
+
title_style = 'bold green'
|
|
1187
|
+
|
|
1188
|
+
panel1 = Panel(r"""
|
|
1189
|
+
[dim bold]:left_arrow_curving_right: Secator basic commands to get you started.[/]
|
|
1190
|
+
|
|
1191
|
+
secator [orange3]x[/] [dim]# list tasks[/]
|
|
1192
|
+
secator [orange3]w[/] [dim]# list workflows[/]
|
|
1193
|
+
secator [orange3]s[/] [dim]# list scans[/]
|
|
1194
|
+
secator [orange3]p[/] [dim]# manage profiles[/]
|
|
1195
|
+
secator [orange3]r[/] [dim]# manage reports[/]
|
|
1196
|
+
secator [orange3]c[/] [dim]# manage configuration[/]
|
|
1197
|
+
secator [orange3]ws[/] [dim]# manage workspaces[/]
|
|
1198
|
+
secator [orange3]update[/] [dim]# update secator[/]
|
|
1199
|
+
|
|
1200
|
+
[dim]# Running tasks, workflows or scans...[/]
|
|
1201
|
+
secator \[[orange3]x[/]|[orange3]w[/]|[orange3]s[/]] [NAME] [OPTIONS] [INPUTS] [dim]# run a task ([bold orange3]x[/]), workflow ([bold orange3]w[/]) or scan ([bold orange3]s[/])[/]
|
|
1202
|
+
secator [orange3]x[/] [red]httpx[/] example.com [dim]# run an [bold red]httpx[/] task ([bold orange3]x[/] is for e[bold orange3]x[/]ecute)[/]
|
|
1203
|
+
secator [orange3]w[/] [red]url_crawl[/] https://example.com [dim]# run a [bold red]url crawl[/] workflow ([bold orange3]w[/])[/]
|
|
1204
|
+
secator [orange3]s[/] [red]host[/] example.com [dim]# run a [bold red]host[/] scan ([bold orange3]s[/])[/]
|
|
1205
|
+
|
|
1206
|
+
[dim]# Show information on tasks, workflows or scans ...[/]
|
|
1207
|
+
secator s host [blue]-dry[/] [dim]# show dry run (show exact commands that will be run)[/]
|
|
1208
|
+
secator s host [blue]-tree[/] [dim]# show config tree (workflows and scans only)[/]
|
|
1209
|
+
secator s host [blue]-yaml[/] [dim]# show config yaml (workflows and scans only)[/]
|
|
1210
|
+
|
|
1211
|
+
[dim]# Organize your results (workspace, database)[/]
|
|
1212
|
+
secator s host [blue]-ws[/] [bright_magenta]prod[/] example.com [dim]# save results to 'prod' workspace[/]
|
|
1213
|
+
secator s host [blue]-driver[/] [bright_magenta]mongodb[/] example.com [dim]# save results to mongodb database[/]
|
|
1214
|
+
|
|
1215
|
+
[dim]# Input types are flexible ...[/]
|
|
1216
|
+
secator s host [cyan]example.com[/] [dim]# single input[/]
|
|
1217
|
+
secator s host [cyan]host1,host2,host3[/] [dim]# multiple inputs (comma-separated)[/]
|
|
1218
|
+
secator s host [cyan]hosts.txt[/] [dim]# multiple inputs (txt file)[/]
|
|
1219
|
+
[cyan]cat hosts.txt | [/]secator s host [dim]# piped inputs[/]
|
|
1220
|
+
|
|
1221
|
+
[dim]# Options are mutualized ...[/]
|
|
1222
|
+
secator s host [blue]-rl[/] [bright_magenta]10[/] [blue]-delay[/] [bright_magenta]1[/] [blue]-proxy[/] [bright_magenta]http://127.0.0.1:9090[/] example.com [dim]# set rate limit, delay and proxy for all subtasks[/]
|
|
1223
|
+
secator s host [blue]-pf[/] [bright_magenta]aggressive[/] example.com [dim]# ... or use a profile to automatically set options[/]
|
|
1224
|
+
|
|
1225
|
+
[dim]:point_right: and [bold]YES[/], the above options and inputs work with any scan ([bold orange3]s[/]), workflow ([bold orange3]w[/]), and task ([bold orange3]x[/]), not just the host scan shown here![/]
|
|
1226
|
+
""", # noqa: E501
|
|
1227
|
+
title=f":shield: [{title_style}]Some basics[/]", **kwargs)
|
|
1228
|
+
|
|
1229
|
+
panel2 = Panel(r"""
|
|
1230
|
+
[dim bold]:left_arrow_curving_right: Secator aliases are useful to stop typing [bold cyan]secator <something>[/] and focus on what you want to run. Aliases are a must to increase your productivity.[/]
|
|
1231
|
+
|
|
1232
|
+
[bold]To enable aliases:[/]
|
|
1233
|
+
|
|
1234
|
+
secator alias enable [dim]# enable aliases[/]
|
|
1235
|
+
source ~/.secator/.aliases [dim]# load aliases in current shell[/]
|
|
1236
|
+
|
|
1237
|
+
[dim]# Now you can use aliases...[/]
|
|
1238
|
+
a list [dim]# list all aliases[/]
|
|
1239
|
+
httpx [dim]# aliased httpx ![/]
|
|
1240
|
+
nmap -sV -p 443 --script vulners example.com [dim]# aliased nmap ![/]
|
|
1241
|
+
w subdomain_recon [dim]# aliased subdomain_recon ![/]
|
|
1242
|
+
s domain [dim]# aliased domain scan ![/]
|
|
1243
|
+
cat hosts.txt | subfinder | naabu | httpx | w url_crawl [dim]# pipes to chain tasks ![/]
|
|
1244
|
+
""", # noqa: E501
|
|
1245
|
+
title=f":shorts: [{title_style}]Aliases[/]", **kwargs)
|
|
1246
|
+
|
|
1247
|
+
panel3 = Panel(r"""
|
|
1248
|
+
[dim bold]:left_arrow_curving_right: Secator configuration is stored in a YAML file located at [bold cyan]~/.secator/config.yaml[/]. You can edit it manually or use the following commands to get/set values.[/]
|
|
1249
|
+
|
|
1250
|
+
c get [dim]# get config value[/]
|
|
1251
|
+
c get --user [dim]# get user config value[/]
|
|
1252
|
+
c edit [dim]# edit user config in editor[/]
|
|
1253
|
+
c set profiles.defaults aggressive [dim]# set 'aggressive' profile as default[/]
|
|
1254
|
+
c set drivers.defaults mongodb [dim]# set mongodb as default driver[/]
|
|
1255
|
+
c set wordlists.defaults.http https://example.com/wordlist.txt [dim]# set default wordlist for http fuzzing[/]
|
|
1256
|
+
""", # noqa: E501
|
|
1257
|
+
title=f":gear: [{title_style}]Configuration[/]", **kwargs)
|
|
1258
|
+
|
|
1259
|
+
panel4 = Panel(r"""
|
|
1260
|
+
[dim bold]:left_arrow_curving_right: By default, tasks are run sequentially. You can use a worker to run tasks in parallel and massively speed up your scans.[/]
|
|
1261
|
+
|
|
1262
|
+
wk [dim]# or [bold cyan]secator worker[/] if you don't use aliases ...[/]
|
|
1263
|
+
httpx testphp.vulnweb.com [dim]# <-- will run in worker and output results normally[/]
|
|
1264
|
+
|
|
1265
|
+
[dim]:question: Want to use a remote worker ?[/]
|
|
1266
|
+
[dim]:point_right: Spawn a Celery worker on your remote server, a Redis instance and set the following config values to connect to it, both in the worker and locally:[/]
|
|
1267
|
+
c set celery.result_backend redis://<remote_ip>:6379/0 [dim]# set redis backend[/]
|
|
1268
|
+
c set celery.broker_url redis://<remote_ip>:6379/0 [dim]# set redis broker[/]
|
|
1269
|
+
[dim]:point_right: Then, run your tasks, workflows or scans like you would locally ![/]
|
|
1270
|
+
""", # noqa: E501
|
|
1271
|
+
title=f":zap: [{title_style}]Too slow ? Use a worker[/]", **kwargs)
|
|
1272
|
+
|
|
1273
|
+
panel5 = Panel(r"""
|
|
1274
|
+
[dim bold]:left_arrow_curving_right: Reports are stored in the [bold cyan]~/.secator/reports[/] directory. You can list, show, filter and export reports using the following commands.[/]
|
|
1275
|
+
|
|
1276
|
+
[dim]# List and filter reports...[/]
|
|
1277
|
+
r list [dim]# list all reports[/]
|
|
1278
|
+
r list [blue]-ws[/] [bright_magenta]prod[/] [dim]# list reports from the workspace 'prod'[/]
|
|
1279
|
+
r list [blue]-d[/] [bright_magenta]1h[/] [dim]# list reports from the last hour[/]
|
|
1280
|
+
|
|
1281
|
+
[dim]# Show and filter results...[/]
|
|
1282
|
+
r show [blue]-q[/] [bright_magenta]"url.status_code not in ['401', '403']"[/] [blue]-o[/] [bright_magenta]txt[/] [dim]# show urls with status 401 or 403, save to txt file[/]
|
|
1283
|
+
r show tasks/10,tasks/11 [blue]-q[/] [bright_magenta]"tag.match and 'signup.php' in tag.match"[/] [blue]--unified[/] [blue]-o[/] [bright_magenta]json[/] [dim]# show tags with targets matching 'signup.php' from tasks 10 and 11[/]
|
|
1284
|
+
""", # noqa: E501
|
|
1285
|
+
title=f":file_cabinet: [{title_style}]Digging into reports[/]", **kwargs)
|
|
1286
|
+
|
|
1287
|
+
panel6 = Panel(r"""
|
|
1288
|
+
[dim bold]:left_arrow_curving_right: Commands to manage secator installation.[/]
|
|
1289
|
+
|
|
1290
|
+
update [dim]# update secator to the latest version[/]
|
|
1291
|
+
|
|
1292
|
+
[dim]:point_right: Tools are automatically installed when first running a task, workflow or scan, but you can still install them manually.[/]
|
|
1293
|
+
i tools httpx [dim]# install tool 'httpx'[/]
|
|
1294
|
+
i tools [dim]# install all tools[/]
|
|
1295
|
+
|
|
1296
|
+
[dim]:point_right: Addons are optional dependencies required to enable certain features.[/]
|
|
1297
|
+
i addon redis [dim]# install addon 'redis'[/]
|
|
1298
|
+
i addon gcs [dim]# install addon 'gcs'[/]
|
|
1299
|
+
i addon worker [dim]# install addon 'worker'[/]
|
|
1300
|
+
i addon gdrive [dim]# install addon 'gdrive'[/]
|
|
1301
|
+
i addon mongodb [dim]# install addon 'mongodb'[/]
|
|
1302
|
+
""", # noqa: E501
|
|
1303
|
+
title=f":wrench: [{title_style}]Updates[/]", **kwargs)
|
|
1304
|
+
|
|
1305
|
+
panel7 = Panel(r"""
|
|
1306
|
+
[dim bold]:left_arrow_curving_right: Some useful scans and workflows we use day-to-day for recon.[/]
|
|
1307
|
+
|
|
1308
|
+
[orange3]:warning: Don't forget to add [bold blue]-dry[/] or [bold blue]-tree[/] before running your scans to see what will be done ![/]
|
|
1309
|
+
|
|
1310
|
+
[bold orange3]:trophy: Domain recon + Subdomain recon + Port scanning + URL crawl + URL vulns (XSS, SQLi, RCE, ...)[/]
|
|
1311
|
+
s domain <DOMAIN> [dim]# light[/]
|
|
1312
|
+
s domain <DOMAIN> -pf all_ports [dim]# light + full port scan[/]
|
|
1313
|
+
s domain <DOMAIN> -pf full [dim]# all features (full port scan, nuclei, pattern hunting, headless crawling, screenshots, etc.)[/]
|
|
1314
|
+
s domain <DOMAIN> -pf passive [dim]# passive (0 requests to targets)[/]
|
|
1315
|
+
|
|
1316
|
+
[bold orange3]:trophy: Subdomain recon[/]
|
|
1317
|
+
w subdomain_recon <DOMAIN> [dim]# standard[/]
|
|
1318
|
+
w subdomain_recon <DOMAIN> -brute-dns -brute-http [dim]# bruteforce subdomains (DNS queries + HTTP Host header fuzzing)[/]
|
|
1319
|
+
w subdomain_recon <DOMAIN> -pf passive [dim]# passive (0 requests to targets)[/]
|
|
1320
|
+
|
|
1321
|
+
[bold orange3]:trophy: URL fuzzing[/]
|
|
1322
|
+
w url_fuzz <URL> [dim]# standard fuzzing (ffuf)[/]
|
|
1323
|
+
w url_fuzz <URL> -hs [dim]# hunt secrets in HTTP responses (trufflehog)[/]
|
|
1324
|
+
w url_fuzz <URL> -mc 200,301 -fs 204 [dim]# match 200, 301, and filter size equal to 204 bytes[/]
|
|
1325
|
+
w url_fuzz -fuzzers ffuf,dirsearch <URL> -w <URL> [dim]# choose fuzzers, use remote wordlist[/]
|
|
1326
|
+
|
|
1327
|
+
[bold orange3]:trophy: Vuln and secret scan:[/]
|
|
1328
|
+
w code_scan <PATH> [dim]# on a local path or git repo[/]
|
|
1329
|
+
w code_scan https://github.com/freelabz/secator [dim]# on a github repo[/]
|
|
1330
|
+
w code_scan https://github.com/freelabz [dim]# on a github org (all repos)[/]
|
|
1331
|
+
|
|
1332
|
+
[bold orange3]:trophy: Hunt user accounts[/]
|
|
1333
|
+
w user_hunt elonmusk [dim]# by username[/]
|
|
1334
|
+
w user_hunt elonmusk@tesla.com [dim]# by email[/]
|
|
1335
|
+
|
|
1336
|
+
[bold orange3]:trophy: Custom pipeline to find HTTP servers and fuzz alive ones[/]
|
|
1337
|
+
subfinder vulnweb.com | naabu | httpx | ffuf -mc 200,301 -recursion
|
|
1338
|
+
""", # noqa: E501
|
|
1339
|
+
title=f":trophy: [{title_style}]Quick wins[/]", **kwargs)
|
|
1340
|
+
|
|
1341
|
+
console.print(panel1)
|
|
1342
|
+
console.print(panel2)
|
|
1343
|
+
console.print(panel3)
|
|
1344
|
+
console.print(panel4)
|
|
1345
|
+
console.print(panel5)
|
|
1346
|
+
console.print(panel6)
|
|
1347
|
+
console.print(panel7)
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
#---------#
|
|
1351
|
+
# INSTALL #
|
|
1352
|
+
#---------#
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def run_install(title=None, cmd=None, packages=None, next_steps=None):
|
|
1356
|
+
if CONFIG.offline_mode:
|
|
1357
|
+
console.print(Error(message='Cannot run this command in offline mode.'))
|
|
1358
|
+
sys.exit(1)
|
|
1359
|
+
# with console.status(f'[bold yellow] Installing {title}...'):
|
|
1360
|
+
if cmd:
|
|
1361
|
+
from secator.installer import SourceInstaller
|
|
1362
|
+
status = SourceInstaller.install(cmd)
|
|
1363
|
+
elif packages:
|
|
1364
|
+
from secator.installer import PackageInstaller
|
|
1365
|
+
status = PackageInstaller.install(packages)
|
|
1366
|
+
return_code = 1
|
|
1367
|
+
if status.is_ok():
|
|
1368
|
+
return_code = 0
|
|
1369
|
+
if next_steps:
|
|
1370
|
+
console.print('[bold gold3]:wrench: Next steps:[/]')
|
|
1371
|
+
for ix, step in enumerate(next_steps):
|
|
1372
|
+
console.print(f' :keycap_{ix}: {step}')
|
|
1373
|
+
sys.exit(return_code)
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
@cli.group(aliases=['i'])
|
|
1377
|
+
def install():
|
|
1378
|
+
"""Install langs, tools and addons."""
|
|
1379
|
+
pass
|
|
1380
|
+
|
|
1381
|
+
|
|
1382
|
+
@install.group()
|
|
1383
|
+
def addons():
|
|
1384
|
+
"Install addons."
|
|
1385
|
+
pass
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
@addons.command('worker')
|
|
1389
|
+
def install_worker():
|
|
1390
|
+
"Install Celery worker addon."
|
|
1391
|
+
run_install(
|
|
1392
|
+
cmd=f'{sys.executable} -m pip install secator[worker]',
|
|
1393
|
+
title='Celery worker addon',
|
|
1394
|
+
next_steps=[
|
|
1395
|
+
'Run [bold green4]secator worker[/] to run a Celery worker using the file system as a backend and broker.',
|
|
1396
|
+
'Run [bold green4]secator x httpx testphp.vulnweb.com[/] to admire your task running in a worker.',
|
|
1397
|
+
r'[dim]\[optional][/dim] Run [bold green4]secator install addons redis[/] to setup Redis backend / broker.'
|
|
1398
|
+
]
|
|
1399
|
+
)
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
@addons.command('gdrive')
|
|
1403
|
+
def install_gdrive():
|
|
1404
|
+
"Install Google Drive addon."
|
|
1405
|
+
run_install(
|
|
1406
|
+
cmd=f'{sys.executable} -m pip install secator[google]',
|
|
1407
|
+
title='Google Drive addon',
|
|
1408
|
+
next_steps=[
|
|
1409
|
+
'Run [bold green4]secator config set addons.gdrive.credentials_path <VALUE>[/].',
|
|
1410
|
+
'Run [bold green4]secator config set addons.gdrive.drive_parent_folder_id <VALUE>[/].',
|
|
1411
|
+
'Run [bold green4]secator x httpx testphp.vulnweb.com -o gdrive[/] to send reports to Google Drive.'
|
|
1412
|
+
]
|
|
1413
|
+
)
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
@addons.command('gcs')
|
|
1417
|
+
def install_gcs():
|
|
1418
|
+
"Install Google Cloud Storage addon."
|
|
1419
|
+
run_install(
|
|
1420
|
+
cmd=f'{sys.executable} -m pip install secator[gcs]',
|
|
1421
|
+
title='Google Cloud Storage addon',
|
|
1422
|
+
next_steps=[
|
|
1423
|
+
'Run [bold green4]secator config set addons.gcs.bucket_name <VALUE>[/].',
|
|
1424
|
+
'Run [bold green4]secator config set addons.gcs.credentials_path <VALUE>[/]. [dim](optional if using default credentials)[/]', # noqa: E501
|
|
1425
|
+
]
|
|
1426
|
+
)
|
|
1427
|
+
|
|
1428
|
+
|
|
1429
|
+
@addons.command('mongodb')
|
|
1430
|
+
def install_mongodb():
|
|
1431
|
+
"Install MongoDB addon."
|
|
1432
|
+
run_install(
|
|
1433
|
+
cmd=f'{sys.executable} -m pip install secator[mongodb]',
|
|
1434
|
+
title='MongoDB addon',
|
|
1435
|
+
next_steps=[
|
|
1436
|
+
r'[dim]\[optional][/] Run [bold green4]docker run --name mongo -p 27017:27017 -d mongo:latest[/] to run a local MongoDB instance.', # noqa: E501
|
|
1437
|
+
'Run [bold green4]secator config set addons.mongodb.url mongodb://<URL>[/].',
|
|
1438
|
+
'Run [bold green4]secator x httpx testphp.vulnweb.com -driver mongodb[/] to save results to MongoDB.'
|
|
1439
|
+
]
|
|
1440
|
+
)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
@addons.command('redis')
|
|
1444
|
+
def install_redis():
|
|
1445
|
+
"Install Redis addon."
|
|
1446
|
+
run_install(
|
|
1447
|
+
cmd=f'{sys.executable} -m pip install secator[redis]',
|
|
1448
|
+
title='Redis addon',
|
|
1449
|
+
next_steps=[
|
|
1450
|
+
r'[dim]\[optional][/] Run [bold green4]docker run --name redis -p 6379:6379 -d redis[/] to run a local Redis instance.', # noqa: E501
|
|
1451
|
+
'Run [bold green4]secator config set celery.broker_url redis://<URL>[/]',
|
|
1452
|
+
'Run [bold green4]secator config set celery.result_backend redis://<URL>[/]',
|
|
1453
|
+
'Run [bold green4]secator worker[/] to run a worker.',
|
|
1454
|
+
'Run [bold green4]secator x httpx testphp.vulnweb.com[/] to run a test task.'
|
|
1455
|
+
]
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
@addons.command('dev')
|
|
1460
|
+
def install_dev():
|
|
1461
|
+
"Install dev addon."
|
|
1462
|
+
run_install(
|
|
1463
|
+
cmd=f'{sys.executable} -m pip install secator[dev]',
|
|
1464
|
+
title='dev addon',
|
|
1465
|
+
next_steps=[
|
|
1466
|
+
'Run [bold green4]secator test lint[/] to run lint tests.',
|
|
1467
|
+
'Run [bold green4]secator test unit[/] to run unit tests.',
|
|
1468
|
+
'Run [bold green4]secator test integration[/] to run integration tests.',
|
|
1469
|
+
]
|
|
1470
|
+
)
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
@addons.command('trace')
|
|
1474
|
+
def install_trace():
|
|
1475
|
+
"Install trace addon."
|
|
1476
|
+
run_install(
|
|
1477
|
+
cmd=f'{sys.executable} -m pip install secator[trace]',
|
|
1478
|
+
title='trace addon',
|
|
1479
|
+
next_steps=[
|
|
1480
|
+
]
|
|
1481
|
+
)
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
@addons.command('build')
|
|
1485
|
+
def install_build():
|
|
1486
|
+
"Install build addon."
|
|
1487
|
+
run_install(
|
|
1488
|
+
cmd=f'{sys.executable} -m pip install secator[build]',
|
|
1489
|
+
title='build addon',
|
|
1490
|
+
next_steps=[
|
|
1491
|
+
'Run [bold green4]secator u build pypi[/] to build the PyPI package.',
|
|
1492
|
+
'Run [bold green4]secator u publish pypi[/] to publish the PyPI package.',
|
|
1493
|
+
'Run [bold green4]secator u build docker[/] to build the Docker image.',
|
|
1494
|
+
'Run [bold green4]secator u publish docker[/] to publish the Docker image.',
|
|
1495
|
+
]
|
|
1496
|
+
)
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
@install.group()
|
|
1500
|
+
def langs():
|
|
1501
|
+
"Install languages."
|
|
1502
|
+
pass
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
@langs.command('go')
|
|
1506
|
+
def install_go():
|
|
1507
|
+
"""Install Go."""
|
|
1508
|
+
run_install(
|
|
1509
|
+
cmd='wget -O - https://raw.githubusercontent.com/freelabz/secator/main/scripts/install_go.sh | sudo sh',
|
|
1510
|
+
title='Go',
|
|
1511
|
+
next_steps=[
|
|
1512
|
+
'Add ~/go/bin to your $PATH'
|
|
1513
|
+
]
|
|
1514
|
+
)
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
@langs.command('ruby')
|
|
1518
|
+
def install_ruby():
|
|
1519
|
+
"""Install Ruby."""
|
|
1520
|
+
run_install(
|
|
1521
|
+
packages={
|
|
1522
|
+
'apt': ['ruby-full', 'rubygems'],
|
|
1523
|
+
'apk': ['ruby', 'ruby-dev'],
|
|
1524
|
+
'pacman': ['ruby', 'ruby-dev'],
|
|
1525
|
+
'brew': ['ruby']
|
|
1526
|
+
},
|
|
1527
|
+
title='Ruby'
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
@install.command('tools')
|
|
1532
|
+
@click.argument('cmds', required=False)
|
|
1533
|
+
@click.option('--cleanup', is_flag=True, default=False, help='Clean up tools after installation.')
|
|
1534
|
+
@click.option('--fail-fast', is_flag=True, default=False, help='Fail fast if any tool fails to install.')
|
|
1535
|
+
def install_tools(cmds, cleanup, fail_fast):
|
|
1536
|
+
"""Install supported tools."""
|
|
1537
|
+
if CONFIG.offline_mode:
|
|
1538
|
+
console.print(Error(message='Cannot run this command in offline mode.'))
|
|
1539
|
+
sys.exit(1)
|
|
1540
|
+
tools = []
|
|
1541
|
+
if cmds is not None:
|
|
1542
|
+
cmds = cmds.split(',')
|
|
1543
|
+
for cmd in cmds:
|
|
1544
|
+
if '==' in cmd:
|
|
1545
|
+
cmd, version = tuple(cmd.split('=='))
|
|
1546
|
+
else:
|
|
1547
|
+
cmd, version = cmd, None
|
|
1548
|
+
cls = next((cls for cls in discover_tasks() if cls.__name__ == cmd), None)
|
|
1549
|
+
if cls:
|
|
1550
|
+
if version:
|
|
1551
|
+
if cls.install_version and cls.install_version.startswith('v') and not version.startswith('v'):
|
|
1552
|
+
version = f'v{version}'
|
|
1553
|
+
cls.install_version = version
|
|
1554
|
+
tools.append(cls)
|
|
1555
|
+
else:
|
|
1556
|
+
console.print(Warning(message=f'Tool {cmd} is not supported or inexistent.'))
|
|
1557
|
+
else:
|
|
1558
|
+
tools = discover_tasks()
|
|
1559
|
+
tools.sort(key=lambda x: x.__name__)
|
|
1560
|
+
return_code = 0
|
|
1561
|
+
if not tools:
|
|
1562
|
+
console.print(Error(message='No tools found for installing.'))
|
|
1563
|
+
return
|
|
1564
|
+
for ix, cls in enumerate(tools):
|
|
1565
|
+
# with console.status(f'[bold yellow][{ix + 1}/{len(tools)}] Installing {cls.__name__} ...'):
|
|
1566
|
+
status = ToolInstaller.install(cls)
|
|
1567
|
+
if not status.is_ok():
|
|
1568
|
+
return_code = 1
|
|
1569
|
+
if fail_fast:
|
|
1570
|
+
sys.exit(return_code)
|
|
1571
|
+
console.print()
|
|
1572
|
+
if cleanup:
|
|
1573
|
+
distro = get_distro_config()
|
|
1574
|
+
cleanup_cmds = [
|
|
1575
|
+
'go clean -cache',
|
|
1576
|
+
'go clean -modcache',
|
|
1577
|
+
'pip cache purge',
|
|
1578
|
+
'gem cleanup --user-install',
|
|
1579
|
+
'gem clean --user-install',
|
|
1580
|
+
]
|
|
1581
|
+
if distro.pm_finalizer:
|
|
1582
|
+
cleanup_cmds.append(f'sudo {distro.pm_finalizer}')
|
|
1583
|
+
cmd = ' && '.join(cleanup_cmds)
|
|
1584
|
+
Command.execute(cmd, cls_attributes={'shell': True}, quiet=False)
|
|
1585
|
+
sys.exit(return_code)
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
#--------#
|
|
1589
|
+
# UPDATE #
|
|
1590
|
+
#--------#
|
|
1591
|
+
|
|
1592
|
+
@cli.command('update')
|
|
1593
|
+
@click.option('--all', '-a', is_flag=True, help='Update all secator dependencies (addons, tools, ...)')
|
|
1594
|
+
def update(all):
|
|
1595
|
+
"""Update to latest version."""
|
|
1596
|
+
if CONFIG.offline_mode:
|
|
1597
|
+
console.print(Error(message='Cannot run this command in offline mode.'))
|
|
1598
|
+
sys.exit(1)
|
|
1599
|
+
|
|
1600
|
+
# Check current and latest version
|
|
1601
|
+
info = get_version_info('secator', '-version', 'freelabz/secator', version=VERSION)
|
|
1602
|
+
latest_version = info['latest_version']
|
|
1603
|
+
do_update = True
|
|
1604
|
+
|
|
1605
|
+
# Skip update if latest
|
|
1606
|
+
if info['status'] == 'latest':
|
|
1607
|
+
console.print(Info(message=f'secator is already at the newest version {latest_version} !'))
|
|
1608
|
+
do_update = False
|
|
1609
|
+
|
|
1610
|
+
# Fail if unknown latest
|
|
1611
|
+
if not latest_version:
|
|
1612
|
+
console.print(Error(message='Could not fetch latest secator version.'))
|
|
1613
|
+
sys.exit(1)
|
|
1614
|
+
|
|
1615
|
+
# Update secator
|
|
1616
|
+
if do_update:
|
|
1617
|
+
console.print(f'[bold gold3]:wrench: Updating secator from {VERSION} to {latest_version} ...[/]')
|
|
1618
|
+
if 'pipx' in sys.executable:
|
|
1619
|
+
ret = Command.execute(f'pipx install secator=={latest_version} --force')
|
|
1620
|
+
else:
|
|
1621
|
+
ret = Command.execute(f'pip install secator=={latest_version}')
|
|
1622
|
+
if not ret.return_code == 0:
|
|
1623
|
+
sys.exit(1)
|
|
1624
|
+
|
|
1625
|
+
# Update tools
|
|
1626
|
+
if all:
|
|
1627
|
+
return_code = 0
|
|
1628
|
+
for cls in discover_tasks():
|
|
1629
|
+
base_cmd = getattr(cls, 'cmd', None)
|
|
1630
|
+
if not base_cmd:
|
|
1631
|
+
continue
|
|
1632
|
+
cmd = base_cmd.split(' ')[0]
|
|
1633
|
+
version_flag = cls.get_version_flag()
|
|
1634
|
+
info = get_version_info(cmd, version_flag, cls.github_handle, cls.install_github_version_prefix)
|
|
1635
|
+
if not info['installed'] or info['outdated'] or not info['latest_version']:
|
|
1636
|
+
# with console.status(f'[bold yellow]Installing {cls.__name__} ...'):
|
|
1637
|
+
status = ToolInstaller.install(cls)
|
|
1638
|
+
if not status.is_ok():
|
|
1639
|
+
return_code = 1
|
|
1640
|
+
sys.exit(return_code)
|
|
1641
|
+
|
|
1642
|
+
|
|
1643
|
+
#------#
|
|
1644
|
+
# TEST #
|
|
1645
|
+
#------#
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
@cli.group(cls=OrderedGroup)
|
|
1649
|
+
def test():
|
|
1650
|
+
"""[dim]Run tests (dev build only)."""
|
|
1651
|
+
if not DEV_PACKAGE:
|
|
1652
|
+
console.print(Error(message='You MUST use a development version of secator to run tests.'))
|
|
1653
|
+
sys.exit(1)
|
|
1654
|
+
if not ADDONS_ENABLED['dev']:
|
|
1655
|
+
console.print(Error(message='Missing dev addon: please run "secator install addons dev"'))
|
|
1656
|
+
sys.exit(1)
|
|
1657
|
+
pass
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
def run_test(cmd, name=None, exit=True, verbose=False, use_os_system=False):
|
|
1661
|
+
"""Run a test and return the result.
|
|
1662
|
+
|
|
1663
|
+
Args:
|
|
1664
|
+
cmd (str): Command to run.
|
|
1665
|
+
name (str, optional): Name of the test.
|
|
1666
|
+
exit (bool, optional): Exit after running the test with the return code.
|
|
1667
|
+
verbose (bool, optional): Print verbose output.
|
|
1668
|
+
use_os_system (bool, optional): Use os.system to run the command.
|
|
1669
|
+
|
|
1670
|
+
Returns:
|
|
1671
|
+
Return code of the test.
|
|
1672
|
+
"""
|
|
1673
|
+
cmd_name = name + ' tests' if name else 'tests'
|
|
1674
|
+
if use_os_system:
|
|
1675
|
+
console.print(f'[bold red]{cmd}[/]')
|
|
1676
|
+
if not verbose:
|
|
1677
|
+
cmd += ' >/dev/null 2>&1'
|
|
1678
|
+
ret = os.system(cmd)
|
|
1679
|
+
if exit:
|
|
1680
|
+
sys.exit(os.waitstatus_to_exitcode(ret))
|
|
1681
|
+
return ret
|
|
1682
|
+
else:
|
|
1683
|
+
result = Command.execute(cmd, name=cmd_name, cwd=ROOT_FOLDER, quiet=not verbose)
|
|
1684
|
+
if name:
|
|
1685
|
+
if result.return_code == 0:
|
|
1686
|
+
console.print(f':tada: {name.capitalize()} tests passed !', style='bold green')
|
|
1687
|
+
else:
|
|
1688
|
+
console.print(f':x: {name.capitalize()} tests failed !', style='bold red')
|
|
1689
|
+
if exit:
|
|
1690
|
+
sys.exit(result.return_code)
|
|
1691
|
+
return result.return_code
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
@test.command()
|
|
1695
|
+
@click.option('--linter', '-l', type=click.Choice(['flake8', 'ruff', 'isort', 'pylint']), default='flake8', help='Linter to use') # noqa: E501
|
|
1696
|
+
def lint(linter):
|
|
1697
|
+
"""Run lint tests."""
|
|
1698
|
+
opts = ''
|
|
1699
|
+
if linter == 'pylint':
|
|
1700
|
+
opts = '--indent-string "\t" --max-line-length 160 --disable=R,C,W'
|
|
1701
|
+
elif linter == 'ruff':
|
|
1702
|
+
opts = ' check'
|
|
1703
|
+
cmd = f'{sys.executable} -m {linter} {opts} secator/'
|
|
1704
|
+
run_test(cmd, 'lint', verbose=True, use_os_system=True)
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
@test.command()
|
|
1708
|
+
@click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
|
|
1709
|
+
@click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
|
|
1710
|
+
@click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
|
|
1711
|
+
@click.option('--test', '-t', type=str, help='Secator test to run')
|
|
1712
|
+
@click.option('--no-coverage', is_flag=True, help='Disable coverage')
|
|
1713
|
+
def unit(tasks, workflows, scans, test, no_coverage):
|
|
1714
|
+
"""Run unit tests."""
|
|
1715
|
+
os.environ['TEST_TASKS'] = tasks or ''
|
|
1716
|
+
os.environ['TEST_WORKFLOWS'] = workflows or ''
|
|
1717
|
+
os.environ['TEST_SCANS'] = scans or ''
|
|
1718
|
+
os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
|
|
1719
|
+
os.environ['SECATOR_OFFLINE_MODE'] = "1"
|
|
1720
|
+
os.environ['SECATOR_HTTP_STORE_RESPONSES'] = '0'
|
|
1721
|
+
os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
|
|
1722
|
+
|
|
1723
|
+
if not test:
|
|
1724
|
+
if tasks:
|
|
1725
|
+
test = 'test_tasks'
|
|
1726
|
+
elif workflows:
|
|
1727
|
+
test = 'test_workflows'
|
|
1728
|
+
elif scans:
|
|
1729
|
+
test = 'test_scans'
|
|
1730
|
+
|
|
1731
|
+
import shutil
|
|
1732
|
+
shutil.rmtree('/tmp/.secator', ignore_errors=True)
|
|
1733
|
+
if not no_coverage:
|
|
1734
|
+
cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.unit -m pytest -s -vv tests/unit --durations=5' # noqa: E501
|
|
1735
|
+
else:
|
|
1736
|
+
cmd = f'{sys.executable} -m pytest -s -vv tests/unit --durations=5'
|
|
1737
|
+
if test:
|
|
1738
|
+
test_str = ' or '.join(test.split(','))
|
|
1739
|
+
cmd += f' -k "{test_str}"'
|
|
1740
|
+
run_test(cmd, 'unit', verbose=True, use_os_system=True)
|
|
1741
|
+
|
|
1742
|
+
|
|
1743
|
+
@test.command()
|
|
1744
|
+
@click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
|
|
1745
|
+
@click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
|
|
1746
|
+
@click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
|
|
1747
|
+
@click.option('--test', '-t', type=str, help='Secator test to run')
|
|
1748
|
+
@click.option('--no-cleanup', '-nc', is_flag=True, help='Do not perform cleanup (keep lab running, faster for relaunching tests)') # noqa: E501
|
|
1749
|
+
def integration(tasks, workflows, scans, test, no_cleanup):
|
|
1750
|
+
"""Run integration tests."""
|
|
1751
|
+
os.environ['TEST_TASKS'] = tasks or ''
|
|
1752
|
+
os.environ['TEST_WORKFLOWS'] = workflows or ''
|
|
1753
|
+
os.environ['TEST_SCANS'] = scans or ''
|
|
1754
|
+
os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
|
|
1755
|
+
os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
|
|
1756
|
+
os.environ['TEST_NO_CLEANUP'] = '1' if no_cleanup else '0'
|
|
1757
|
+
|
|
1758
|
+
if not test:
|
|
1759
|
+
if tasks:
|
|
1760
|
+
test = 'test_tasks'
|
|
1761
|
+
elif workflows:
|
|
1762
|
+
test = 'test_workflows'
|
|
1763
|
+
elif scans:
|
|
1764
|
+
test = 'test_scans'
|
|
1765
|
+
|
|
1766
|
+
import shutil
|
|
1767
|
+
shutil.rmtree('/tmp/.secator', ignore_errors=True)
|
|
1768
|
+
|
|
1769
|
+
cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.integration -m pytest -s -vv tests/integration --durations=5' # noqa: E501
|
|
1770
|
+
if test:
|
|
1771
|
+
test_str = ' or '.join(test.split(','))
|
|
1772
|
+
cmd += f' -k "{test_str}"'
|
|
1773
|
+
run_test(cmd, 'integration', verbose=True, use_os_system=True)
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
@test.command()
|
|
1777
|
+
@click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
|
|
1778
|
+
@click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
|
|
1779
|
+
@click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
|
|
1780
|
+
@click.option('--test', '-t', type=str, help='Secator test to run')
|
|
1781
|
+
def template(tasks, workflows, scans, test):
|
|
1782
|
+
"""Run integration tests."""
|
|
1783
|
+
os.environ['TEST_TASKS'] = tasks or ''
|
|
1784
|
+
os.environ['TEST_WORKFLOWS'] = workflows or ''
|
|
1785
|
+
os.environ['TEST_SCANS'] = scans or ''
|
|
1786
|
+
os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
|
|
1787
|
+
os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
|
|
1788
|
+
|
|
1789
|
+
if not test:
|
|
1790
|
+
if tasks:
|
|
1791
|
+
test = 'test_tasks'
|
|
1792
|
+
elif workflows:
|
|
1793
|
+
test = 'test_workflows'
|
|
1794
|
+
elif scans:
|
|
1795
|
+
test = 'test_scans'
|
|
1796
|
+
|
|
1797
|
+
import shutil
|
|
1798
|
+
shutil.rmtree('/tmp/.secator', ignore_errors=True)
|
|
1799
|
+
|
|
1800
|
+
cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.templates -m pytest -s -vv tests/template --durations=5' # noqa: E501
|
|
1801
|
+
if test:
|
|
1802
|
+
test_str = ' or '.join(test.split(','))
|
|
1803
|
+
cmd += f' -k "{test_str}"'
|
|
1804
|
+
run_test(cmd, 'template', verbose=True)
|
|
1805
|
+
|
|
1806
|
+
|
|
1807
|
+
@test.command()
|
|
1808
|
+
@click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
|
|
1809
|
+
@click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
|
|
1810
|
+
@click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
|
|
1811
|
+
@click.option('--test', '-t', type=str, help='Secator test to run')
|
|
1812
|
+
def performance(tasks, workflows, scans, test):
|
|
1813
|
+
"""Run integration tests."""
|
|
1814
|
+
os.environ['TEST_TASKS'] = tasks or ''
|
|
1815
|
+
os.environ['TEST_WORKFLOWS'] = workflows or ''
|
|
1816
|
+
os.environ['TEST_SCANS'] = scans or ''
|
|
1817
|
+
os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
|
|
1818
|
+
os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
|
|
1819
|
+
|
|
1820
|
+
# import shutil
|
|
1821
|
+
# shutil.rmtree('/tmp/.secator', ignore_errors=True)
|
|
1822
|
+
|
|
1823
|
+
cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.performance -m pytest -s -v tests/performance' # noqa: E501
|
|
1824
|
+
if test:
|
|
1825
|
+
test_str = ' or '.join(test.split(','))
|
|
1826
|
+
cmd += f' -k "{test_str}"'
|
|
1827
|
+
run_test(cmd, 'performance', verbose=True, use_os_system=True)
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
@test.command()
|
|
1831
|
+
@click.argument('name', type=str)
|
|
1832
|
+
@click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
|
|
1833
|
+
@click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
|
|
1834
|
+
@click.option('--system-exit', '-e', is_flag=True, default=True, help='Exit with system exit code')
|
|
1835
|
+
def task(name, verbose, check, system_exit):
|
|
1836
|
+
"""Test a single task for semantics errors, and run unit + integration tests."""
|
|
1837
|
+
console.print(f'[bold gold3]:wrench: Testing task {name} ...[/]')
|
|
1838
|
+
task = [task for task in discover_tasks() if task.__name__ == name.strip()]
|
|
1839
|
+
warnings = []
|
|
1840
|
+
errors = []
|
|
1841
|
+
exit_code = 0
|
|
1842
|
+
|
|
1843
|
+
# Check if task is correctly registered
|
|
1844
|
+
check_test(
|
|
1845
|
+
len(task) == 1,
|
|
1846
|
+
'Check task is registered',
|
|
1847
|
+
'Task is not registered. Please check your task name.',
|
|
1848
|
+
errors
|
|
1849
|
+
)
|
|
1850
|
+
if errors:
|
|
1851
|
+
if system_exit:
|
|
1852
|
+
sys.exit(1)
|
|
1853
|
+
else:
|
|
1854
|
+
return False
|
|
1855
|
+
|
|
1856
|
+
task = task[0]
|
|
1857
|
+
task_name = task.__name__
|
|
1858
|
+
|
|
1859
|
+
# Check task command is set
|
|
1860
|
+
check_test(
|
|
1861
|
+
task.cmd,
|
|
1862
|
+
'Check task command is set (cls.cmd)',
|
|
1863
|
+
'Task has no cmd attribute.',
|
|
1864
|
+
errors
|
|
1865
|
+
)
|
|
1866
|
+
if errors:
|
|
1867
|
+
if system_exit:
|
|
1868
|
+
sys.exit(1)
|
|
1869
|
+
else:
|
|
1870
|
+
return False
|
|
1871
|
+
|
|
1872
|
+
# Run install
|
|
1873
|
+
cmd = f'secator install tools {task_name}'
|
|
1874
|
+
ret_code = Command.execute(cmd, name='install', quiet=not verbose, cwd=ROOT_FOLDER)
|
|
1875
|
+
version_info = task.get_version_info()
|
|
1876
|
+
if verbose:
|
|
1877
|
+
console.print(f'Version info:\n{version_info}')
|
|
1878
|
+
status = version_info['status']
|
|
1879
|
+
check_test(
|
|
1880
|
+
version_info['installed'],
|
|
1881
|
+
'Check task is installed',
|
|
1882
|
+
'Failed to install command. Fix your installation command.',
|
|
1883
|
+
errors
|
|
1884
|
+
)
|
|
1885
|
+
check_test(
|
|
1886
|
+
any(cmd for cmd in [task.install_pre, task.install_cmd, task.github_handle]),
|
|
1887
|
+
'Check task installation command is defined',
|
|
1888
|
+
'Task has no installation command. Please define one or more of the following class attributes: `install_pre`, `install_cmd`, `install_post`, `github_handle`.', # noqa: E501
|
|
1889
|
+
errors
|
|
1890
|
+
)
|
|
1891
|
+
check_test(
|
|
1892
|
+
version_info['version'],
|
|
1893
|
+
'Check task version can be fetched',
|
|
1894
|
+
'Failed to detect current version. Consider updating your `version_flag` class attribute.',
|
|
1895
|
+
warnings,
|
|
1896
|
+
warn=True
|
|
1897
|
+
)
|
|
1898
|
+
check_test(
|
|
1899
|
+
status != 'latest unknown',
|
|
1900
|
+
'Check latest version',
|
|
1901
|
+
'Failed to detect latest version.',
|
|
1902
|
+
warnings,
|
|
1903
|
+
warn=True
|
|
1904
|
+
)
|
|
1905
|
+
check_test(
|
|
1906
|
+
not version_info['outdated'],
|
|
1907
|
+
'Check task version is up to date',
|
|
1908
|
+
f'Task is not up to date (current version: {version_info["version"]}, latest: {version_info["latest_version"]}). Consider updating your `install_version` class attribute.', # noqa: E501
|
|
1909
|
+
warnings,
|
|
1910
|
+
warn=True
|
|
1911
|
+
)
|
|
1912
|
+
|
|
1913
|
+
# Run task-specific tests
|
|
1914
|
+
check_test(
|
|
1915
|
+
task.__doc__,
|
|
1916
|
+
'Check task description is set (cls.__doc__)',
|
|
1917
|
+
'Task has no description (class docstring).',
|
|
1918
|
+
errors
|
|
1919
|
+
)
|
|
1920
|
+
check_test(
|
|
1921
|
+
task.input_types,
|
|
1922
|
+
'Check task input type is set (cls.input_type)',
|
|
1923
|
+
'Task has no input_type attribute.',
|
|
1924
|
+
warnings,
|
|
1925
|
+
warn=True
|
|
1926
|
+
)
|
|
1927
|
+
check_test(
|
|
1928
|
+
task.output_types,
|
|
1929
|
+
'Check task output types is set (cls.output_types)',
|
|
1930
|
+
'Task has no output_types attribute. Consider setting some so that secator can load your task outputs.',
|
|
1931
|
+
warnings,
|
|
1932
|
+
warn=True
|
|
1933
|
+
)
|
|
1934
|
+
check_test(
|
|
1935
|
+
task.install_version,
|
|
1936
|
+
'Check task install_version is set (cls.install_version)',
|
|
1937
|
+
'Task has no install_version attribute. Consider setting it to pin the tool version and ensure it does not break in the future.', # noqa: E501
|
|
1938
|
+
warnings,
|
|
1939
|
+
warn=True
|
|
1940
|
+
)
|
|
1941
|
+
|
|
1942
|
+
if not check:
|
|
1943
|
+
|
|
1944
|
+
# Run unit tests
|
|
1945
|
+
cmd = f'secator test unit --tasks {name}'
|
|
1946
|
+
ret_code = run_test(cmd, exit=False, verbose=verbose)
|
|
1947
|
+
check_test(
|
|
1948
|
+
ret_code == 0,
|
|
1949
|
+
'Check unit tests pass',
|
|
1950
|
+
'Unit tests failed.',
|
|
1951
|
+
errors
|
|
1952
|
+
)
|
|
1953
|
+
|
|
1954
|
+
# Run integration tests
|
|
1955
|
+
cmd = f'secator test integration --tasks {name}'
|
|
1956
|
+
ret_code = run_test(cmd, exit=False, verbose=verbose)
|
|
1957
|
+
check_test(
|
|
1958
|
+
ret_code == 0,
|
|
1959
|
+
'Check integration tests pass',
|
|
1960
|
+
'Integration tests failed.',
|
|
1961
|
+
errors
|
|
1962
|
+
)
|
|
1963
|
+
|
|
1964
|
+
# Exit with exit code
|
|
1965
|
+
exit_code = 1 if len(errors) > 0 else 0
|
|
1966
|
+
if exit_code == 0:
|
|
1967
|
+
console.print(f':tada: Task {name} tests passed !', style='bold green')
|
|
1968
|
+
else:
|
|
1969
|
+
console.print('\n[bold gold3]Errors:[/]')
|
|
1970
|
+
for error in errors:
|
|
1971
|
+
console.print(error)
|
|
1972
|
+
console.print(Error(message=f'Task {name} tests failed. Please fix the issues above.'))
|
|
1973
|
+
|
|
1974
|
+
if warnings:
|
|
1975
|
+
console.print('\n[bold gold3]Warnings:[/]')
|
|
1976
|
+
for warning in warnings:
|
|
1977
|
+
console.print(warning)
|
|
1978
|
+
|
|
1979
|
+
console.print("\n")
|
|
1980
|
+
if system_exit:
|
|
1981
|
+
sys.exit(exit_code)
|
|
1982
|
+
else:
|
|
1983
|
+
return True if exit_code == 0 else False
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
@test.command()
|
|
1987
|
+
@click.pass_context
|
|
1988
|
+
@click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
|
|
1989
|
+
@click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
|
|
1990
|
+
def tasks(ctx, check, verbose):
|
|
1991
|
+
"""Test all tasks for semantics errors, and run unit + integration tests."""
|
|
1992
|
+
results = []
|
|
1993
|
+
for cls in discover_tasks():
|
|
1994
|
+
success = ctx.invoke(task, name=cls.__name__, verbose=verbose, check=check, system_exit=False)
|
|
1995
|
+
results.append(success)
|
|
1996
|
+
|
|
1997
|
+
if any(not success for success in results):
|
|
1998
|
+
console.print(Error(message='Tasks checks failed. Please check the output for more details.'))
|
|
1999
|
+
sys.exit(1)
|
|
2000
|
+
console.print(Info(message='All tasks checks passed.'))
|
|
2001
|
+
sys.exit(0)
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
def check_test(condition, message, fail_message, results=[], warn=False):
|
|
2005
|
+
console.print(f'[bold magenta]:zap: {message} ...[/]', end='')
|
|
2006
|
+
if not condition:
|
|
2007
|
+
if not warn:
|
|
2008
|
+
error = Error(message=fail_message)
|
|
2009
|
+
console.print(' [bold red]FAILED[/]', style='dim')
|
|
2010
|
+
results.append(error)
|
|
2011
|
+
else:
|
|
2012
|
+
warning = Warning(message=fail_message)
|
|
2013
|
+
console.print(' [bold yellow]WARNING[/]', style='dim')
|
|
2014
|
+
results.append(warning)
|
|
2015
|
+
else:
|
|
2016
|
+
console.print(' [bold green]OK[/]', style='dim')
|
|
2017
|
+
return True
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
@test.command()
|
|
2021
|
+
@click.option('--unit-only', '-u', is_flag=True, default=False, help='Only generate coverage for unit tests')
|
|
2022
|
+
@click.option('--integration-only', '-i', is_flag=True, default=False, help='Only generate coverage for integration tests') # noqa: E501
|
|
2023
|
+
@click.option('--template-only', '-t', is_flag=True, default=False, help='Only generate coverage for template tests') # noqa: E501
|
|
2024
|
+
def coverage(unit_only, integration_only, template_only):
|
|
2025
|
+
"""Run coverage combine + coverage report."""
|
|
2026
|
+
cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*,*/templates/*'
|
|
2027
|
+
if unit_only:
|
|
2028
|
+
cmd += ' --data-file=.coverage.unit'
|
|
2029
|
+
elif integration_only:
|
|
2030
|
+
cmd += ' --data-file=.coverage.integration'
|
|
2031
|
+
elif template_only:
|
|
2032
|
+
cmd += ' --data-file=.coverage.template'
|
|
2033
|
+
else:
|
|
2034
|
+
Command.execute(f'{sys.executable} -m coverage combine --keep', name='coverage combine', cwd=ROOT_FOLDER)
|
|
2035
|
+
run_test(cmd, 'coverage', use_os_system=True)
|