secator 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of secator might be problematic. Click here for more details.
- secator/celery.py +40 -24
- secator/celery_signals.py +71 -68
- secator/celery_utils.py +43 -27
- secator/cli.py +520 -280
- secator/cli_helper.py +394 -0
- secator/click.py +87 -0
- secator/config.py +67 -39
- secator/configs/profiles/http_headless.yaml +6 -0
- secator/configs/profiles/http_record.yaml +6 -0
- secator/configs/profiles/tor.yaml +1 -1
- secator/configs/scans/domain.yaml +4 -2
- secator/configs/scans/host.yaml +1 -1
- secator/configs/scans/network.yaml +1 -4
- secator/configs/scans/subdomain.yaml +13 -1
- secator/configs/scans/url.yaml +1 -2
- secator/configs/workflows/cidr_recon.yaml +6 -4
- secator/configs/workflows/code_scan.yaml +1 -1
- secator/configs/workflows/host_recon.yaml +29 -3
- secator/configs/workflows/subdomain_recon.yaml +67 -16
- secator/configs/workflows/url_crawl.yaml +44 -15
- secator/configs/workflows/url_dirsearch.yaml +4 -4
- secator/configs/workflows/url_fuzz.yaml +25 -17
- secator/configs/workflows/url_params_fuzz.yaml +7 -0
- secator/configs/workflows/url_vuln.yaml +33 -8
- secator/configs/workflows/user_hunt.yaml +4 -2
- secator/configs/workflows/wordpress.yaml +5 -3
- secator/cve.py +718 -0
- secator/decorators.py +0 -454
- secator/definitions.py +49 -30
- secator/exporters/_base.py +2 -2
- secator/exporters/console.py +2 -2
- secator/exporters/table.py +4 -3
- secator/exporters/txt.py +1 -1
- secator/hooks/mongodb.py +2 -4
- secator/installer.py +77 -49
- secator/loader.py +116 -0
- secator/output_types/_base.py +3 -0
- secator/output_types/certificate.py +63 -63
- secator/output_types/error.py +4 -5
- secator/output_types/info.py +2 -2
- secator/output_types/ip.py +3 -1
- secator/output_types/progress.py +5 -9
- secator/output_types/state.py +17 -17
- secator/output_types/tag.py +3 -0
- secator/output_types/target.py +10 -2
- secator/output_types/url.py +19 -7
- secator/output_types/vulnerability.py +11 -7
- secator/output_types/warning.py +2 -2
- secator/report.py +27 -15
- secator/rich.py +18 -10
- secator/runners/_base.py +446 -233
- secator/runners/_helpers.py +133 -24
- secator/runners/command.py +182 -102
- secator/runners/scan.py +33 -5
- secator/runners/task.py +13 -7
- secator/runners/workflow.py +105 -72
- secator/scans/__init__.py +2 -2
- secator/serializers/dataclass.py +20 -20
- secator/tasks/__init__.py +4 -4
- secator/tasks/_categories.py +39 -27
- secator/tasks/arjun.py +9 -5
- secator/tasks/bbot.py +53 -21
- secator/tasks/bup.py +19 -5
- secator/tasks/cariddi.py +24 -3
- secator/tasks/dalfox.py +26 -7
- secator/tasks/dirsearch.py +10 -4
- secator/tasks/dnsx.py +70 -25
- secator/tasks/feroxbuster.py +11 -3
- secator/tasks/ffuf.py +42 -6
- secator/tasks/fping.py +20 -8
- secator/tasks/gau.py +3 -1
- secator/tasks/gf.py +3 -3
- secator/tasks/gitleaks.py +2 -2
- secator/tasks/gospider.py +7 -1
- secator/tasks/grype.py +5 -4
- secator/tasks/h8mail.py +2 -1
- secator/tasks/httpx.py +18 -5
- secator/tasks/katana.py +35 -15
- secator/tasks/maigret.py +4 -4
- secator/tasks/mapcidr.py +3 -3
- secator/tasks/msfconsole.py +4 -4
- secator/tasks/naabu.py +2 -2
- secator/tasks/nmap.py +12 -14
- secator/tasks/nuclei.py +3 -3
- secator/tasks/searchsploit.py +4 -5
- secator/tasks/subfinder.py +2 -2
- secator/tasks/testssl.py +264 -263
- secator/tasks/trivy.py +5 -5
- secator/tasks/wafw00f.py +21 -3
- secator/tasks/wpprobe.py +90 -83
- secator/tasks/wpscan.py +6 -5
- secator/template.py +218 -104
- secator/thread.py +15 -15
- secator/tree.py +196 -0
- secator/utils.py +131 -123
- secator/utils_test.py +60 -19
- secator/workflows/__init__.py +2 -2
- {secator-0.15.1.dist-info → secator-0.16.1.dist-info}/METADATA +36 -36
- secator-0.16.1.dist-info/RECORD +132 -0
- secator/configs/profiles/default.yaml +0 -8
- secator/configs/workflows/url_nuclei.yaml +0 -11
- secator/tasks/dnsxbrute.py +0 -42
- secator-0.15.1.dist-info/RECORD +0 -128
- {secator-0.15.1.dist-info → secator-0.16.1.dist-info}/WHEEL +0 -0
- {secator-0.15.1.dist-info → secator-0.16.1.dist-info}/entry_points.txt +0 -0
- {secator-0.15.1.dist-info → secator-0.16.1.dist-info}/licenses/LICENSE +0 -0
secator/runners/command.py
CHANGED
|
@@ -7,7 +7,6 @@ import shlex
|
|
|
7
7
|
import signal
|
|
8
8
|
import subprocess
|
|
9
9
|
import sys
|
|
10
|
-
import uuid
|
|
11
10
|
|
|
12
11
|
from time import time
|
|
13
12
|
|
|
@@ -16,7 +15,7 @@ from fp.fp import FreeProxy
|
|
|
16
15
|
|
|
17
16
|
from secator.definitions import OPT_NOT_SUPPORTED, OPT_PIPE_INPUT
|
|
18
17
|
from secator.config import CONFIG
|
|
19
|
-
from secator.output_types import Info, Warning, Error,
|
|
18
|
+
from secator.output_types import Info, Warning, Error, Stat
|
|
20
19
|
from secator.runners import Runner
|
|
21
20
|
from secator.template import TemplateLoader
|
|
22
21
|
from secator.utils import debug, rich_escape as _s
|
|
@@ -95,6 +94,7 @@ class Command(Runner):
|
|
|
95
94
|
# Hooks
|
|
96
95
|
hooks = [
|
|
97
96
|
'on_cmd',
|
|
97
|
+
'on_cmd_opts',
|
|
98
98
|
'on_cmd_done',
|
|
99
99
|
'on_line'
|
|
100
100
|
]
|
|
@@ -105,6 +105,9 @@ class Command(Runner):
|
|
|
105
105
|
# Return code
|
|
106
106
|
return_code = -1
|
|
107
107
|
|
|
108
|
+
# Exit ok
|
|
109
|
+
exit_ok = False
|
|
110
|
+
|
|
108
111
|
# Output
|
|
109
112
|
output = ''
|
|
110
113
|
|
|
@@ -122,6 +125,7 @@ class Command(Runner):
|
|
|
122
125
|
config = TemplateLoader(input={
|
|
123
126
|
'name': self.__class__.__name__,
|
|
124
127
|
'type': 'task',
|
|
128
|
+
'input_types': self.input_types,
|
|
125
129
|
'description': run_opts.get('description', None)
|
|
126
130
|
})
|
|
127
131
|
|
|
@@ -130,6 +134,12 @@ class Command(Runner):
|
|
|
130
134
|
caller = run_opts.get('caller', None)
|
|
131
135
|
results = run_opts.pop('results', [])
|
|
132
136
|
context = run_opts.pop('context', {})
|
|
137
|
+
node_id = context.get('node_id', None)
|
|
138
|
+
node_name = context.get('node_name', None)
|
|
139
|
+
if node_id:
|
|
140
|
+
config.node_id = node_id
|
|
141
|
+
if node_name:
|
|
142
|
+
config.node_name = context.get('node_name')
|
|
133
143
|
self.skip_if_no_inputs = run_opts.pop('skip_if_no_inputs', False)
|
|
134
144
|
|
|
135
145
|
# Prepare validators
|
|
@@ -182,7 +192,7 @@ class Command(Runner):
|
|
|
182
192
|
self._build_cmd()
|
|
183
193
|
|
|
184
194
|
# Run on_cmd hook
|
|
185
|
-
self.run_hooks('on_cmd')
|
|
195
|
+
self.run_hooks('on_cmd', sub='init')
|
|
186
196
|
|
|
187
197
|
# Add sudo to command if it is required
|
|
188
198
|
if self.requires_sudo:
|
|
@@ -208,7 +218,8 @@ class Command(Runner):
|
|
|
208
218
|
many_targets = len(self.inputs) > 1
|
|
209
219
|
targets_over_chunk_size = self.input_chunk_size and len(self.inputs) > self.input_chunk_size
|
|
210
220
|
has_file_flag = self.file_flag is not None
|
|
211
|
-
|
|
221
|
+
is_chunk = self.chunk
|
|
222
|
+
chunk_it = (sync and many_targets and not has_file_flag and not is_chunk) or (not sync and many_targets and targets_over_chunk_size and not is_chunk) # noqa: E501
|
|
212
223
|
return chunk_it
|
|
213
224
|
|
|
214
225
|
@classmethod
|
|
@@ -218,26 +229,41 @@ class Command(Runner):
|
|
|
218
229
|
results = kwargs.get('results', [])
|
|
219
230
|
kwargs['sync'] = False
|
|
220
231
|
name = cls.__name__
|
|
221
|
-
|
|
232
|
+
profile = cls.profile(kwargs) if callable(cls.profile) else cls.profile
|
|
233
|
+
return run_command.apply_async(args=[results, name] + list(args), kwargs={'opts': kwargs}, queue=profile)
|
|
222
234
|
|
|
223
235
|
@classmethod
|
|
224
236
|
def s(cls, *args, **kwargs):
|
|
225
237
|
# TODO: Move this to TaskBase
|
|
226
238
|
from secator.celery import run_command
|
|
227
|
-
|
|
239
|
+
profile = cls.profile(kwargs) if callable(cls.profile) else cls.profile
|
|
240
|
+
return run_command.s(cls.__name__, *args, opts=kwargs).set(queue=profile)
|
|
228
241
|
|
|
229
242
|
@classmethod
|
|
230
|
-
def si(cls, *args, results=
|
|
243
|
+
def si(cls, *args, results=None, **kwargs):
|
|
231
244
|
# TODO: Move this to TaskBase
|
|
232
245
|
from secator.celery import run_command
|
|
233
|
-
|
|
246
|
+
profile = cls.profile(kwargs) if callable(cls.profile) else cls.profile
|
|
247
|
+
return run_command.si(results or [], cls.__name__, *args, opts=kwargs).set(queue=profile)
|
|
248
|
+
|
|
249
|
+
def get_opt_value(self, opt_name, preprocess=False, process=False):
|
|
250
|
+
"""Get option value as inputed by the user.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
opt_name (str): Option name.
|
|
254
|
+
preprocess (bool): Preprocess the value with the option preprocessor function if it exists.
|
|
255
|
+
process (bool): Process the value with the option processor function if it exists.
|
|
234
256
|
|
|
235
|
-
|
|
257
|
+
Returns:
|
|
258
|
+
Any: Option value.
|
|
259
|
+
"""
|
|
236
260
|
return Command._get_opt_value(
|
|
237
261
|
self.run_opts,
|
|
238
262
|
opt_name,
|
|
239
263
|
dict(self.opts, **self.meta_opts),
|
|
240
|
-
|
|
264
|
+
opt_aliases=self.opt_aliases,
|
|
265
|
+
preprocess=preprocess,
|
|
266
|
+
process=process)
|
|
241
267
|
|
|
242
268
|
@classmethod
|
|
243
269
|
def get_version_flag(cls):
|
|
@@ -246,17 +272,20 @@ class Command(Runner):
|
|
|
246
272
|
return cls.version_flag or f'{cls.opt_prefix}version'
|
|
247
273
|
|
|
248
274
|
@classmethod
|
|
249
|
-
def get_version_info(cls):
|
|
275
|
+
def get_version_info(cls, bleeding=False):
|
|
250
276
|
from secator.installer import get_version_info
|
|
251
277
|
return get_version_info(
|
|
252
278
|
cls.cmd.split(' ')[0],
|
|
253
279
|
cls.get_version_flag(),
|
|
254
280
|
cls.install_github_handle,
|
|
255
|
-
cls.install_cmd
|
|
281
|
+
cls.install_cmd,
|
|
282
|
+
cls.install_version,
|
|
283
|
+
bleeding=bleeding
|
|
256
284
|
)
|
|
257
285
|
|
|
258
286
|
@classmethod
|
|
259
287
|
def get_supported_opts(cls):
|
|
288
|
+
# TODO: Replace this with get_command_options called on the command class
|
|
260
289
|
def convert(d):
|
|
261
290
|
for k, v in d.items():
|
|
262
291
|
if hasattr(v, '__name__') and v.__name__ in ['str', 'int', 'float']:
|
|
@@ -305,7 +334,7 @@ class Command(Runner):
|
|
|
305
334
|
name = name or cmd.split(' ')[0]
|
|
306
335
|
kwargs['print_cmd'] = not kwargs.get('quiet', False)
|
|
307
336
|
kwargs['print_line'] = True
|
|
308
|
-
kwargs['
|
|
337
|
+
kwargs['process'] = kwargs.get('process', False)
|
|
309
338
|
cmd_instance = type(name, (Command,), {'cmd': cmd, 'input_required': False})(**kwargs)
|
|
310
339
|
for k, v in cls_attributes.items():
|
|
311
340
|
setattr(cmd_instance, k, v)
|
|
@@ -344,8 +373,8 @@ class Command(Runner):
|
|
|
344
373
|
self.run_opts['proxy'] = proxy
|
|
345
374
|
|
|
346
375
|
if proxy != 'proxychains' and self.proxy and not proxy:
|
|
347
|
-
self.
|
|
348
|
-
|
|
376
|
+
warning = Warning(message=rf'Ignoring proxy "{self.proxy}" (reason: not supported) \[[bold yellow3]{self.unique_name}[/]]') # noqa: E501
|
|
377
|
+
self._print(repr(warning))
|
|
349
378
|
|
|
350
379
|
#----------#
|
|
351
380
|
# Internal #
|
|
@@ -375,30 +404,24 @@ class Command(Runner):
|
|
|
375
404
|
|
|
376
405
|
# Abort if dry run
|
|
377
406
|
if self.dry_run:
|
|
378
|
-
self.
|
|
407
|
+
self.print_description()
|
|
379
408
|
self.print_command()
|
|
409
|
+
yield Info(message=self.cmd)
|
|
380
410
|
return
|
|
381
411
|
|
|
382
|
-
# Print task description
|
|
383
|
-
self.print_description()
|
|
384
|
-
|
|
385
412
|
# Abort if no inputs
|
|
386
413
|
if len(self.inputs) == 0 and self.skip_if_no_inputs:
|
|
387
|
-
yield Warning(message=f'{self.unique_name} skipped (no inputs)'
|
|
414
|
+
yield Warning(message=f'{self.unique_name} skipped (no inputs)')
|
|
388
415
|
return
|
|
389
416
|
|
|
390
|
-
#
|
|
391
|
-
|
|
392
|
-
|
|
417
|
+
# Print command
|
|
418
|
+
self.print_description()
|
|
419
|
+
self.print_command()
|
|
393
420
|
|
|
394
421
|
# Check for sudo requirements and prepare the password if needed
|
|
395
422
|
sudo_password, error = self._prompt_sudo(self.cmd)
|
|
396
423
|
if error:
|
|
397
|
-
yield Error(
|
|
398
|
-
message=error,
|
|
399
|
-
_source=self.unique_name,
|
|
400
|
-
_uuid=str(uuid.uuid4())
|
|
401
|
-
)
|
|
424
|
+
yield Error(message=error)
|
|
402
425
|
return
|
|
403
426
|
|
|
404
427
|
# Prepare cmds
|
|
@@ -408,18 +431,10 @@ class Command(Runner):
|
|
|
408
431
|
if not self.no_process and not self.is_installed():
|
|
409
432
|
if CONFIG.security.auto_install_commands:
|
|
410
433
|
from secator.installer import ToolInstaller
|
|
411
|
-
yield Info(
|
|
412
|
-
message=f'Command {self.name} is missing but auto-installing since security.autoinstall_commands is set', # noqa: E501
|
|
413
|
-
_source=self.unique_name,
|
|
414
|
-
_uuid=str(uuid.uuid4())
|
|
415
|
-
)
|
|
434
|
+
yield Info(message=f'Command {self.name} is missing but auto-installing since security.autoinstall_commands is set') # noqa: E501
|
|
416
435
|
status = ToolInstaller.install(self.__class__)
|
|
417
436
|
if not status.is_ok():
|
|
418
|
-
yield Error(
|
|
419
|
-
message=f'Failed installing {self.cmd_name}',
|
|
420
|
-
_source=self.unique_name,
|
|
421
|
-
_uuid=str(uuid.uuid4())
|
|
422
|
-
)
|
|
437
|
+
yield Error(message=f'Failed installing {self.cmd_name}')
|
|
423
438
|
return
|
|
424
439
|
|
|
425
440
|
# Output and results
|
|
@@ -437,7 +452,6 @@ class Command(Runner):
|
|
|
437
452
|
shell=self.shell,
|
|
438
453
|
env=env,
|
|
439
454
|
cwd=self.cwd)
|
|
440
|
-
self.print_command()
|
|
441
455
|
|
|
442
456
|
# If sudo password is provided, send it to stdin
|
|
443
457
|
if sudo_password:
|
|
@@ -452,7 +466,7 @@ class Command(Runner):
|
|
|
452
466
|
yield from self.process_line(line)
|
|
453
467
|
|
|
454
468
|
# Run hooks after cmd has completed successfully
|
|
455
|
-
result = self.run_hooks('on_cmd_done')
|
|
469
|
+
result = self.run_hooks('on_cmd_done', sub='end')
|
|
456
470
|
if result:
|
|
457
471
|
yield from result
|
|
458
472
|
|
|
@@ -460,9 +474,9 @@ class Command(Runner):
|
|
|
460
474
|
yield from self.handle_file_not_found(e)
|
|
461
475
|
|
|
462
476
|
except BaseException as e:
|
|
463
|
-
self.debug(f'{self.unique_name}: {type(e).__name__}.', sub='
|
|
477
|
+
self.debug(f'{self.unique_name}: {type(e).__name__}.', sub='end')
|
|
464
478
|
self.stop_process()
|
|
465
|
-
yield Error.from_exception(e
|
|
479
|
+
yield Error.from_exception(e)
|
|
466
480
|
|
|
467
481
|
finally:
|
|
468
482
|
yield from self._wait_for_end()
|
|
@@ -493,7 +507,7 @@ class Command(Runner):
|
|
|
493
507
|
line = line.replace('\\x0d\\x0a', '\n')
|
|
494
508
|
|
|
495
509
|
# Run on_line hooks
|
|
496
|
-
line = self.run_hooks('on_line', line)
|
|
510
|
+
line = self.run_hooks('on_line', line, sub='line.process')
|
|
497
511
|
if line is None:
|
|
498
512
|
return
|
|
499
513
|
|
|
@@ -518,18 +532,18 @@ class Command(Runner):
|
|
|
518
532
|
|
|
519
533
|
def print_description(self):
|
|
520
534
|
"""Print description"""
|
|
521
|
-
if self.sync and not self.has_children and self.caller and self.description:
|
|
535
|
+
if self.sync and not self.has_children and self.caller and self.description and self.print_cmd:
|
|
522
536
|
self._print(f'\n[bold gold3]:wrench: {self.description} [dim cyan]({self.config.name})[/][/] ...', rich=True)
|
|
523
537
|
|
|
524
538
|
def print_command(self):
|
|
525
539
|
"""Print command."""
|
|
526
540
|
if self.print_cmd:
|
|
527
|
-
cmd_str = _s(self.cmd)
|
|
541
|
+
cmd_str = f':zap: {_s(self.cmd)}'
|
|
528
542
|
if self.sync and self.chunk and self.chunk_count:
|
|
529
543
|
cmd_str += f' [dim gray11]({self.chunk}/{self.chunk_count})[/]'
|
|
530
|
-
self._print(cmd_str, color='bold
|
|
531
|
-
self.debug('
|
|
532
|
-
self.debug('
|
|
544
|
+
self._print(cmd_str, color='bold green', rich=True)
|
|
545
|
+
self.debug('command', obj={'cmd': self.cmd}, sub='start')
|
|
546
|
+
self.debug('options', obj=self.cmd_options, sub='start')
|
|
533
547
|
|
|
534
548
|
def handle_file_not_found(self, exc):
|
|
535
549
|
"""Handle case where binary is not found.
|
|
@@ -540,6 +554,7 @@ class Command(Runner):
|
|
|
540
554
|
Yields:
|
|
541
555
|
secator.output_types.Error: the error.
|
|
542
556
|
"""
|
|
557
|
+
self.debug('command not found', sub='end')
|
|
543
558
|
self.return_code = 127
|
|
544
559
|
if self.config.name in str(exc):
|
|
545
560
|
message = 'Executable not found.'
|
|
@@ -548,16 +563,16 @@ class Command(Runner):
|
|
|
548
563
|
error = Error(message=message)
|
|
549
564
|
else:
|
|
550
565
|
error = Error.from_exception(exc)
|
|
551
|
-
error._source = self.unique_name
|
|
552
|
-
error._uuid = str(uuid.uuid4())
|
|
553
566
|
yield error
|
|
554
567
|
|
|
555
|
-
def stop_process(self):
|
|
568
|
+
def stop_process(self, exit_ok=False):
|
|
556
569
|
"""Sends SIGINT to running process, if any."""
|
|
557
570
|
if not self.process:
|
|
558
571
|
return
|
|
559
572
|
self.debug(f'Sending SIGINT to process {self.process.pid}.', sub='error')
|
|
560
573
|
self.process.send_signal(signal.SIGINT)
|
|
574
|
+
if exit_ok:
|
|
575
|
+
self.exit_ok = True
|
|
561
576
|
|
|
562
577
|
def stats(self):
|
|
563
578
|
"""Gather stats about the current running process, if any."""
|
|
@@ -673,33 +688,23 @@ class Command(Runner):
|
|
|
673
688
|
for line in self.process.stdout.readlines():
|
|
674
689
|
yield from self.process_line(line)
|
|
675
690
|
self.process.wait()
|
|
676
|
-
self.return_code = self.process.returncode
|
|
691
|
+
self.return_code = 0 if self.exit_ok else self.process.returncode
|
|
677
692
|
self.process.stdout.close()
|
|
678
693
|
self.return_code = 0 if self.ignore_return_code else self.return_code
|
|
679
694
|
self.output = self.output.strip()
|
|
680
695
|
self.killed = self.return_code == -2 or self.killed
|
|
681
|
-
self.debug(f'
|
|
696
|
+
self.debug(f'return code: {self.return_code}', sub='end')
|
|
682
697
|
|
|
683
698
|
if self.killed:
|
|
684
699
|
error = 'Process was killed manually (CTRL+C / CTRL+X)'
|
|
685
|
-
yield Error(
|
|
686
|
-
message=error,
|
|
687
|
-
_source=self.unique_name,
|
|
688
|
-
_uuid=str(uuid.uuid4())
|
|
689
|
-
)
|
|
700
|
+
yield Error(message=error)
|
|
690
701
|
|
|
691
702
|
elif self.return_code != 0:
|
|
692
703
|
error = f'Command failed with return code {self.return_code}'
|
|
693
704
|
last_lines = self.output.split('\n')
|
|
694
705
|
last_lines = last_lines[max(0, len(last_lines) - 2):]
|
|
695
706
|
last_lines = [line for line in last_lines if line != '']
|
|
696
|
-
yield Error(
|
|
697
|
-
message=error,
|
|
698
|
-
traceback='\n'.join(last_lines),
|
|
699
|
-
traceback_title='Last stdout lines',
|
|
700
|
-
_source=self.unique_name,
|
|
701
|
-
_uuid=str(uuid.uuid4())
|
|
702
|
-
)
|
|
707
|
+
yield Error(message=error, traceback='\n'.join(last_lines), traceback_title='Last stdout lines')
|
|
703
708
|
|
|
704
709
|
@staticmethod
|
|
705
710
|
def _process_opts(
|
|
@@ -708,7 +713,9 @@ class Command(Runner):
|
|
|
708
713
|
opt_key_map={},
|
|
709
714
|
opt_value_map={},
|
|
710
715
|
opt_prefix='-',
|
|
711
|
-
|
|
716
|
+
opt_aliases=None,
|
|
717
|
+
preprocess=False,
|
|
718
|
+
process=True):
|
|
712
719
|
"""Process a dict of options using a config, option key map / value map and option character like '-' or '--'.
|
|
713
720
|
|
|
714
721
|
Args:
|
|
@@ -717,56 +724,62 @@ class Command(Runner):
|
|
|
717
724
|
opt_key_map (dict[str, str | Callable]): A dict to map option key with their actual values.
|
|
718
725
|
opt_value_map (dict, str | Callable): A dict to map option values with their actual values.
|
|
719
726
|
opt_prefix (str, default: '-'): Option prefix.
|
|
720
|
-
|
|
727
|
+
opt_aliases (str | None, default: None): Aliases to try.
|
|
728
|
+
preprocess (bool, default: True): Preprocess the value with the option preprocessor function if it exists.
|
|
729
|
+
process (bool, default: True): Process the value with the option processor function if it exists.
|
|
721
730
|
|
|
722
731
|
Returns:
|
|
723
732
|
dict: Processed options dict.
|
|
724
733
|
"""
|
|
725
734
|
opts_dict = {}
|
|
726
735
|
for opt_name, opt_conf in opts_conf.items():
|
|
727
|
-
debug('before get_opt_value', obj={'name': opt_name, 'conf': opt_conf}, obj_after=False, sub='
|
|
736
|
+
debug('before get_opt_value', obj={'name': opt_name, 'conf': opt_conf}, obj_after=False, sub='init.options', verbose=True) # noqa: E501
|
|
728
737
|
|
|
729
738
|
# Save original opt name
|
|
730
739
|
original_opt_name = opt_name
|
|
731
740
|
|
|
741
|
+
# Copy opt conf
|
|
742
|
+
conf = opt_conf.copy()
|
|
743
|
+
|
|
732
744
|
# Get opt value
|
|
733
|
-
default_val =
|
|
745
|
+
default_val = conf.get('default')
|
|
734
746
|
opt_val = Command._get_opt_value(
|
|
735
747
|
opts,
|
|
736
748
|
opt_name,
|
|
737
749
|
opts_conf,
|
|
738
|
-
|
|
739
|
-
default=default_val
|
|
750
|
+
opt_aliases=opt_aliases,
|
|
751
|
+
default=default_val,
|
|
752
|
+
preprocess=preprocess,
|
|
753
|
+
process=process)
|
|
740
754
|
|
|
741
|
-
debug('after get_opt_value', obj={'name': opt_name, 'value': opt_val, 'conf':
|
|
755
|
+
debug('after get_opt_value', obj={'name': opt_name, 'value': opt_val, 'conf': conf}, obj_after=False, sub='init.options', verbose=True) # noqa: E501
|
|
742
756
|
|
|
743
757
|
# Skip option if value is falsy
|
|
744
758
|
if opt_val in [None, False, []]:
|
|
745
|
-
debug('skipped (falsy)', obj={'name': opt_name, 'value': opt_val}, obj_after=False, sub='
|
|
759
|
+
debug('skipped (falsy)', obj={'name': opt_name, 'value': opt_val}, obj_after=False, sub='init.options', verbose=True) # noqa: E501
|
|
746
760
|
continue
|
|
747
761
|
|
|
748
|
-
# Apply process function on opt value
|
|
749
|
-
if 'process' in opt_conf:
|
|
750
|
-
func = opt_conf['process']
|
|
751
|
-
opt_val = func(opt_val)
|
|
752
|
-
|
|
753
762
|
# Convert opt value to expected command opt value
|
|
754
763
|
mapped_opt_val = opt_value_map.get(opt_name)
|
|
755
764
|
if mapped_opt_val:
|
|
765
|
+
conf.pop('pre_process', None)
|
|
766
|
+
conf.pop('process', None)
|
|
756
767
|
if callable(mapped_opt_val):
|
|
757
768
|
opt_val = mapped_opt_val(opt_val)
|
|
758
769
|
else:
|
|
759
770
|
opt_val = mapped_opt_val
|
|
771
|
+
elif 'pre_process' in conf:
|
|
772
|
+
opt_val = conf['pre_process'](opt_val)
|
|
760
773
|
|
|
761
774
|
# Convert opt name to expected command opt name
|
|
762
775
|
mapped_opt_name = opt_key_map.get(opt_name)
|
|
763
776
|
if mapped_opt_name is not None:
|
|
764
777
|
if mapped_opt_name == OPT_NOT_SUPPORTED:
|
|
765
|
-
debug('skipped (unsupported)', obj={'name': opt_name, 'value': opt_val}, sub='
|
|
778
|
+
debug('skipped (unsupported)', obj={'name': opt_name, 'value': opt_val}, sub='init.options', verbose=True) # noqa: E501
|
|
766
779
|
continue
|
|
767
780
|
else:
|
|
768
781
|
opt_name = mapped_opt_name
|
|
769
|
-
debug('mapped key / value', obj={'name': opt_name, 'value': opt_val}, obj_after=False, sub='
|
|
782
|
+
debug('mapped key / value', obj={'name': opt_name, 'value': opt_val}, obj_after=False, sub='init.options', verbose=True) # noqa: E501
|
|
770
783
|
|
|
771
784
|
# Avoid shell injections and detect opt prefix
|
|
772
785
|
opt_name = str(opt_name).split(' ')[0] # avoid cmd injection
|
|
@@ -780,14 +793,14 @@ class Command(Runner):
|
|
|
780
793
|
|
|
781
794
|
# Append opt name + opt value to option string.
|
|
782
795
|
# Note: does not append opt value if value is True (flag)
|
|
783
|
-
opts_dict[original_opt_name] = {'name': opt_name, 'value': opt_val, 'conf':
|
|
784
|
-
debug('final', obj={'name': original_opt_name, 'value': opt_val}, sub='
|
|
796
|
+
opts_dict[original_opt_name] = {'name': opt_name, 'value': opt_val, 'conf': conf}
|
|
797
|
+
debug('final', obj={'name': original_opt_name, 'value': opt_val}, sub='init.options', obj_after=False, verbose=True) # noqa: E501
|
|
785
798
|
|
|
786
799
|
return opts_dict
|
|
787
800
|
|
|
788
801
|
@staticmethod
|
|
789
802
|
def _validate_chunked_input(self, inputs):
|
|
790
|
-
"""Command does not
|
|
803
|
+
"""Command does not support multiple inputs in non-worker mode. Consider running with a remote worker instead."""
|
|
791
804
|
if len(inputs) > 1 and self.sync and self.file_flag is None:
|
|
792
805
|
return False
|
|
793
806
|
return True
|
|
@@ -807,28 +820,79 @@ class Command(Runner):
|
|
|
807
820
|
|
|
808
821
|
@staticmethod
|
|
809
822
|
def _get_opt_default(opt_name, opts_conf):
|
|
823
|
+
"""Get the default value of an option.
|
|
824
|
+
|
|
825
|
+
Args:
|
|
826
|
+
opt_name (str): The name of the option to get the default value of (no aliases allowed).
|
|
827
|
+
opts_conf (dict): The options configuration, indexed by option name.
|
|
828
|
+
|
|
829
|
+
Returns:
|
|
830
|
+
any: The default value of the option.
|
|
831
|
+
"""
|
|
810
832
|
for k, v in opts_conf.items():
|
|
811
833
|
if k == opt_name:
|
|
812
834
|
return v.get('default', None)
|
|
813
835
|
return None
|
|
814
836
|
|
|
815
837
|
@staticmethod
|
|
816
|
-
def _get_opt_value(opts, opt_name, opts_conf={},
|
|
838
|
+
def _get_opt_value(opts, opt_name, opts_conf={}, opt_aliases=None, default=None, preprocess=False, process=False):
|
|
839
|
+
"""Get the value of an option.
|
|
840
|
+
|
|
841
|
+
Args:
|
|
842
|
+
opts (dict): The options dict to search (input opts).
|
|
843
|
+
opt_name (str): The name of the option to get the value of.
|
|
844
|
+
opts_conf (dict): The options configuration, indexed by option name.
|
|
845
|
+
opt_aliases (list): The aliases to try.
|
|
846
|
+
default (any): The default value to return if the option is not found.
|
|
847
|
+
preprocess (bool): Whether to preprocess the value using the option preprocessor function.
|
|
848
|
+
process (bool): Whether to process the value using the option processor function.
|
|
849
|
+
|
|
850
|
+
Returns:
|
|
851
|
+
any: The value of the option.
|
|
852
|
+
|
|
853
|
+
Example:
|
|
854
|
+
opts = {'target': 'example.com'}
|
|
855
|
+
opts_conf = {'target': {'type': 'str', 'short': 't', 'default': 'example.com', 'pre_process': lambda x: x.upper()}} # noqa: E501
|
|
856
|
+
opt_aliases = ['prefix_target', 'target']
|
|
857
|
+
|
|
858
|
+
# Example 1:
|
|
859
|
+
opt_name = 'target'
|
|
860
|
+
opt_value = Command._get_opt_value(opts, opt_name, opts_conf, opt_aliases, preprocess=True) # noqa: E501
|
|
861
|
+
print(opt_value)
|
|
862
|
+
# Output: EXAMPLE.COM
|
|
863
|
+
|
|
864
|
+
# Example 2:
|
|
865
|
+
opt_name = 'prefix_target'
|
|
866
|
+
opt_value = Command._get_opt_value(opts, opt_name, opts_conf, opt_aliases)
|
|
867
|
+
print(opt_value)
|
|
868
|
+
# Output: example.com
|
|
869
|
+
"""
|
|
817
870
|
default = default or Command._get_opt_default(opt_name, opts_conf)
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
opt_name,
|
|
822
|
-
|
|
871
|
+
opt_aliases = opt_aliases or []
|
|
872
|
+
opt_names = []
|
|
873
|
+
for prefix in opt_aliases:
|
|
874
|
+
opt_names.extend([f'{prefix}.{opt_name}', f'{prefix}_{opt_name}'])
|
|
875
|
+
opt_names.append(opt_name)
|
|
876
|
+
opt_names = list(dict.fromkeys(opt_names))
|
|
823
877
|
opt_values = [opts.get(o) for o in opt_names]
|
|
824
|
-
|
|
825
|
-
if
|
|
826
|
-
|
|
878
|
+
opt_conf = [conf for _, conf in opts_conf.items() if _ == opt_name]
|
|
879
|
+
if opt_conf:
|
|
880
|
+
opt_conf = opt_conf[0]
|
|
881
|
+
alias = opt_conf.get('short')
|
|
882
|
+
if alias:
|
|
883
|
+
opt_values.append(opts.get(alias))
|
|
827
884
|
if OPT_NOT_SUPPORTED in opt_values:
|
|
828
|
-
debug('skipped (unsupported)', obj={'name': opt_name}, obj_after=False, sub='
|
|
885
|
+
debug('skipped (unsupported)', obj={'name': opt_name}, obj_after=False, sub='init.options', verbose=True)
|
|
829
886
|
return None
|
|
830
887
|
value = next((v for v in opt_values if v is not None), default)
|
|
831
|
-
|
|
888
|
+
if opt_conf:
|
|
889
|
+
preprocessor = opt_conf.get('pre_process')
|
|
890
|
+
processor = opt_conf.get('process')
|
|
891
|
+
if preprocess and preprocessor:
|
|
892
|
+
value = preprocessor(value)
|
|
893
|
+
if process and processor:
|
|
894
|
+
value = processor(value)
|
|
895
|
+
debug('got opt value', obj={'name': opt_name, 'value': value, 'aliases': opt_names, 'values': opt_values}, obj_after=False, sub='init.options', verbose=True) # noqa: E501
|
|
832
896
|
return value
|
|
833
897
|
|
|
834
898
|
def _build_cmd(self):
|
|
@@ -849,7 +913,9 @@ class Command(Runner):
|
|
|
849
913
|
self.opt_key_map,
|
|
850
914
|
self.opt_value_map,
|
|
851
915
|
self.opt_prefix,
|
|
852
|
-
|
|
916
|
+
opt_aliases=self.opt_aliases,
|
|
917
|
+
preprocess=False,
|
|
918
|
+
process=False)
|
|
853
919
|
|
|
854
920
|
# Add meta options to cmd
|
|
855
921
|
meta_opts_dict = Command._process_opts(
|
|
@@ -858,22 +924,31 @@ class Command(Runner):
|
|
|
858
924
|
self.opt_key_map,
|
|
859
925
|
self.opt_value_map,
|
|
860
926
|
self.opt_prefix,
|
|
861
|
-
|
|
927
|
+
opt_aliases=self.opt_aliases,
|
|
928
|
+
preprocess=False,
|
|
929
|
+
process=False)
|
|
862
930
|
|
|
863
931
|
if opts_dict:
|
|
864
932
|
opts.update(opts_dict)
|
|
865
933
|
if meta_opts_dict:
|
|
866
934
|
opts.update(meta_opts_dict)
|
|
867
935
|
|
|
936
|
+
opts = self.run_hooks('on_cmd_opts', opts, sub='init')
|
|
937
|
+
|
|
868
938
|
if opts:
|
|
869
939
|
for opt_conf in opts.values():
|
|
870
940
|
conf = opt_conf['conf']
|
|
941
|
+
process = conf.get('process')
|
|
942
|
+
if process:
|
|
943
|
+
opt_conf['value'] = process(opt_conf['value'])
|
|
871
944
|
internal = conf.get('internal', False)
|
|
872
945
|
if internal:
|
|
873
946
|
continue
|
|
874
947
|
if conf.get('requires_sudo', False):
|
|
875
948
|
self.requires_sudo = True
|
|
876
949
|
opts_str += ' ' + Command._build_opt_str(opt_conf)
|
|
950
|
+
if '{target}' in opts_str:
|
|
951
|
+
opts_str = opts_str.replace('{target}', self.inputs[0])
|
|
877
952
|
self.cmd_options = opts
|
|
878
953
|
self.cmd += opts_str
|
|
879
954
|
|
|
@@ -881,14 +956,19 @@ class Command(Runner):
|
|
|
881
956
|
def _build_opt_str(opt):
|
|
882
957
|
"""Build option string."""
|
|
883
958
|
conf = opt['conf']
|
|
884
|
-
opts_str = f'{opt["name"]}'
|
|
885
959
|
shlex_quote = conf.get('shlex', True)
|
|
886
960
|
value = opt['value']
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
961
|
+
opt_name = opt['name']
|
|
962
|
+
opts_str = ''
|
|
963
|
+
value = [value] if not isinstance(value, list) else value
|
|
964
|
+
for val in value:
|
|
965
|
+
if val is True:
|
|
966
|
+
opts_str += f'{opt_name}'
|
|
967
|
+
else:
|
|
968
|
+
if shlex_quote:
|
|
969
|
+
val = shlex.quote(str(val))
|
|
970
|
+
opts_str += f'{opt_name} {val} '
|
|
971
|
+
return opts_str.strip()
|
|
892
972
|
|
|
893
973
|
def _build_cmd_input(self):
|
|
894
974
|
"""Many commands take as input a string or a list. This function facilitate this based on whether we pass a
|
secator/runners/scan.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from dotmap import DotMap
|
|
2
3
|
|
|
3
4
|
from secator.config import CONFIG
|
|
5
|
+
from secator.output_types.info import Info
|
|
4
6
|
from secator.runners._base import Runner
|
|
5
7
|
from secator.runners.workflow import Workflow
|
|
6
8
|
from secator.utils import merge_opts
|
|
7
9
|
|
|
10
|
+
|
|
8
11
|
logger = logging.getLogger(__name__)
|
|
9
12
|
|
|
10
13
|
|
|
@@ -29,16 +32,36 @@ class Scan(Runner):
|
|
|
29
32
|
|
|
30
33
|
scan_opts = self.config.options
|
|
31
34
|
|
|
35
|
+
# Set hooks and reports
|
|
36
|
+
self.enable_hooks = False # Celery will handle hooks
|
|
37
|
+
self.enable_reports = True # Workflow will handle reports
|
|
38
|
+
self.print_item = not self.sync
|
|
39
|
+
|
|
32
40
|
# Build chain of workflows
|
|
33
41
|
sigs = []
|
|
42
|
+
sig = None
|
|
34
43
|
for name, workflow_opts in self.config.workflows.items():
|
|
35
44
|
run_opts = self.run_opts.copy()
|
|
36
45
|
run_opts.pop('profiles', None)
|
|
37
46
|
run_opts['no_poll'] = True
|
|
38
47
|
run_opts['caller'] = 'Scan'
|
|
48
|
+
run_opts['has_parent'] = True
|
|
49
|
+
run_opts['enable_reports'] = False
|
|
50
|
+
run_opts['print_profiles'] = False
|
|
39
51
|
opts = merge_opts(scan_opts, workflow_opts, run_opts)
|
|
40
52
|
name = name.split('/')[0]
|
|
41
53
|
config = TemplateLoader(name=f'workflow/{name}')
|
|
54
|
+
if not config:
|
|
55
|
+
raise ValueError(f'Workflow {name} not found')
|
|
56
|
+
|
|
57
|
+
# Skip workflow if condition is not met
|
|
58
|
+
condition = workflow_opts.pop('if', None) if workflow_opts else None
|
|
59
|
+
local_ns = {'opts': DotMap(opts)}
|
|
60
|
+
if condition and not eval(condition, {"__builtins__": {}}, local_ns):
|
|
61
|
+
self.add_result(Info(message=f'Skipped workflow {name} because condition is not met: {condition}'))
|
|
62
|
+
continue
|
|
63
|
+
|
|
64
|
+
# Build workflow
|
|
42
65
|
workflow = Workflow(
|
|
43
66
|
config,
|
|
44
67
|
self.inputs,
|
|
@@ -52,8 +75,13 @@ class Scan(Runner):
|
|
|
52
75
|
self.add_subtask(task_id, task_info['name'], task_info['descr'])
|
|
53
76
|
sigs.append(celery_workflow)
|
|
54
77
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
78
|
+
for result in workflow.results:
|
|
79
|
+
self.add_result(result, print=False, hooks=False)
|
|
80
|
+
|
|
81
|
+
if sigs:
|
|
82
|
+
sig = chain(
|
|
83
|
+
mark_runner_started.si([], self).set(queue='results'),
|
|
84
|
+
*sigs,
|
|
85
|
+
mark_runner_completed.s(self).set(queue='results'),
|
|
86
|
+
)
|
|
87
|
+
return sig
|