secator 0.15.1__py3-none-any.whl → 0.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of secator might be problematic. Click here for more details.

Files changed (106) hide show
  1. secator/celery.py +40 -24
  2. secator/celery_signals.py +71 -68
  3. secator/celery_utils.py +43 -27
  4. secator/cli.py +520 -280
  5. secator/cli_helper.py +394 -0
  6. secator/click.py +87 -0
  7. secator/config.py +67 -39
  8. secator/configs/profiles/http_headless.yaml +6 -0
  9. secator/configs/profiles/http_record.yaml +6 -0
  10. secator/configs/profiles/tor.yaml +1 -1
  11. secator/configs/scans/domain.yaml +4 -2
  12. secator/configs/scans/host.yaml +1 -1
  13. secator/configs/scans/network.yaml +1 -4
  14. secator/configs/scans/subdomain.yaml +13 -1
  15. secator/configs/scans/url.yaml +1 -2
  16. secator/configs/workflows/cidr_recon.yaml +6 -4
  17. secator/configs/workflows/code_scan.yaml +1 -1
  18. secator/configs/workflows/host_recon.yaml +29 -3
  19. secator/configs/workflows/subdomain_recon.yaml +67 -16
  20. secator/configs/workflows/url_crawl.yaml +44 -15
  21. secator/configs/workflows/url_dirsearch.yaml +4 -4
  22. secator/configs/workflows/url_fuzz.yaml +25 -17
  23. secator/configs/workflows/url_params_fuzz.yaml +7 -0
  24. secator/configs/workflows/url_vuln.yaml +33 -8
  25. secator/configs/workflows/user_hunt.yaml +2 -1
  26. secator/configs/workflows/wordpress.yaml +5 -3
  27. secator/cve.py +718 -0
  28. secator/decorators.py +0 -454
  29. secator/definitions.py +49 -30
  30. secator/exporters/_base.py +2 -2
  31. secator/exporters/console.py +2 -2
  32. secator/exporters/table.py +4 -3
  33. secator/exporters/txt.py +1 -1
  34. secator/hooks/mongodb.py +2 -4
  35. secator/installer.py +77 -49
  36. secator/loader.py +116 -0
  37. secator/output_types/_base.py +3 -0
  38. secator/output_types/certificate.py +63 -63
  39. secator/output_types/error.py +4 -5
  40. secator/output_types/info.py +2 -2
  41. secator/output_types/ip.py +3 -1
  42. secator/output_types/progress.py +5 -9
  43. secator/output_types/state.py +17 -17
  44. secator/output_types/tag.py +3 -0
  45. secator/output_types/target.py +10 -2
  46. secator/output_types/url.py +19 -7
  47. secator/output_types/vulnerability.py +11 -7
  48. secator/output_types/warning.py +2 -2
  49. secator/report.py +27 -15
  50. secator/rich.py +18 -10
  51. secator/runners/_base.py +446 -233
  52. secator/runners/_helpers.py +133 -24
  53. secator/runners/command.py +182 -102
  54. secator/runners/scan.py +33 -5
  55. secator/runners/task.py +13 -7
  56. secator/runners/workflow.py +105 -72
  57. secator/scans/__init__.py +2 -2
  58. secator/serializers/dataclass.py +20 -20
  59. secator/tasks/__init__.py +4 -4
  60. secator/tasks/_categories.py +39 -27
  61. secator/tasks/arjun.py +9 -5
  62. secator/tasks/bbot.py +53 -21
  63. secator/tasks/bup.py +19 -5
  64. secator/tasks/cariddi.py +24 -3
  65. secator/tasks/dalfox.py +26 -7
  66. secator/tasks/dirsearch.py +10 -4
  67. secator/tasks/dnsx.py +70 -25
  68. secator/tasks/feroxbuster.py +11 -3
  69. secator/tasks/ffuf.py +42 -6
  70. secator/tasks/fping.py +20 -8
  71. secator/tasks/gau.py +3 -1
  72. secator/tasks/gf.py +3 -3
  73. secator/tasks/gitleaks.py +2 -2
  74. secator/tasks/gospider.py +7 -1
  75. secator/tasks/grype.py +5 -4
  76. secator/tasks/h8mail.py +2 -1
  77. secator/tasks/httpx.py +18 -5
  78. secator/tasks/katana.py +35 -15
  79. secator/tasks/maigret.py +4 -4
  80. secator/tasks/mapcidr.py +3 -3
  81. secator/tasks/msfconsole.py +4 -4
  82. secator/tasks/naabu.py +2 -2
  83. secator/tasks/nmap.py +12 -14
  84. secator/tasks/nuclei.py +3 -3
  85. secator/tasks/searchsploit.py +4 -5
  86. secator/tasks/subfinder.py +2 -2
  87. secator/tasks/testssl.py +264 -263
  88. secator/tasks/trivy.py +5 -5
  89. secator/tasks/wafw00f.py +21 -3
  90. secator/tasks/wpprobe.py +90 -83
  91. secator/tasks/wpscan.py +6 -5
  92. secator/template.py +218 -104
  93. secator/thread.py +15 -15
  94. secator/tree.py +196 -0
  95. secator/utils.py +131 -123
  96. secator/utils_test.py +60 -19
  97. secator/workflows/__init__.py +2 -2
  98. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/METADATA +36 -36
  99. secator-0.16.0.dist-info/RECORD +132 -0
  100. secator/configs/profiles/default.yaml +0 -8
  101. secator/configs/workflows/url_nuclei.yaml +0 -11
  102. secator/tasks/dnsxbrute.py +0 -42
  103. secator-0.15.1.dist-info/RECORD +0 -128
  104. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/WHEEL +0 -0
  105. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/entry_points.txt +0 -0
  106. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/licenses/LICENSE +0 -0
secator/decorators.py CHANGED
@@ -1,260 +1,3 @@
1
- import sys
2
-
3
- from collections import OrderedDict
4
-
5
- import rich_click as click
6
- from rich_click.rich_click import _get_rich_console
7
- from rich_click.rich_group import RichGroup
8
-
9
- from secator.config import CONFIG
10
- from secator.definitions import ADDONS_ENABLED, OPT_NOT_SUPPORTED
11
- from secator.runners import Scan, Task, Workflow
12
- from secator.utils import (deduplicate, expand_input, get_command_category)
13
-
14
- RUNNER_OPTS = {
15
- 'output': {'type': str, 'default': None, 'help': 'Output options (-o table,json,csv,gdrive)', 'short': 'o'},
16
- 'profiles': {'type': str, 'default': 'default', 'help': 'Profiles', 'short': 'pf'},
17
- 'workspace': {'type': str, 'default': 'default', 'help': 'Workspace', 'short': 'ws'},
18
- 'print_json': {'is_flag': True, 'short': 'json', 'default': False, 'help': 'Print items as JSON lines'},
19
- 'print_raw': {'is_flag': True, 'short': 'raw', 'default': False, 'help': 'Print items in raw format'},
20
- 'print_stat': {'is_flag': True, 'short': 'stat', 'default': False, 'help': 'Print runtime statistics'},
21
- 'print_format': {'default': '', 'short': 'fmt', 'help': 'Output formatting string'},
22
- 'enable_profiler': {'is_flag': True, 'short': 'prof', 'default': False, 'help': 'Enable runner profiling'},
23
- 'no_process': {'is_flag': True, 'short': 'nps', 'default': False, 'help': 'Disable secator processing'},
24
- # 'filter': {'default': '', 'short': 'f', 'help': 'Results filter', 'short': 'of'}, # TODO add this
25
- 'quiet': {'is_flag': True, 'short': 'q', 'default': not CONFIG.runners.show_command_output, 'opposite': 'verbose', 'help': 'Enable quiet mode'}, # noqa: E501
26
- 'dry_run': {'is_flag': True, 'short': 'dr', 'default': False, 'help': 'Enable dry run'},
27
- 'show': {'is_flag': True, 'short': 'yml', 'default': False, 'help': 'Show runner yaml'},
28
- }
29
-
30
- RUNNER_GLOBAL_OPTS = {
31
- 'sync': {'is_flag': True, 'help': 'Run tasks synchronously (automatic if no worker is alive)'},
32
- 'worker': {'is_flag': True, 'default': False, 'help': 'Run tasks in worker'},
33
- 'no_poll': {'is_flag': True, 'short': 'np', 'default': False, 'help': 'Do not live poll for tasks results when running in worker'}, # noqa: E501
34
- 'proxy': {'type': str, 'help': 'HTTP proxy'},
35
- 'driver': {'type': str, 'help': 'Export real-time results. E.g: "mongodb"'}
36
- # 'debug': {'type': int, 'default': 0, 'help': 'Debug mode'},
37
- }
38
-
39
- DEFAULT_CLI_OPTIONS = list(RUNNER_OPTS.keys()) + list(RUNNER_GLOBAL_OPTS.keys())
40
-
41
-
42
- class OrderedGroup(RichGroup):
43
- def __init__(self, name=None, commands=None, **attrs):
44
- super(OrderedGroup, self).__init__(name, commands, **attrs)
45
- self.commands = commands or OrderedDict()
46
-
47
- def command(self, *args, **kwargs):
48
- """Behaves the same as `click.Group.command()` but supports aliases.
49
- """
50
- def decorator(f):
51
- aliases = kwargs.pop("aliases", None)
52
- if aliases:
53
- max_width = _get_rich_console().width
54
- aliases_str = ', '.join(f'[bold cyan]{alias}[/]' for alias in aliases)
55
- padding = max_width // 4
56
-
57
- name = kwargs.pop("name", None)
58
- if not name:
59
- raise click.UsageError("`name` command argument is required when using aliases.")
60
-
61
- f.__doc__ = f.__doc__ or '\0'.ljust(padding+1)
62
- f.__doc__ = f'{f.__doc__:<{padding}}[dim](aliases)[/] {aliases_str}'
63
- base_command = super(OrderedGroup, self).command(
64
- name, *args, **kwargs
65
- )(f)
66
- for alias in aliases:
67
- cmd = super(OrderedGroup, self).command(alias, *args, hidden=True, **kwargs)(f)
68
- cmd.help = f"Alias for '{name}'.\n\n{cmd.help}"
69
- cmd.params = base_command.params
70
-
71
- else:
72
- cmd = super(OrderedGroup, self).command(*args, **kwargs)(f)
73
-
74
- return cmd
75
- return decorator
76
-
77
- def group(self, *args, **kwargs):
78
- """Behaves the same as `click.Group.group()` but supports aliases.
79
- """
80
- def decorator(f):
81
- aliases = kwargs.pop('aliases', [])
82
- aliased_group = []
83
- if aliases:
84
- max_width = _get_rich_console().width
85
- aliases_str = ', '.join(f'[bold cyan]{alias}[/]' for alias in aliases)
86
- padding = max_width // 4
87
- f.__doc__ = f.__doc__ or '\0'.ljust(padding+1)
88
- f.__doc__ = f'{f.__doc__:<{padding}}[dim](aliases)[/] {aliases_str}'
89
- for alias in aliases:
90
- grp = super(OrderedGroup, self).group(
91
- alias, *args, hidden=True, **kwargs)(f)
92
- aliased_group.append(grp)
93
-
94
- # create the main group
95
- grp = super(OrderedGroup, self).group(*args, **kwargs)(f)
96
- grp.aliases = aliases
97
-
98
- # for all of the aliased groups, share the main group commands
99
- for aliased in aliased_group:
100
- aliased.commands = grp.commands
101
-
102
- return grp
103
- return decorator
104
-
105
- def list_commands(self, ctx):
106
- return self.commands
107
-
108
-
109
- def get_command_options(config):
110
- """Get unified list of command options from a list of secator tasks classes and optionally a Runner config.
111
-
112
- Args:
113
- config (TemplateLoader): Current runner config.
114
-
115
- Returns:
116
- list: List of deduplicated options.
117
- """
118
- from secator.utils import debug
119
- opt_cache = []
120
- all_opts = OrderedDict({})
121
- tasks = config.flat_tasks
122
- tasks_cls = set([c['class'] for c in tasks.values()])
123
-
124
- # Loop through tasks and set options
125
- for cls in tasks_cls:
126
- opts = OrderedDict(RUNNER_GLOBAL_OPTS, **RUNNER_OPTS, **cls.meta_opts, **cls.opts)
127
-
128
- # Find opts defined in config corresponding to this task class
129
- # TODO: rework this as this ignores subsequent tasks of the same task class
130
- task_config_opts = {}
131
- if config.type != 'task':
132
- for k, v in tasks.items():
133
- if v['class'] == cls:
134
- task_config_opts = v['opts']
135
-
136
- # Loop through options
137
- for opt, opt_conf in opts.items():
138
-
139
- # Get opt key map if any
140
- opt_key_map = getattr(cls, 'opt_key_map', {})
141
-
142
- # Opt is not supported by this task
143
- if opt not in opt_key_map\
144
- and opt not in cls.opts\
145
- and opt not in RUNNER_OPTS\
146
- and opt not in RUNNER_GLOBAL_OPTS:
147
- continue
148
-
149
- # Opt is defined as unsupported
150
- if opt_key_map.get(opt) == OPT_NOT_SUPPORTED:
151
- continue
152
-
153
- # Get opt prefix
154
- prefix = None
155
- if opt in cls.opts:
156
- prefix = cls.__name__
157
- elif opt in cls.meta_opts:
158
- # TODO: Add options categories
159
- # category = get_command_category(cls)
160
- # prefix = category
161
- prefix = 'Meta'
162
- elif opt in RUNNER_OPTS:
163
- prefix = 'Output'
164
- elif opt in RUNNER_GLOBAL_OPTS:
165
- prefix = 'Execution'
166
-
167
- # Get opt value from YAML config
168
- opt_conf_value = task_config_opts.get(opt)
169
-
170
- # Get opt conf
171
- conf = opt_conf.copy()
172
- opt_is_flag = conf.get('is_flag', False)
173
- opt_default = conf.get('default', False if opt_is_flag else None)
174
- opt_is_required = conf.get('required', False)
175
- conf['show_default'] = True
176
- conf['prefix'] = prefix
177
- conf['default'] = opt_default
178
- conf['reverse'] = False
179
-
180
- # Change CLI opt defaults if opt was overriden in YAML config
181
- if opt_conf_value:
182
- if opt_is_required:
183
- debug('OPT (skipped: opt is required and defined in config)', obj={'opt': opt}, sub=f'cli.{config.name}', verbose=True) # noqa: E501
184
- continue
185
- mapped_value = cls.opt_value_map.get(opt)
186
- if callable(mapped_value):
187
- opt_conf_value = mapped_value(opt_conf_value)
188
- elif mapped_value:
189
- opt_conf_value = mapped_value
190
-
191
- # Handle option defaults
192
- if opt_conf_value != opt_default:
193
- if opt in opt_cache:
194
- continue
195
- if opt_is_flag:
196
- conf['default'] = opt_default = opt_conf_value
197
-
198
- # Add reverse flag
199
- if opt_default is True:
200
- conf['reverse'] = True
201
-
202
- # Check if opt already processed before
203
- if opt in opt_cache:
204
- # debug('OPT (skipped: opt is already in opt cache)', obj={'opt': opt}, sub=f'cli.{config.name}', verbose=True)
205
- continue
206
-
207
- # Build help
208
- opt_cache.append(opt)
209
- opt = opt.replace('_', '-')
210
- all_opts[opt] = conf
211
-
212
- # Debug
213
- debug_conf = OrderedDict({'opt': opt, 'config_val': opt_conf_value or 'N/A', **conf.copy()})
214
- debug('OPT', obj=debug_conf, sub=f'cli.{config.name}', verbose=True)
215
-
216
- return all_opts
217
-
218
-
219
- def decorate_command_options(opts):
220
- """Add click.option decorator to decorate click command.
221
-
222
- Args:
223
- opts (dict): Dict of command options.
224
-
225
- Returns:
226
- function: Decorator.
227
- """
228
- def decorator(f):
229
- reversed_opts = OrderedDict(list(opts.items())[::-1])
230
- for opt_name, opt_conf in reversed_opts.items():
231
- conf = opt_conf.copy()
232
- short_opt = conf.pop('short', None)
233
- internal = conf.pop('internal', False)
234
- display = conf.pop('display', True)
235
- if internal and not display:
236
- continue
237
- conf.pop('prefix', None)
238
- conf.pop('shlex', None)
239
- conf.pop('meta', None)
240
- conf.pop('supported', None)
241
- conf.pop('process', None)
242
- conf.pop('requires_sudo', None)
243
- reverse = conf.pop('reverse', False)
244
- opposite = conf.pop('opposite', None)
245
- long = f'--{opt_name}'
246
- short = f'-{short_opt}' if short_opt else f'-{opt_name}'
247
- if reverse:
248
- if opposite:
249
- long += f'/--{opposite}'
250
- short += f'/-{opposite[0]}'
251
- conf['help'] = conf['help'].replace(opt_name, f'{opt_name} / {opposite}')
252
- else:
253
- long += f'/--no-{opt_name}'
254
- short += f'/-n{short_opt}' if short else f'/-n{opt_name}'
255
- f = click.option(long, short, **conf)(f)
256
- return f
257
- return decorator
258
1
 
259
2
 
260
3
  def task():
@@ -262,200 +5,3 @@ def task():
262
5
  cls.__task__ = True
263
6
  return cls
264
7
  return decorator
265
-
266
-
267
- def generate_cli_subcommand(cli_endpoint, func, **opts):
268
- return cli_endpoint.command(**opts)(func)
269
-
270
-
271
- def register_runner(cli_endpoint, config):
272
- name = config.name
273
- input_required = True
274
- command_opts = {
275
- 'no_args_is_help': True,
276
- 'context_settings': {
277
- 'ignore_unknown_options': False,
278
- 'allow_extra_args': False
279
- }
280
- }
281
-
282
- if cli_endpoint.name == 'scan':
283
- runner_cls = Scan
284
- input_required = False # allow targets from stdin
285
- short_help = config.description or ''
286
- short_help += f' [dim]alias: {config.alias}' if config.alias else ''
287
- command_opts.update({
288
- 'name': name,
289
- 'short_help': short_help,
290
- 'no_args_is_help': False
291
- })
292
- input_types = config.input_types
293
-
294
- elif cli_endpoint.name == 'workflow':
295
- runner_cls = Workflow
296
- input_required = False # allow targets from stdin
297
- short_help = config.description or ''
298
- short_help = f'{short_help:<55} [dim](alias)[/][bold cyan] {config.alias}' if config.alias else ''
299
- command_opts.update({
300
- 'name': name,
301
- 'short_help': short_help,
302
- 'no_args_is_help': False
303
- })
304
- input_types = config.input_types
305
-
306
- elif cli_endpoint.name == 'task':
307
- runner_cls = Task
308
- input_required = False # allow targets from stdin
309
- task_cls = Task.get_task_class(config.name)
310
- task_category = get_command_category(task_cls)
311
- short_help = f'[magenta]{task_category:<25}[/] {task_cls.__doc__}'
312
- command_opts.update({
313
- 'name': name,
314
- 'short_help': short_help,
315
- 'no_args_is_help': False
316
- })
317
- input_types = task_cls.input_types
318
-
319
- else:
320
- raise ValueError(f"Unrecognized runner endpoint name {cli_endpoint.name}")
321
- input_types_str = '|'.join(input_types) if input_types else 'targets'
322
- options = get_command_options(config)
323
-
324
- # TODO: maybe allow this in the future
325
- # def get_unknown_opts(ctx):
326
- # return {
327
- # (ctx.args[i][2:]
328
- # if str(ctx.args[i]).startswith("--") \
329
- # else ctx.args[i][1:]): ctx.args[i+1]
330
- # for i in range(0, len(ctx.args), 2)
331
- # }
332
-
333
- @click.argument(input_types_str, required=input_required)
334
- @decorate_command_options(options)
335
- @click.pass_context
336
- def func(ctx, **opts):
337
- sync = opts['sync']
338
- worker = opts.pop('worker')
339
- ws = opts.pop('workspace')
340
- driver = opts.pop('driver', '')
341
- quiet = opts['quiet']
342
- dry_run = opts['dry_run']
343
- show = opts['show']
344
- context = {'workspace_name': ws}
345
-
346
- # Show runner yaml
347
- if show:
348
- config.print()
349
- sys.exit(0)
350
-
351
- # Remove options whose values are default values
352
- for k, v in options.items():
353
- opt_name = k.replace('-', '_')
354
- if opt_name in opts and opts[opt_name] == v.get('default', None):
355
- del opts[opt_name]
356
-
357
- # TODO: maybe allow this in the future
358
- # unknown_opts = get_unknown_opts(ctx)
359
- # opts.update(unknown_opts)
360
-
361
- # Expand input
362
- inputs = opts.pop(input_types_str)
363
- inputs = expand_input(inputs, ctx)
364
-
365
- # Build hooks from driver name
366
- hooks = []
367
- drivers = driver.split(',') if driver else []
368
- console = _get_rich_console()
369
- supported_drivers = ['mongodb', 'gcs']
370
- for driver in drivers:
371
- if driver in supported_drivers:
372
- if not ADDONS_ENABLED[driver]:
373
- console.print(f'[bold red]Missing "{driver}" addon: please run `secator install addons {driver}`[/].')
374
- sys.exit(1)
375
- from secator.utils import import_dynamic
376
- driver_hooks = import_dynamic(f'secator.hooks.{driver}', 'HOOKS')
377
- if driver_hooks is None:
378
- console.print(f'[bold red]Missing "secator.hooks.{driver}.HOOKS".[/]')
379
- sys.exit(1)
380
- hooks.append(driver_hooks)
381
- else:
382
- supported_drivers_str = ', '.join([f'[bold green]{_}[/]' for _ in supported_drivers])
383
- console.print(f'[bold red]Driver "{driver}" is not supported.[/]')
384
- console.print(f'Supported drivers: {supported_drivers_str}')
385
- sys.exit(1)
386
-
387
- from secator.utils import deep_merge_dicts
388
- hooks = deep_merge_dicts(*hooks)
389
-
390
- # Enable sync or not
391
- if sync or dry_run:
392
- sync = True
393
- else:
394
- from secator.celery import is_celery_worker_alive
395
- worker_alive = is_celery_worker_alive()
396
- if not worker_alive and not worker:
397
- sync = True
398
- else:
399
- sync = False
400
- broker_protocol = CONFIG.celery.broker_url.split('://')[0]
401
- backend_protocol = CONFIG.celery.result_backend.split('://')[0]
402
- if CONFIG.celery.broker_url:
403
- if (broker_protocol == 'redis' or backend_protocol == 'redis') and not ADDONS_ENABLED['redis']:
404
- _get_rich_console().print('[bold red]Missing `redis` addon: please run `secator install addons redis`[/].')
405
- sys.exit(1)
406
-
407
- from secator.utils import debug
408
- debug('Run options', obj=opts, sub='cli')
409
-
410
- # Set run options
411
- opts.update({
412
- 'print_cmd': True,
413
- 'print_item': True,
414
- 'print_line': True,
415
- 'print_progress': True,
416
- 'print_remote_info': not sync,
417
- 'piped_input': ctx.obj['piped_input'],
418
- 'piped_output': ctx.obj['piped_output'],
419
- 'caller': 'cli',
420
- 'sync': sync,
421
- 'quiet': quiet
422
- })
423
-
424
- # Start runner
425
- runner = runner_cls(config, inputs, run_opts=opts, hooks=hooks, context=context)
426
- runner.run()
427
-
428
- generate_cli_subcommand(cli_endpoint, func, **command_opts)
429
- generate_rich_click_opt_groups(cli_endpoint, name, input_types, options)
430
-
431
-
432
- def generate_rich_click_opt_groups(cli_endpoint, name, input_types, options):
433
- sortorder = {
434
- 'Execution': 0,
435
- 'Output': 1,
436
- 'Meta': 2,
437
- }
438
- prefixes = deduplicate([opt['prefix'] for opt in options.values()])
439
- prefixes = sorted(prefixes, key=lambda x: sortorder.get(x, 3))
440
- opt_group = [
441
- {
442
- 'name': 'Targets',
443
- 'options': input_types,
444
- },
445
- ]
446
- for prefix in prefixes:
447
- prefix_opts = [
448
- opt for opt, conf in options.items()
449
- if conf['prefix'] == prefix
450
- ]
451
- opt_names = [f'--{opt_name}' for opt_name in prefix_opts]
452
- if prefix == 'Execution':
453
- opt_names.append('--help')
454
- opt_group.append({
455
- 'name': prefix + ' options',
456
- 'options': opt_names
457
- })
458
- aliases = [cli_endpoint.name, *cli_endpoint.aliases]
459
- for alias in aliases:
460
- endpoint_name = f'secator {alias} {name}'
461
- click.rich_click.OPTION_GROUPS[endpoint_name] = opt_group
secator/definitions.py CHANGED
@@ -20,8 +20,7 @@ ASCII = rf"""
20
20
  """ # noqa: W605,W291
21
21
 
22
22
  # Debug
23
- DEBUG = CONFIG.debug.level
24
- DEBUG_COMPONENT = CONFIG.debug.component.split(',')
23
+ DEBUG = CONFIG.debug.split(',')
25
24
 
26
25
  # Constants
27
26
  OPT_NOT_SUPPORTED = -1
@@ -39,78 +38,98 @@ ALIVE = 'alive'
39
38
  AUTO_CALIBRATION = 'auto_calibration'
40
39
  CONTENT_TYPE = 'content_type'
41
40
  CONTENT_LENGTH = 'content_length'
41
+ CERTIFICATE_STATUS_UNKNOWN = 'Unknown'
42
+ CERTIFICATE_STATUS_REVOKED = 'Revoked'
43
+ CERTIFICATE_STATUS_TRUSTED = 'Trusted'
42
44
  CIDR_RANGE = 'cidr_range'
43
- DOCKER_IMAGE = 'docker_image'
44
- FILENAME = 'filename'
45
- GIT_REPOSITORY = 'git_repository'
45
+ CONFIDENCE = 'confidence'
46
46
  CPES = 'cpes'
47
47
  CVES = 'cves'
48
+ CVSS_SCORE = 'cvss_score'
49
+ DATA = 'data'
48
50
  DELAY = 'delay'
51
+ DESCRIPTION = 'description'
52
+ DOCKER_IMAGE = 'docker_image'
49
53
  DOMAIN = 'domain'
50
54
  DEPTH = 'depth'
51
55
  EXTRA_DATA = 'extra_data'
52
56
  EMAIL = 'email'
57
+ FILENAME = 'filename'
53
58
  FILTER_CODES = 'filter_codes'
54
59
  FILTER_WORDS = 'filter_words'
55
60
  FOLLOW_REDIRECT = 'follow_redirect'
56
61
  FILTER_REGEX = 'filter_regex'
57
62
  FILTER_SIZE = 'filter_size'
63
+ GIT_REPOSITORY = 'git_repository'
58
64
  HEADER = 'header'
59
65
  HOST = 'host'
66
+ HOST_PORT = 'host:port'
67
+ IBAN = 'iban'
68
+ ID = 'id'
60
69
  IP = 'ip'
61
70
  PROTOCOL = 'protocol'
62
71
  LINES = 'lines'
63
72
  METHOD = 'method'
73
+ MAC_ADDRESS = 'mac'
74
+ MATCHED_AT = 'matched_at'
64
75
  MATCH_CODES = 'match_codes'
65
76
  MATCH_REGEX = 'match_regex'
66
77
  MATCH_SIZE = 'match_size'
67
78
  MATCH_WORDS = 'match_words'
79
+ NAME = 'name'
68
80
  ORG_NAME = 'org_name'
69
81
  OUTPUT_PATH = 'output_path'
70
82
  PATH = 'path'
71
83
  PERCENT = 'percent'
72
84
  PORTS = 'ports'
73
85
  PORT = 'port'
86
+ PROVIDER = 'provider'
74
87
  PROXY = 'proxy'
75
88
  RATE_LIMIT = 'rate_limit'
89
+ REFERENCE = 'reference'
90
+ REFERENCES = 'references'
76
91
  RETRIES = 'retries'
92
+ SCRIPT = 'script'
93
+ SERVICE_NAME = 'service_name'
94
+ SEVERITY = 'severity'
95
+ SITE_NAME = 'site_name'
96
+ SLUG = 'slug'
97
+ SOURCES = 'sources'
98
+ STORED_RESPONSE_PATH = 'stored_response_path'
99
+ STATE = 'state'
100
+ STATUS_CODE = 'status_code'
101
+ STRING = 'str'
77
102
  TAGS = 'tags'
103
+ TECH = 'tech'
104
+ TECHNOLOGY = 'technology'
78
105
  THREADS = 'threads'
79
106
  TIME = 'time'
80
107
  TIMEOUT = 'timeout'
108
+ TITLE = 'title'
81
109
  TOP_PORTS = 'top_ports'
82
110
  TYPE = 'type'
83
111
  URL = 'url'
84
112
  USER_AGENT = 'user_agent'
85
113
  USERNAME = 'username'
86
- STORED_RESPONSE_PATH = 'stored_response_path'
87
- SCRIPT = 'script'
88
- SERVICE_NAME = 'service_name'
89
- SOURCES = 'sources'
90
- STATE = 'state'
91
- STATUS_CODE = 'status_code'
92
- TECH = 'tech'
93
- TITLE = 'title'
94
- SITE_NAME = 'site_name'
95
- SERVICE_NAME = 'service_name'
96
- CONFIDENCE = 'confidence'
97
- CVSS_SCORE = 'cvss_score'
98
- DESCRIPTION = 'description'
99
- ID = 'id'
100
- MATCHED_AT = 'matched_at'
101
- NAME = 'name'
102
- PROVIDER = 'provider'
103
- REFERENCE = 'reference'
104
- REFERENCES = 'references'
105
- SEVERITY = 'severity'
106
- TAGS = 'tags'
107
- TECHNOLOGY = 'technology'
114
+ UUID = 'uuid'
108
115
  WEBSERVER = 'webserver'
109
116
  WORDLIST = 'wordlist'
110
117
  WORDS = 'words'
111
- CERTIFICATE_STATUS_UNKNOWN = 'Unknown'
112
- CERTIFICATE_STATUS_TRUSTED = 'Trusted'
113
- CERTIFICATE_STATUS_REVOKED = 'Revoked'
118
+
119
+ # Allowed input types
120
+ INPUT_TYPES = [
121
+ URL,
122
+ IP,
123
+ CIDR_RANGE,
124
+ HOST,
125
+ MAC_ADDRESS,
126
+ EMAIL,
127
+ IBAN,
128
+ UUID,
129
+ PATH,
130
+ SLUG,
131
+ STRING,
132
+ ]
114
133
 
115
134
 
116
135
  def is_importable(module_to_import):
@@ -1,3 +1,3 @@
1
1
  class Exporter:
2
- def __init__(self, report):
3
- self.report = report
2
+ def __init__(self, report):
3
+ self.report = report
@@ -1,5 +1,5 @@
1
1
  from secator.exporters._base import Exporter
2
- from secator.rich import console
2
+ from secator.rich import console_stdout
3
3
 
4
4
 
5
5
  class ConsoleExporter(Exporter):
@@ -7,4 +7,4 @@ class ConsoleExporter(Exporter):
7
7
  results = self.report.data['results']
8
8
  for items in results.values():
9
9
  for item in items:
10
- console.print(item)
10
+ console_stdout.print(item)
@@ -1,8 +1,9 @@
1
- from secator.exporters._base import Exporter
2
- from secator.utils import pluralize
3
- from secator.rich import build_table, console
4
1
  from rich.markdown import Markdown
2
+
3
+ from secator.exporters._base import Exporter
5
4
  from secator.output_types import OutputType
5
+ from secator.rich import build_table, console
6
+ from secator.utils import pluralize
6
7
 
7
8
 
8
9
  class TableExporter(Exporter):
secator/exporters/txt.py CHANGED
@@ -11,7 +11,7 @@ class TxtExporter(Exporter):
11
11
  txt_paths = []
12
12
 
13
13
  for output_type, items in results.items():
14
- items = [str(i) for i in items]
14
+ items = list(set(str(i) for i in items))
15
15
  if not items:
16
16
  continue
17
17
  txt_path = f'{self.report.output_folder}/report_{output_type}.txt'
secator/hooks/mongodb.py CHANGED
@@ -147,13 +147,11 @@ def tag_duplicates(ws_id: str = None):
147
147
  workspace_query = list(
148
148
  db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': True}).sort('_timestamp', -1))
149
149
  untagged_query = list(
150
- db.findings.find({'_context.workspace_id': str(ws_id)}).sort('_timestamp', -1))
151
- # TODO: use this instead when duplicate removal logic is final
152
- # untagged_query = list(
153
- # db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': False}).sort('_timestamp', -1))
150
+ db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': {'$ne': True}}).sort('_timestamp', -1))
154
151
  if not untagged_query:
155
152
  debug('no untagged findings. Skipping.', id=ws_id, sub='hooks.mongodb')
156
153
  return
154
+ debug(f'found {len(untagged_query)} untagged findings', id=ws_id, sub='hooks.mongodb')
157
155
 
158
156
  untagged_findings = load_findings(untagged_query)
159
157
  workspace_findings = load_findings(workspace_query)