secator 0.15.1__py3-none-any.whl → 0.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of secator might be problematic. Click here for more details.

Files changed (106) hide show
  1. secator/celery.py +40 -24
  2. secator/celery_signals.py +71 -68
  3. secator/celery_utils.py +43 -27
  4. secator/cli.py +520 -280
  5. secator/cli_helper.py +394 -0
  6. secator/click.py +87 -0
  7. secator/config.py +67 -39
  8. secator/configs/profiles/http_headless.yaml +6 -0
  9. secator/configs/profiles/http_record.yaml +6 -0
  10. secator/configs/profiles/tor.yaml +1 -1
  11. secator/configs/scans/domain.yaml +4 -2
  12. secator/configs/scans/host.yaml +1 -1
  13. secator/configs/scans/network.yaml +1 -4
  14. secator/configs/scans/subdomain.yaml +13 -1
  15. secator/configs/scans/url.yaml +1 -2
  16. secator/configs/workflows/cidr_recon.yaml +6 -4
  17. secator/configs/workflows/code_scan.yaml +1 -1
  18. secator/configs/workflows/host_recon.yaml +29 -3
  19. secator/configs/workflows/subdomain_recon.yaml +67 -16
  20. secator/configs/workflows/url_crawl.yaml +44 -15
  21. secator/configs/workflows/url_dirsearch.yaml +4 -4
  22. secator/configs/workflows/url_fuzz.yaml +25 -17
  23. secator/configs/workflows/url_params_fuzz.yaml +7 -0
  24. secator/configs/workflows/url_vuln.yaml +33 -8
  25. secator/configs/workflows/user_hunt.yaml +2 -1
  26. secator/configs/workflows/wordpress.yaml +5 -3
  27. secator/cve.py +718 -0
  28. secator/decorators.py +0 -454
  29. secator/definitions.py +49 -30
  30. secator/exporters/_base.py +2 -2
  31. secator/exporters/console.py +2 -2
  32. secator/exporters/table.py +4 -3
  33. secator/exporters/txt.py +1 -1
  34. secator/hooks/mongodb.py +2 -4
  35. secator/installer.py +77 -49
  36. secator/loader.py +116 -0
  37. secator/output_types/_base.py +3 -0
  38. secator/output_types/certificate.py +63 -63
  39. secator/output_types/error.py +4 -5
  40. secator/output_types/info.py +2 -2
  41. secator/output_types/ip.py +3 -1
  42. secator/output_types/progress.py +5 -9
  43. secator/output_types/state.py +17 -17
  44. secator/output_types/tag.py +3 -0
  45. secator/output_types/target.py +10 -2
  46. secator/output_types/url.py +19 -7
  47. secator/output_types/vulnerability.py +11 -7
  48. secator/output_types/warning.py +2 -2
  49. secator/report.py +27 -15
  50. secator/rich.py +18 -10
  51. secator/runners/_base.py +446 -233
  52. secator/runners/_helpers.py +133 -24
  53. secator/runners/command.py +182 -102
  54. secator/runners/scan.py +33 -5
  55. secator/runners/task.py +13 -7
  56. secator/runners/workflow.py +105 -72
  57. secator/scans/__init__.py +2 -2
  58. secator/serializers/dataclass.py +20 -20
  59. secator/tasks/__init__.py +4 -4
  60. secator/tasks/_categories.py +39 -27
  61. secator/tasks/arjun.py +9 -5
  62. secator/tasks/bbot.py +53 -21
  63. secator/tasks/bup.py +19 -5
  64. secator/tasks/cariddi.py +24 -3
  65. secator/tasks/dalfox.py +26 -7
  66. secator/tasks/dirsearch.py +10 -4
  67. secator/tasks/dnsx.py +70 -25
  68. secator/tasks/feroxbuster.py +11 -3
  69. secator/tasks/ffuf.py +42 -6
  70. secator/tasks/fping.py +20 -8
  71. secator/tasks/gau.py +3 -1
  72. secator/tasks/gf.py +3 -3
  73. secator/tasks/gitleaks.py +2 -2
  74. secator/tasks/gospider.py +7 -1
  75. secator/tasks/grype.py +5 -4
  76. secator/tasks/h8mail.py +2 -1
  77. secator/tasks/httpx.py +18 -5
  78. secator/tasks/katana.py +35 -15
  79. secator/tasks/maigret.py +4 -4
  80. secator/tasks/mapcidr.py +3 -3
  81. secator/tasks/msfconsole.py +4 -4
  82. secator/tasks/naabu.py +2 -2
  83. secator/tasks/nmap.py +12 -14
  84. secator/tasks/nuclei.py +3 -3
  85. secator/tasks/searchsploit.py +4 -5
  86. secator/tasks/subfinder.py +2 -2
  87. secator/tasks/testssl.py +264 -263
  88. secator/tasks/trivy.py +5 -5
  89. secator/tasks/wafw00f.py +21 -3
  90. secator/tasks/wpprobe.py +90 -83
  91. secator/tasks/wpscan.py +6 -5
  92. secator/template.py +218 -104
  93. secator/thread.py +15 -15
  94. secator/tree.py +196 -0
  95. secator/utils.py +131 -123
  96. secator/utils_test.py +60 -19
  97. secator/workflows/__init__.py +2 -2
  98. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/METADATA +36 -36
  99. secator-0.16.0.dist-info/RECORD +132 -0
  100. secator/configs/profiles/default.yaml +0 -8
  101. secator/configs/workflows/url_nuclei.yaml +0 -11
  102. secator/tasks/dnsxbrute.py +0 -42
  103. secator-0.15.1.dist-info/RECORD +0 -128
  104. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/WHEEL +0 -0
  105. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/entry_points.txt +0 -0
  106. {secator-0.15.1.dist-info → secator-0.16.0.dist-info}/licenses/LICENSE +0 -0
secator/cli.py CHANGED
@@ -16,8 +16,9 @@ from rich.markdown import Markdown
16
16
  from rich.rule import Rule
17
17
  from rich.table import Table
18
18
 
19
- from secator.config import CONFIG, ROOT_FOLDER, Config, default_config, config_path
20
- from secator.decorators import OrderedGroup, register_runner
19
+ from secator.config import CONFIG, ROOT_FOLDER, Config, default_config, config_path, download_files
20
+ from secator.click import OrderedGroup
21
+ from secator.cli_helper import register_runner
21
22
  from secator.definitions import ADDONS_ENABLED, ASCII, DEV_PACKAGE, VERSION, STATE_COLORS
22
23
  from secator.installer import ToolInstaller, fmt_health_table_row, get_health_table, get_version_info, get_distro_config
23
24
  from secator.output_types import FINDING_TYPES, Info, Warning, Error
@@ -25,20 +26,23 @@ from secator.report import Report
25
26
  from secator.rich import console
26
27
  from secator.runners import Command, Runner
27
28
  from secator.serializers.dataclass import loads_dataclass
28
- from secator.template import TEMPLATES, TemplateLoader
29
+ from secator.loader import get_configs_by_type, discover_tasks
29
30
  from secator.utils import (
30
- debug, detect_host, discover_tasks, flatten, print_version, get_file_date,
31
+ debug, detect_host, flatten, print_version, get_file_date,
31
32
  sort_files_by_date, get_file_timestamp, list_reports, get_info_from_report_path, human_to_timedelta
32
33
  )
33
-
34
+ from contextlib import nullcontext
34
35
  click.rich_click.USE_RICH_MARKUP = True
36
+ click.rich_click.STYLE_ARGUMENT = ""
37
+ click.rich_click.STYLE_OPTION_HELP = ""
38
+
35
39
 
36
- ALL_TASKS = discover_tasks()
37
- ALL_WORKFLOWS = [t for t in TEMPLATES if t.type == 'workflow']
38
- ALL_SCANS = [t for t in TEMPLATES if t.type == 'scan']
39
- ALL_PROFILES = [t for t in TEMPLATES if t.type == 'profile']
40
40
  FINDING_TYPES_LOWER = [c.__name__.lower() for c in FINDING_TYPES]
41
41
  CONTEXT_SETTINGS = dict(help_option_names=['-h', '-help', '--help'])
42
+ TASKS = get_configs_by_type('task')
43
+ WORKFLOWS = get_configs_by_type('workflow')
44
+ SCANS = get_configs_by_type('scan')
45
+ PROFILES = get_configs_by_type('profile')
42
46
 
43
47
 
44
48
  #-----#
@@ -69,20 +73,15 @@ def cli(ctx, version, quiet):
69
73
  # TASK #
70
74
  #------#
71
75
 
72
- @cli.group(aliases=['x', 't'], invoke_without_command=True)
73
- @click.option('--list', '-list', is_flag=True, default=False)
76
+ @cli.group(aliases=['x', 't', 'tasks'], invoke_without_command=True)
74
77
  @click.pass_context
75
- def task(ctx, list=False):
78
+ def task(ctx):
76
79
  """Run a task."""
77
- if list:
78
- print("\n".join(sorted([t.__name__ for t in ALL_TASKS])))
79
- return
80
80
  if ctx.invoked_subcommand is None:
81
81
  ctx.get_help()
82
82
 
83
83
 
84
- for cls in ALL_TASKS:
85
- config = TemplateLoader(input={'name': cls.__name__, 'type': 'task'})
84
+ for config in TASKS:
86
85
  register_runner(task, config)
87
86
 
88
87
  #----------#
@@ -90,19 +89,15 @@ for cls in ALL_TASKS:
90
89
  #----------#
91
90
 
92
91
 
93
- @cli.group(cls=OrderedGroup, aliases=['w'], invoke_without_command=True)
94
- @click.option('--list', '-list', is_flag=True, default=False)
92
+ @cli.group(cls=OrderedGroup, aliases=['w', 'workflows'], invoke_without_command=True)
95
93
  @click.pass_context
96
- def workflow(ctx, list=False):
94
+ def workflow(ctx):
97
95
  """Run a workflow."""
98
- if list:
99
- print("\n".join(sorted([t.name for t in ALL_WORKFLOWS])))
100
- return
101
96
  if ctx.invoked_subcommand is None:
102
97
  ctx.get_help()
103
98
 
104
99
 
105
- for config in sorted(ALL_WORKFLOWS, key=lambda x: x['name']):
100
+ for config in WORKFLOWS:
106
101
  register_runner(workflow, config)
107
102
 
108
103
 
@@ -110,41 +105,18 @@ for config in sorted(ALL_WORKFLOWS, key=lambda x: x['name']):
110
105
  # SCAN #
111
106
  #------#
112
107
 
113
- @cli.group(cls=OrderedGroup, aliases=['s'], invoke_without_command=True)
114
- @click.option('--list', '-list', is_flag=True, default=False)
108
+ @cli.group(cls=OrderedGroup, aliases=['s', 'scans'], invoke_without_command=True)
115
109
  @click.pass_context
116
- def scan(ctx, list=False):
110
+ def scan(ctx):
117
111
  """Run a scan."""
118
- if list:
119
- print("\n".join(sorted([t.name for t in ALL_SCANS])))
120
- return
121
112
  if ctx.invoked_subcommand is None:
122
113
  ctx.get_help()
123
114
 
124
115
 
125
- for config in sorted(ALL_SCANS, key=lambda x: x['name']):
116
+ for config in SCANS:
126
117
  register_runner(scan, config)
127
118
 
128
119
 
129
- @cli.group(aliases=['p'])
130
- @click.pass_context
131
- def profile(ctx):
132
- """Show profiles"""
133
- pass
134
-
135
-
136
- @profile.command('list')
137
- def profile_list():
138
- table = Table()
139
- table.add_column("Profile name", style="bold gold3")
140
- table.add_column("Description", overflow='fold')
141
- table.add_column("Options", overflow='fold')
142
- for profile in ALL_PROFILES:
143
- opts_str = ','.join(f'{k}={v}' for k, v in profile.opts.items())
144
- table.add_row(profile.name, profile.description, opts_str)
145
- console.print(table)
146
-
147
-
148
120
  #--------#
149
121
  # WORKER #
150
122
  #--------#
@@ -161,7 +133,8 @@ def profile_list():
161
133
  @click.option('--dev', is_flag=True, help='Start a worker in dev mode (celery multi).')
162
134
  @click.option('--stop', is_flag=True, help='Stop a worker in dev mode (celery multi).')
163
135
  @click.option('--show', is_flag=True, help='Show command (celery multi).')
164
- def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, dev, stop, show):
136
+ @click.option('--use-command-runner', is_flag=True, default=False, help='Use command runner to run the command.')
137
+ def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, dev, stop, show, use_command_runner):
165
138
  """Run a worker."""
166
139
 
167
140
  # Check Celery addon is installed
@@ -172,10 +145,11 @@ def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, d
172
145
  # Check broken / backend addon is installed
173
146
  broker_protocol = CONFIG.celery.broker_url.split('://')[0]
174
147
  backend_protocol = CONFIG.celery.result_backend.split('://')[0]
175
- if CONFIG.celery.broker_url:
176
- if (broker_protocol == 'redis' or backend_protocol == 'redis') and not ADDONS_ENABLED['redis']:
177
- console.print(Error(message='Missing redis addon: please run "secator install addons redis".'))
178
- sys.exit(1)
148
+ if CONFIG.celery.broker_url and \
149
+ (broker_protocol == 'redis' or backend_protocol == 'redis') and \
150
+ not ADDONS_ENABLED['redis']:
151
+ console.print(Error(message='Missing redis addon: please run "secator install addons redis".'))
152
+ sys.exit(1)
179
153
 
180
154
  # Debug Celery config
181
155
  from secator.celery import app, is_celery_worker_alive
@@ -213,8 +187,13 @@ def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, d
213
187
  patterns = "celery.py;tasks/*.py;runners/*.py;serializers/*.py;output_types/*.py;hooks/*.py;exporters/*.py"
214
188
  cmd = f'watchmedo auto-restart --directory=./ --patterns="{patterns}" --recursive -- {cmd}'
215
189
 
216
- ret = Command.execute(cmd, name='secator_worker')
217
- sys.exit(ret.return_code)
190
+ if use_command_runner:
191
+ ret = Command.execute(cmd, name='secator_worker')
192
+ sys.exit(ret.return_code)
193
+ else:
194
+ console.print(f'[bold red]{cmd}[/]')
195
+ ret = os.system(cmd)
196
+ sys.exit(os.waitstatus_to_exitcode(ret))
218
197
 
219
198
 
220
199
  #-------#
@@ -235,7 +214,7 @@ def proxy(timeout, number):
235
214
  """Get random proxies from FreeProxy."""
236
215
  if CONFIG.offline_mode:
237
216
  console.print(Error(message='Cannot run this command in offline mode.'))
238
- return
217
+ sys.exit(1)
239
218
  proxy = FreeProxy(timeout=timeout, rand=True, anonym=True)
240
219
  for _ in range(number):
241
220
  url = proxy.get()
@@ -255,7 +234,7 @@ def revshell(name, host, port, interface, listen, force):
255
234
  host = detect_host(interface)
256
235
  if not host:
257
236
  console.print(Error(message=f'Interface "{interface}" could not be found. Run "ifconfig" to see the list of available interfaces')) # noqa: E501
258
- return
237
+ sys.exit(1)
259
238
  else:
260
239
  console.print(Info(message=f'Detected host IP: {host}'))
261
240
 
@@ -264,7 +243,7 @@ def revshell(name, host, port, interface, listen, force):
264
243
  if not os.path.exists(revshells_json) or force:
265
244
  if CONFIG.offline_mode:
266
245
  console.print(Error(message='Cannot run this command in offline mode'))
267
- return
246
+ sys.exit(1)
268
247
  ret = Command.execute(
269
248
  f'wget https://raw.githubusercontent.com/freelabz/secator/main/scripts/revshells.json && mv revshells.json {CONFIG.dirs.revshells}', # noqa: E501
270
249
  cls_attributes={'shell': True}
@@ -330,9 +309,16 @@ def revshell(name, host, port, interface, listen, force):
330
309
  @click.option('--interface', '-i', type=str, default=None, help='Interface to use to auto-detect host IP')
331
310
  def serve(directory, host, port, interface):
332
311
  """Run HTTP server to serve payloads."""
312
+ fnames = list(os.listdir(directory))
313
+ if not fnames:
314
+ console.print(Warning(message=f'No payloads found in {directory}.'))
315
+ download_files(CONFIG.payloads.templates, CONFIG.dirs.payloads, CONFIG.offline_mode, 'payload')
316
+ fnames = list(os.listdir(directory))
317
+
333
318
  console.print(Rule())
334
319
  console.print(f'Available payloads in {directory}: ', style='bold yellow')
335
- for fname in os.listdir(directory):
320
+ fnames.sort()
321
+ for fname in fnames:
336
322
  if not host:
337
323
  host = detect_host(interface)
338
324
  if not host:
@@ -342,7 +328,7 @@ def serve(directory, host, port, interface):
342
328
  console.print(f'wget http://{host}:{port}/{fname}', style='dim italic')
343
329
  console.print('')
344
330
  console.print(Rule())
345
- console.print(f'Started HTTP server on port {port}, waiting for incoming connections ...', style='bold yellow')
331
+ console.print(Info(message=f'[bold yellow]Started HTTP server on port {port}, waiting for incoming connections ...[/]')) # noqa: E501
346
332
  Command.execute(f'{sys.executable} -m http.server {port}', cwd=directory)
347
333
 
348
334
 
@@ -491,12 +477,12 @@ def config():
491
477
 
492
478
 
493
479
  @config.command('get')
494
- @click.option('--full', is_flag=True, help='Show full config (with defaults)')
480
+ @click.option('--user/--full', is_flag=True, help='Show config (user/full)')
495
481
  @click.argument('key', required=False)
496
- def config_get(full, key=None):
482
+ def config_get(user, key=None):
497
483
  """Get config value."""
498
484
  if key is None:
499
- partial = not full and CONFIG != default_config
485
+ partial = user and default_config != CONFIG
500
486
  CONFIG.print(partial=partial)
501
487
  return
502
488
  CONFIG.get(key)
@@ -519,6 +505,21 @@ def config_set(key, value):
519
505
  console.print(Error(message='Invalid config, not saving it.'))
520
506
 
521
507
 
508
+ @config.command('unset')
509
+ @click.argument('key')
510
+ def config_unset(key):
511
+ """Unset a config value."""
512
+ CONFIG.unset(key)
513
+ config = CONFIG.validate()
514
+ if config:
515
+ saved = CONFIG.save()
516
+ if not saved:
517
+ return
518
+ console.print(f'[bold green]:tada: Saved config to [/]{CONFIG._path}')
519
+ else:
520
+ console.print(Error(message='Invalid config, not saving it.'))
521
+
522
+
522
523
  @config.command('edit')
523
524
  @click.option('--resume', is_flag=True)
524
525
  def config_edit(resume):
@@ -560,7 +561,7 @@ def config_default(save):
560
561
  #-----------#
561
562
  # WORKSPACE #
562
563
  #-----------#
563
- @cli.group(aliases=['ws'])
564
+ @cli.group(aliases=['ws', 'workspaces'])
564
565
  def workspace():
565
566
  """Workspaces."""
566
567
  pass
@@ -593,57 +594,228 @@ def workspace_list():
593
594
  console.print(table)
594
595
 
595
596
 
597
+ #----------#
598
+ # PROFILES #
599
+ #----------#
600
+
601
+ @cli.group(aliases=['p', 'profiles'])
602
+ @click.pass_context
603
+ def profile(ctx):
604
+ """Profiles"""
605
+ pass
606
+
607
+
608
+ @profile.command('list')
609
+ def profile_list():
610
+ table = Table()
611
+ table.add_column("Profile name", style="bold gold3")
612
+ table.add_column("Description", overflow='fold')
613
+ table.add_column("Options", overflow='fold')
614
+ for profile in PROFILES:
615
+ opts_str = ', '.join(f'[yellow3]{k}[/]=[dim yellow3]{v}[/]' for k, v in profile.opts.items())
616
+ table.add_row(profile.name, profile.description or '', opts_str)
617
+ console.print(table)
618
+
619
+
620
+ #-------#
621
+ # ALIAS #
622
+ #-------#
623
+
624
+ @cli.group(aliases=['a', 'aliases'])
625
+ def alias():
626
+ """Aliases."""
627
+ pass
628
+
629
+
630
+ @alias.command('enable')
631
+ @click.pass_context
632
+ def enable_aliases(ctx):
633
+ """Enable aliases."""
634
+ fpath = f'{CONFIG.dirs.data}/.aliases'
635
+ aliases = ctx.invoke(list_aliases, silent=True)
636
+ aliases_str = '\n'.join(aliases)
637
+ with open(fpath, 'w') as f:
638
+ f.write(aliases_str)
639
+ console.print('')
640
+ console.print(f':file_cabinet: Alias file written to {fpath}', style='bold green')
641
+ console.print('To load the aliases, run:')
642
+ md = f"""
643
+ ```sh
644
+ source {fpath} # load the aliases in the current shell
645
+ echo "source {fpath} >> ~/.bashrc" # or add this line to your ~/.bashrc to load them automatically
646
+ ```
647
+ """
648
+ console.print(Markdown(md))
649
+ console.print()
650
+
651
+
652
+ @alias.command('disable')
653
+ @click.pass_context
654
+ def disable_aliases(ctx):
655
+ """Disable aliases."""
656
+ fpath = f'{CONFIG.dirs.data}/.unalias'
657
+ aliases = ctx.invoke(list_aliases, silent=True)
658
+ aliases_str = ''
659
+ for alias in aliases:
660
+ alias_name = alias.split('=')[0]
661
+ if alias.strip().startswith('alias'):
662
+ alias_name = 'un' + alias_name
663
+ aliases_str += alias_name + '\n'
664
+ console.print(f':file_cabinet: Unalias file written to {fpath}', style='bold green')
665
+ console.print('To unload the aliases, run:')
666
+ with open(fpath, 'w') as f:
667
+ f.write(aliases_str)
668
+ md = f"""
669
+ ```sh
670
+ source {fpath}
671
+ ```
672
+ """
673
+ console.print(Markdown(md))
674
+ console.print()
675
+
676
+
677
+ @alias.command('list')
678
+ @click.option('--silent', is_flag=True, default=False, help='No print')
679
+ def list_aliases(silent):
680
+ """List aliases"""
681
+ aliases = []
682
+ aliases.append('\n# Global commands')
683
+ aliases.append('alias x="secator tasks"')
684
+ aliases.append('alias w="secator workflows"')
685
+ aliases.append('alias s="secator scans"')
686
+ aliases.append('alias wk="secator worker"')
687
+ aliases.append('alias ut="secator util"')
688
+ aliases.append('alias c="secator config"')
689
+ aliases.append('alias ws="secator workspaces"')
690
+ aliases.append('alias p="secator profiles"')
691
+ aliases.append('alias a="secator alias"')
692
+ aliases.append('alias aliases="secator alias list"')
693
+ aliases.append('alias r="secator reports"')
694
+ aliases.append('alias h="secator health"')
695
+ aliases.append('alias i="secator install"')
696
+ aliases.append('alias u="secator update"')
697
+ aliases.append('alias t="secator test"')
698
+ aliases.append('\n# Tasks')
699
+ for task in [t for t in discover_tasks()]:
700
+ alias_str = f'alias {task.__name__}="secator task {task.__name__}"'
701
+ if task.__external__:
702
+ alias_str += ' # external'
703
+ aliases.append(alias_str)
704
+
705
+ if silent:
706
+ return aliases
707
+ console.print('[bold gold3]:wrench: Aliases:[/]')
708
+ for alias in aliases:
709
+ alias_split = alias.split('=')
710
+ if len(alias_split) != 2:
711
+ console.print(f'[bold magenta]{alias}')
712
+ continue
713
+ alias_name, alias_cmd = alias_split[0].replace('alias ', ''), alias_split[1].replace('"', '')
714
+ if '# external' in alias_cmd:
715
+ alias_cmd = alias_cmd.replace('# external', ' [bold red]# external[/]')
716
+ console.print(f'[bold gold3]{alias_name:<15}[/] [dim]->[/] [bold green]{alias_cmd}[/]')
717
+
718
+ return aliases
719
+
720
+
596
721
  #--------#
597
722
  # REPORT #
598
723
  #--------#
599
724
 
600
725
 
601
- @cli.group(aliases=['r'])
726
+ @cli.group(aliases=['r', 'reports'])
602
727
  def report():
603
728
  """Reports."""
604
729
  pass
605
730
 
606
731
 
732
+ def process_query(query, fields=None):
733
+ if fields is None:
734
+ fields = []
735
+ otypes = [o.__name__.lower() for o in FINDING_TYPES]
736
+ extractors = []
737
+
738
+ # Process fields
739
+ fields_filter = {}
740
+ if fields:
741
+ for field in fields:
742
+ parts = field.split('.')
743
+ if len(parts) == 2:
744
+ _type, field = parts
745
+ else:
746
+ _type = parts[0]
747
+ field = None
748
+ if _type not in otypes:
749
+ console.print(Error(message='Invalid output type: ' + _type))
750
+ sys.exit(1)
751
+ fields_filter[_type] = field
752
+
753
+ # No query
754
+ if not query:
755
+ if fields:
756
+ extractors = [{'type': field_type, 'field': field, 'condition': 'True', 'op': 'or'} for field_type, field in fields_filter.items()] # noqa: E501
757
+ return extractors
758
+
759
+ # Get operator
760
+ operator = '||'
761
+ if '&&' in query and '||' in query:
762
+ console.print(Error(message='Cannot mix && and || in the same query'))
763
+ sys.exit(1)
764
+ elif '&&' in query:
765
+ operator = '&&'
766
+ elif '||' in query:
767
+ operator = '||'
768
+
769
+ # Process query
770
+ query = query.split(operator)
771
+ for part in query:
772
+ part = part.strip()
773
+ split_part = part.split('.')
774
+ _type = split_part[0]
775
+ if _type not in otypes:
776
+ console.print(Error(message='Invalid output type: ' + _type))
777
+ sys.exit(1)
778
+ if fields and _type not in fields_filter:
779
+ console.print(Warning(message='Type not allowed by --filter field: ' + _type + ' (allowed: ' + ', '.join(fields_filter.keys()) + '). Ignoring extractor.')) # noqa: E501
780
+ continue
781
+ extractor = {
782
+ 'type': _type,
783
+ 'condition': part or 'True',
784
+ 'op': 'and' if operator == '&&' else 'or'
785
+ }
786
+ field = fields_filter.get(_type)
787
+ if field:
788
+ extractor['field'] = field
789
+ extractors.append(extractor)
790
+ return extractors
791
+
792
+
607
793
  @report.command('show')
608
794
  @click.argument('report_query', required=False)
609
795
  @click.option('-o', '--output', type=str, default='console', help='Exporters')
610
796
  @click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
611
797
  @click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
612
- @click.option('-t', '--type', type=str, default='', help=f'Filter by output type. Choices: {FINDING_TYPES_LOWER}')
798
+ @click.option('-f', '--format', "_format", type=str, default='', help=f'Format output, comma-separated of: <output_type> or <output_type>.<field>. [bold]Allowed output types[/]: {", ".join(FINDING_TYPES_LOWER)}') # noqa: E501
613
799
  @click.option('-q', '--query', type=str, default=None, help='Query results using a Python expression')
614
800
  @click.option('-w', '-ws', '--workspace', type=str, default=None, help='Filter by workspace name')
615
801
  @click.option('-u', '--unified', is_flag=True, default=False, help='Show unified results (merge reports and de-duplicates results)') # noqa: E501
616
- def report_show(report_query, output, runner_type, time_delta, type, query, workspace, unified):
802
+ @click.pass_context
803
+ def report_show(ctx, report_query, output, runner_type, time_delta, _format, query, workspace, unified):
617
804
  """Show report results and filter on them."""
618
805
 
806
+ # Get report query from piped input
807
+ if ctx.obj['piped_input']:
808
+ report_query = ','.join(sys.stdin.read().splitlines())
809
+ unified = True
810
+
619
811
  # Get extractors
620
- otypes = [o.__name__.lower() for o in FINDING_TYPES]
621
- extractors = []
622
- if type:
623
- type = type.split(',')
624
- for typedef in type:
625
- if typedef:
626
- if '.' in typedef:
627
- _type, _field = tuple(typedef.split('.'))
628
- else:
629
- _type = typedef
630
- _field = None
631
- extractors.append({
632
- 'type': _type,
633
- 'field': _field,
634
- 'condition': query or 'True'
635
- })
636
- elif query:
637
- query = query.split(';')
638
- for part in query:
639
- _type = part.split('.')[0]
640
- if _type in otypes:
641
- part = part.replace(_type, 'item')
642
- extractor = {
643
- 'type': _type,
644
- 'condition': part or 'True'
645
- }
646
- extractors.append(extractor)
812
+ extractors = process_query(query, fields=_format.split(',') if _format else [])
813
+ if extractors:
814
+ console.print(':wrench: [bold gold3]Showing query summary[/]')
815
+ op = extractors[0]['op']
816
+ console.print(f':carousel_horse: [bold blue]Op[/] [bold orange3]->[/] [bold green]{op.upper()}[/]')
817
+ for extractor in extractors:
818
+ console.print(f':zap: [bold blue]{extractor["type"].title()}[/] [bold orange3]->[/] [bold green]{extractor["condition"]}[/]', highlight=False) # noqa: E501
647
819
 
648
820
  # Build runner instance
649
821
  current = get_file_timestamp()
@@ -659,19 +831,13 @@ def report_show(report_query, output, runner_type, time_delta, type, query, work
659
831
 
660
832
  # Build report queries from fuzzy input
661
833
  paths = []
662
- if report_query:
663
- report_query = report_query.split(',')
664
- else:
665
- report_query = []
666
-
667
- # Load all report paths
668
- load_all_reports = any([not Path(p).exists() for p in report_query])
834
+ report_query = report_query.split(',') if report_query else []
835
+ load_all_reports = not report_query or any([not Path(p).exists() for p in report_query]) # fuzzy query, need to load all reports # noqa: E501
669
836
  all_reports = []
670
837
  if load_all_reports or workspace:
671
838
  all_reports = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
672
839
  if not report_query:
673
840
  report_query = all_reports
674
-
675
841
  for query in report_query:
676
842
  query = str(query)
677
843
  if not query.endswith('/'):
@@ -694,10 +860,12 @@ def report_show(report_query, output, runner_type, time_delta, type, query, work
694
860
  all_results = []
695
861
  for ix, path in enumerate(paths):
696
862
  if unified:
697
- console.print(rf'Loading {path} \[[bold yellow4]{ix + 1}[/]/[bold yellow4]{len(paths)}[/]] \[results={len(all_results)}]...') # noqa: E501
863
+ if ix == 0:
864
+ console.print(f'\n:wrench: [bold gold3]Loading {len(paths)} reports ...[/]')
865
+ console.print(rf':file_cabinet: Loading {path} \[[bold yellow4]{ix + 1}[/]/[bold yellow4]{len(paths)}[/]] \[results={len(all_results)}]...') # noqa: E501
698
866
  with open(path, 'r') as f:
699
- data = loads_dataclass(f.read())
700
867
  try:
868
+ data = loads_dataclass(f.read())
701
869
  info = get_info_from_report_path(path)
702
870
  runner_type = info.get('type', 'unknowns')[:-1]
703
871
  runner.results = flatten(list(data['results'].values()))
@@ -708,21 +876,20 @@ def report_show(report_query, output, runner_type, time_delta, type, query, work
708
876
  report.build(extractors=extractors if not unified else [], dedupe=unified)
709
877
  file_date = get_file_date(path)
710
878
  runner_name = data['info']['name']
711
- console.print(
712
- f'\n{path} ([bold blue]{runner_name}[/] [dim]{runner_type}[/]) ([dim]{file_date}[/]):')
879
+ if not report.is_empty():
880
+ console.print(
881
+ f'\n{path} ([bold blue]{runner_name}[/] [dim]{runner_type}[/]) ([dim]{file_date}[/]):')
713
882
  if report.is_empty():
714
883
  if len(paths) == 1:
715
884
  console.print(Warning(message='No results in report.'))
716
- else:
717
- console.print(Warning(message='No new results since previous scan.'))
718
885
  continue
719
886
  report.send()
720
887
  except json.decoder.JSONDecodeError as e:
721
888
  console.print(Error(message=f'Could not load {path}: {str(e)}'))
722
889
 
723
890
  if unified:
724
- console.print(f'\n:wrench: [bold gold3]Building report by crunching {len(all_results)} results ...[/]')
725
- console.print(':coffee: [dim]Note that this can take a while when the result count is high...[/]')
891
+ console.print(f'\n:wrench: [bold gold3]Building report by crunching {len(all_results)} results ...[/]', end='')
892
+ console.print(' (:coffee: [dim]this can take a while ...[/])')
726
893
  runner.results = all_results
727
894
  report = Report(runner, title=f"Consolidated report - {current}", exporters=exporters)
728
895
  report.build(extractors=extractors, dedupe=True)
@@ -733,7 +900,8 @@ def report_show(report_query, output, runner_type, time_delta, type, query, work
733
900
  @click.option('-ws', '-w', '--workspace', type=str)
734
901
  @click.option('-r', '--runner-type', type=str, default=None, help='Filter by runner type. Choices: task, workflow, scan') # noqa: E501
735
902
  @click.option('-d', '--time-delta', type=str, default=None, help='Keep results newer than time delta. E.g: 26m, 1d, 1y') # noqa: E501
736
- def report_list(workspace, runner_type, time_delta):
903
+ @click.pass_context
904
+ def report_list(ctx, workspace, runner_type, time_delta):
737
905
  """List all secator reports."""
738
906
  paths = list_reports(workspace=workspace, type=runner_type, timedelta=human_to_timedelta(time_delta))
739
907
  paths = sorted(paths, key=lambda x: x.stat().st_mtime, reverse=False)
@@ -747,6 +915,15 @@ def report_list(workspace, runner_type, time_delta):
747
915
  table.add_column("Date")
748
916
  table.add_column("Status", style="green")
749
917
 
918
+ # Print paths if piped
919
+ if ctx.obj['piped_output']:
920
+ if not paths:
921
+ console.print(Error(message='No reports found.'))
922
+ return
923
+ for path in paths:
924
+ print(path)
925
+ return
926
+
750
927
  # Load each report
751
928
  for path in paths:
752
929
  try:
@@ -776,23 +953,26 @@ def report_list(workspace, runner_type, time_delta):
776
953
 
777
954
  if len(paths) > 0:
778
955
  console.print(table)
956
+ console.print(Info(message=f'Found {len(paths)} reports.'))
779
957
  else:
780
- console.print(Error(message='No results found.'))
958
+ console.print(Error(message='No reports found.'))
781
959
 
782
960
 
783
961
  @report.command('export')
784
962
  @click.argument('json_path', type=str)
785
963
  @click.option('--output-folder', '-of', type=str)
786
- @click.option('-output', '-o', type=str, required=True)
964
+ @click.option('--output', '-o', type=str, required=True)
787
965
  def report_export(json_path, output_folder, output):
788
966
  with open(json_path, 'r') as f:
789
967
  data = loads_dataclass(f.read())
790
968
 
969
+ split = json_path.split('/')
970
+ workspace_name = '/'.join(split[:-4]) if len(split) > 4 else '_default'
791
971
  runner_instance = DotMap({
792
972
  "config": {
793
973
  "name": data['info']['name']
794
974
  },
795
- "workspace_name": json_path.split('/')[-4],
975
+ "workspace_name": workspace_name,
796
976
  "reports_folder": output_folder or Path.cwd(),
797
977
  "data": data,
798
978
  "results": flatten(list(data['results'].values()))
@@ -829,92 +1009,151 @@ def report_export(json_path, output_folder, output):
829
1009
  # HEALTH #
830
1010
  #--------#
831
1011
 
832
- @cli.command(name='health')
833
- @click.option('--json', '-json', is_flag=True, default=False, help='JSON lines output')
1012
+ @cli.command(name='health', aliases=['h'])
1013
+ @click.option('--json', '-json', 'json_', is_flag=True, default=False, help='JSON lines output')
834
1014
  @click.option('--debug', '-debug', is_flag=True, default=False, help='Debug health output')
835
1015
  @click.option('--strict', '-strict', is_flag=True, default=False, help='Fail if missing tools')
836
- def health(json, debug, strict):
837
- """[dim]Get health status.[/]"""
838
- tools = ALL_TASKS
839
- status = {'secator': {}, 'languages': {}, 'tools': {}, 'addons': {}}
1016
+ @click.option('--bleeding', '-bleeding', is_flag=True, default=False, help='Check bleeding edge version of tools')
1017
+ def health(json_, debug, strict, bleeding):
1018
+ """Get health status."""
1019
+ tools = discover_tasks()
1020
+ upgrade_cmd = ''
1021
+ results = []
1022
+ messages = []
1023
+
1024
+ # Abort if offline mode is enabled
1025
+ if CONFIG.offline_mode:
1026
+ console.print(Error(message='Cannot run this command in offline mode.'))
1027
+ sys.exit(1)
840
1028
 
841
1029
  # Check secator
842
- console.print(':wrench: [bold gold3]Checking secator ...[/]')
1030
+ console.print(':wrench: [bold gold3]Checking secator ...[/]') if not json_ else None
843
1031
  info = get_version_info('secator', '-version', 'freelabz/secator')
1032
+ info['_type'] = 'core'
1033
+ if info['outdated']:
1034
+ messages.append(f'secator is outdated (latest:{info["latest_version"]}).')
1035
+ results.append(info)
844
1036
  table = get_health_table()
845
- with Live(table, console=console):
1037
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1038
+ with contextmanager:
846
1039
  row = fmt_health_table_row(info)
847
1040
  table.add_row(*row)
848
- status['secator'] = info
849
1041
 
850
1042
  # Check addons
851
- console.print('\n:wrench: [bold gold3]Checking installed addons ...[/]')
1043
+ console.print('\n:wrench: [bold gold3]Checking addons ...[/]') if not json_ else None
852
1044
  table = get_health_table()
853
- with Live(table, console=console):
1045
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1046
+ with contextmanager:
854
1047
  for addon, installed in ADDONS_ENABLED.items():
855
1048
  info = {
856
1049
  'name': addon,
857
1050
  'version': None,
858
- 'status': 'ok' if installed else 'missing',
1051
+ 'status': 'ok' if installed else 'missing_ok',
859
1052
  'latest_version': None,
860
1053
  'installed': installed,
861
1054
  'location': None
862
1055
  }
1056
+ info['_type'] = 'addon'
1057
+ results.append(info)
863
1058
  row = fmt_health_table_row(info, 'addons')
864
1059
  table.add_row(*row)
865
- status['addons'][addon] = info
1060
+ if json_:
1061
+ print(json.dumps(info))
866
1062
 
867
1063
  # Check languages
868
- console.print('\n:wrench: [bold gold3]Checking installed languages ...[/]')
1064
+ console.print('\n:wrench: [bold gold3]Checking languages ...[/]') if not json_ else None
869
1065
  version_cmds = {'go': 'version', 'python3': '--version', 'ruby': '--version'}
870
1066
  table = get_health_table()
871
- with Live(table, console=console):
1067
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1068
+ with contextmanager:
872
1069
  for lang, version_flag in version_cmds.items():
873
1070
  info = get_version_info(lang, version_flag)
874
1071
  row = fmt_health_table_row(info, 'langs')
875
1072
  table.add_row(*row)
876
- status['languages'][lang] = info
1073
+ info['_type'] = 'lang'
1074
+ results.append(info)
1075
+ if json_:
1076
+ print(json.dumps(info))
877
1077
 
878
1078
  # Check tools
879
- console.print('\n:wrench: [bold gold3]Checking installed tools ...[/]')
1079
+ console.print('\n:wrench: [bold gold3]Checking tools ...[/]') if not json_ else None
880
1080
  table = get_health_table()
881
- with Live(table, console=console):
1081
+ error = False
1082
+ contextmanager = Live(table, console=console) if not json_ else nullcontext()
1083
+ upgrade_cmd = 'secator install tools'
1084
+ with contextmanager:
882
1085
  for tool in tools:
883
1086
  info = get_version_info(
884
1087
  tool.cmd.split(' ')[0],
885
1088
  tool.version_flag or f'{tool.opt_prefix}version',
886
1089
  tool.install_github_handle,
887
- tool.install_cmd
1090
+ tool.install_cmd,
1091
+ tool.install_version,
1092
+ bleeding=bleeding
888
1093
  )
1094
+ info['_name'] = tool.__name__
1095
+ info['_type'] = 'tool'
889
1096
  row = fmt_health_table_row(info, 'tools')
890
1097
  table.add_row(*row)
891
- status['tools'][tool.__name__] = info
892
- console.print('')
893
-
894
- # Print JSON health
895
- if json:
896
- import json as _json
897
- print(_json.dumps(status))
898
-
899
- # Print errors and warnings
900
- error = False
901
- for tool, info in status['tools'].items():
902
- if not info['installed']:
903
- console.print(Warning(message=f'{tool} is not installed.'))
904
- error = True
905
- elif info['outdated']:
906
- message = (
907
- f'{tool} is outdated (current:{info["version"]}, latest:{info["latest_version"]}).'
908
- f' Run `secator install tools {tool}` to update it.'
909
- )
1098
+ if not info['installed']:
1099
+ messages.append(f'{tool.__name__} is not installed.')
1100
+ info['next_version'] = tool.install_version
1101
+ error = True
1102
+ elif info['outdated']:
1103
+ msg = 'latest' if bleeding else 'supported'
1104
+ message = (
1105
+ f'{tool.__name__} is outdated (current:{info["version"]}, {msg}:{info["latest_version"]}).'
1106
+ )
1107
+ messages.append(message)
1108
+ info['upgrade'] = True
1109
+ info['next_version'] = info['latest_version']
1110
+
1111
+ elif info['bleeding']:
1112
+ msg = 'latest' if bleeding else 'supported'
1113
+ message = (
1114
+ f'{tool.__name__} is bleeding edge (current:{info["version"]}, {msg}:{info["latest_version"]}).'
1115
+ )
1116
+ messages.append(message)
1117
+ info['downgrade'] = True
1118
+ info['next_version'] = info['latest_version']
1119
+ results.append(info)
1120
+ if json_:
1121
+ print(json.dumps(info))
1122
+ console.print('') if not json_ else None
1123
+
1124
+ if not json_ and messages:
1125
+ console.print('\n[bold red]Issues found:[/]')
1126
+ for message in messages:
910
1127
  console.print(Warning(message=message))
911
1128
 
912
1129
  # Strict mode
913
1130
  if strict:
914
1131
  if error:
915
1132
  sys.exit(1)
916
- console.print(Info(message='Strict healthcheck passed !'))
917
-
1133
+ console.print(Info(message='Strict healthcheck passed !')) if not json_ else None
1134
+
1135
+ # Build upgrade command
1136
+ cmds = []
1137
+ tool_cmd = ''
1138
+ for info in results:
1139
+ if info['_type'] == 'core' and info['outdated']:
1140
+ cmds.append('secator update')
1141
+ elif info['_type'] == 'tool' and info.get('next_version'):
1142
+ tool_cmd += f',{info["_name"]}=={info["next_version"]}'
1143
+
1144
+ if tool_cmd:
1145
+ tool_cmd = f'secator install tools {tool_cmd.lstrip(",")}'
1146
+ cmds.append(tool_cmd)
1147
+ upgrade_cmd = ' && '.join(cmds)
1148
+ console.print('') if not json_ else None
1149
+ if upgrade_cmd:
1150
+ console.print(Info(message='Run the following to upgrade secator and tools:')) if not json_ else None
1151
+ if json_:
1152
+ print(json.dumps({'upgrade_cmd': upgrade_cmd}))
1153
+ else:
1154
+ print(upgrade_cmd)
1155
+ else:
1156
+ console.print(Info(message='Everything is up to date !')) if not json_ else None
918
1157
 
919
1158
  #---------#
920
1159
  # INSTALL #
@@ -924,7 +1163,7 @@ def health(json, debug, strict):
924
1163
  def run_install(title=None, cmd=None, packages=None, next_steps=None):
925
1164
  if CONFIG.offline_mode:
926
1165
  console.print(Error(message='Cannot run this command in offline mode.'))
927
- return
1166
+ sys.exit(1)
928
1167
  # with console.status(f'[bold yellow] Installing {title}...'):
929
1168
  if cmd:
930
1169
  from secator.installer import SourceInstaller
@@ -942,9 +1181,9 @@ def run_install(title=None, cmd=None, packages=None, next_steps=None):
942
1181
  sys.exit(return_code)
943
1182
 
944
1183
 
945
- @cli.group()
1184
+ @cli.group(aliases=['i'])
946
1185
  def install():
947
- """[dim]Install langs, tools and addons.[/]"""
1186
+ """Install langs, tools and addons."""
948
1187
  pass
949
1188
 
950
1189
 
@@ -1104,7 +1343,7 @@ def install_tools(cmds, cleanup, fail_fast):
1104
1343
  """Install supported tools."""
1105
1344
  if CONFIG.offline_mode:
1106
1345
  console.print(Error(message='Cannot run this command in offline mode.'))
1107
- return
1346
+ sys.exit(1)
1108
1347
  tools = []
1109
1348
  if cmds is not None:
1110
1349
  cmds = cmds.split(',')
@@ -1113,15 +1352,17 @@ def install_tools(cmds, cleanup, fail_fast):
1113
1352
  cmd, version = tuple(cmd.split('=='))
1114
1353
  else:
1115
1354
  cmd, version = cmd, None
1116
- cls = next((cls for cls in ALL_TASKS if cls.__name__ == cmd), None)
1355
+ cls = next((cls for cls in discover_tasks() if cls.__name__ == cmd), None)
1117
1356
  if cls:
1118
1357
  if version:
1358
+ if cls.install_version and cls.install_version.startswith('v') and not version.startswith('v'):
1359
+ version = f'v{version}'
1119
1360
  cls.install_version = version
1120
1361
  tools.append(cls)
1121
1362
  else:
1122
1363
  console.print(Warning(message=f'Tool {cmd} is not supported or inexistent.'))
1123
1364
  else:
1124
- tools = ALL_TASKS
1365
+ tools = discover_tasks()
1125
1366
  tools.sort(key=lambda x: x.__name__)
1126
1367
  return_code = 0
1127
1368
  if not tools:
@@ -1158,7 +1399,7 @@ def install_tools(cmds, cleanup, fail_fast):
1158
1399
  @cli.command('update')
1159
1400
  @click.option('--all', '-a', is_flag=True, help='Update all secator dependencies (addons, tools, ...)')
1160
1401
  def update(all):
1161
- """[dim]Update to latest version.[/]"""
1402
+ """Update to latest version."""
1162
1403
  if CONFIG.offline_mode:
1163
1404
  console.print(Error(message='Cannot run this command in offline mode.'))
1164
1405
  sys.exit(1)
@@ -1191,7 +1432,7 @@ def update(all):
1191
1432
  # Update tools
1192
1433
  if all:
1193
1434
  return_code = 0
1194
- for cls in ALL_TASKS:
1435
+ for cls in discover_tasks():
1195
1436
  cmd = cls.cmd.split(' ')[0]
1196
1437
  version_flag = cls.get_version_flag()
1197
1438
  info = get_version_info(cmd, version_flag, cls.install_github_handle)
@@ -1202,96 +1443,6 @@ def update(all):
1202
1443
  return_code = 1
1203
1444
  sys.exit(return_code)
1204
1445
 
1205
- #-------#
1206
- # ALIAS #
1207
- #-------#
1208
-
1209
-
1210
- @cli.group()
1211
- def alias():
1212
- """[dim]Configure aliases.[/]"""
1213
- pass
1214
-
1215
-
1216
- @alias.command('enable')
1217
- @click.pass_context
1218
- def enable_aliases(ctx):
1219
- """Enable aliases."""
1220
- fpath = f'{CONFIG.dirs.data}/.aliases'
1221
- aliases = ctx.invoke(list_aliases, silent=True)
1222
- aliases_str = '\n'.join(aliases)
1223
- with open(fpath, 'w') as f:
1224
- f.write(aliases_str)
1225
- console.print('')
1226
- console.print(f':file_cabinet: Alias file written to {fpath}', style='bold green')
1227
- console.print('To load the aliases, run:')
1228
- md = f"""
1229
- ```sh
1230
- source {fpath} # load the aliases in the current shell
1231
- echo "source {fpath} >> ~/.bashrc" # or add this line to your ~/.bashrc to load them automatically
1232
- ```
1233
- """
1234
- console.print(Markdown(md))
1235
- console.print()
1236
-
1237
-
1238
- @alias.command('disable')
1239
- @click.pass_context
1240
- def disable_aliases(ctx):
1241
- """Disable aliases."""
1242
- fpath = f'{CONFIG.dirs.data}/.unalias'
1243
- aliases = ctx.invoke(list_aliases, silent=True)
1244
- aliases_str = ''
1245
- for alias in aliases:
1246
- aliases_str += alias.split('=')[0].replace('alias', 'unalias') + '\n'
1247
- console.print(f':file_cabinet: Unalias file written to {fpath}', style='bold green')
1248
- console.print('To unload the aliases, run:')
1249
- with open(fpath, 'w') as f:
1250
- f.write(aliases_str)
1251
- md = f"""
1252
- ```sh
1253
- source {fpath}
1254
- ```
1255
- """
1256
- console.print(Markdown(md))
1257
- console.print()
1258
-
1259
-
1260
- @alias.command('list')
1261
- @click.option('--silent', is_flag=True, default=False, help='No print')
1262
- def list_aliases(silent):
1263
- """List aliases"""
1264
- aliases = []
1265
- aliases.extend([
1266
- f'alias {task.__name__}="secator x {task.__name__}"'
1267
- for task in ALL_TASKS
1268
- ])
1269
- aliases.extend([
1270
- f'alias {workflow.alias}="secator w {workflow.name}"'
1271
- for workflow in ALL_WORKFLOWS
1272
- ])
1273
- aliases.extend([
1274
- f'alias {workflow.name}="secator w {workflow.name}"'
1275
- for workflow in ALL_WORKFLOWS
1276
- ])
1277
- aliases.extend([
1278
- f'alias scan_{scan.name}="secator s {scan.name}"'
1279
- for scan in ALL_SCANS
1280
- ])
1281
- aliases.append('alias listx="secator x"')
1282
- aliases.append('alias listw="secator w"')
1283
- aliases.append('alias lists="secator s"')
1284
-
1285
- if silent:
1286
- return aliases
1287
- console.print('Aliases:')
1288
- for alias in aliases:
1289
- alias_split = alias.split('=')
1290
- alias_name, alias_cmd = alias_split[0].replace('alias ', ''), alias_split[1].replace('"', '')
1291
- console.print(f'[bold magenta]{alias_name:<15}-> {alias_cmd}')
1292
-
1293
- return aliases
1294
-
1295
1446
 
1296
1447
  #------#
1297
1448
  # TEST #
@@ -1300,7 +1451,7 @@ def list_aliases(silent):
1300
1451
 
1301
1452
  @cli.group(cls=OrderedGroup)
1302
1453
  def test():
1303
- """[dim]Run tests."""
1454
+ """[dim]Run tests (dev build only)."""
1304
1455
  if not DEV_PACKAGE:
1305
1456
  console.print(Error(message='You MUST use a development version of secator to run tests.'))
1306
1457
  sys.exit(1)
@@ -1310,7 +1461,7 @@ def test():
1310
1461
  pass
1311
1462
 
1312
1463
 
1313
- def run_test(cmd, name=None, exit=True, verbose=False):
1464
+ def run_test(cmd, name=None, exit=True, verbose=False, use_os_system=False):
1314
1465
  """Run a test and return the result.
1315
1466
 
1316
1467
  Args:
@@ -1318,27 +1469,43 @@ def run_test(cmd, name=None, exit=True, verbose=False):
1318
1469
  name (str, optional): Name of the test.
1319
1470
  exit (bool, optional): Exit after running the test with the return code.
1320
1471
  verbose (bool, optional): Print verbose output.
1472
+ use_os_system (bool, optional): Use os.system to run the command.
1321
1473
 
1322
1474
  Returns:
1323
1475
  Return code of the test.
1324
1476
  """
1325
1477
  cmd_name = name + ' tests' if name else 'tests'
1326
- result = Command.execute(cmd, name=cmd_name, cwd=ROOT_FOLDER, quiet=not verbose)
1327
- if name:
1328
- if result.return_code == 0:
1329
- console.print(f':tada: {name.capitalize()} tests passed !', style='bold green')
1330
- else:
1331
- console.print(f':x: {name.capitalize()} tests failed !', style='bold red')
1332
- if exit:
1333
- sys.exit(result.return_code)
1334
- return result.return_code
1478
+ if use_os_system:
1479
+ console.print(f'[bold red]{cmd}[/]')
1480
+ if not verbose:
1481
+ cmd += ' >/dev/null 2>&1'
1482
+ ret = os.system(cmd)
1483
+ if exit:
1484
+ sys.exit(os.waitstatus_to_exitcode(ret))
1485
+ return ret
1486
+ else:
1487
+ result = Command.execute(cmd, name=cmd_name, cwd=ROOT_FOLDER, quiet=not verbose)
1488
+ if name:
1489
+ if result.return_code == 0:
1490
+ console.print(f':tada: {name.capitalize()} tests passed !', style='bold green')
1491
+ else:
1492
+ console.print(f':x: {name.capitalize()} tests failed !', style='bold red')
1493
+ if exit:
1494
+ sys.exit(result.return_code)
1495
+ return result.return_code
1335
1496
 
1336
1497
 
1337
1498
  @test.command()
1338
- def lint():
1499
+ @click.option('--linter', '-l', type=click.Choice(['flake8', 'ruff', 'isort', 'pylint']), default='flake8', help='Linter to use') # noqa: E501
1500
+ def lint(linter):
1339
1501
  """Run lint tests."""
1340
- cmd = f'{sys.executable} -m flake8 secator/'
1341
- run_test(cmd, 'lint', verbose=True)
1502
+ opts = ''
1503
+ if linter == 'pylint':
1504
+ opts = '--indent-string "\t" --max-line-length 160 --disable=R,C,W'
1505
+ elif linter == 'ruff':
1506
+ opts = ' check'
1507
+ cmd = f'{sys.executable} -m {linter} {opts} secator/'
1508
+ run_test(cmd, 'lint', verbose=True, use_os_system=True)
1342
1509
 
1343
1510
 
1344
1511
  @test.command()
@@ -1366,11 +1533,11 @@ def unit(tasks, workflows, scans, test):
1366
1533
 
1367
1534
  import shutil
1368
1535
  shutil.rmtree('/tmp/.secator', ignore_errors=True)
1369
- cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.unit -m pytest -s -v tests/unit'
1536
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.unit -m pytest -s -vv tests/unit --durations=5' # noqa: E501
1370
1537
  if test:
1371
1538
  test_str = ' or '.join(test.split(','))
1372
1539
  cmd += f' -k "{test_str}"'
1373
- run_test(cmd, 'unit', verbose=True)
1540
+ run_test(cmd, 'unit', verbose=True, use_os_system=True)
1374
1541
 
1375
1542
 
1376
1543
  @test.command()
@@ -1378,13 +1545,15 @@ def unit(tasks, workflows, scans, test):
1378
1545
  @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1379
1546
  @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1380
1547
  @click.option('--test', '-t', type=str, help='Secator test to run')
1381
- def integration(tasks, workflows, scans, test):
1548
+ @click.option('--no-cleanup', '-nc', is_flag=True, help='Do not perform cleanup (keep lab running, faster for relaunching tests)') # noqa: E501
1549
+ def integration(tasks, workflows, scans, test, no_cleanup):
1382
1550
  """Run integration tests."""
1383
1551
  os.environ['TEST_TASKS'] = tasks or ''
1384
1552
  os.environ['TEST_WORKFLOWS'] = workflows or ''
1385
1553
  os.environ['TEST_SCANS'] = scans or ''
1386
1554
  os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1387
1555
  os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1556
+ os.environ['TEST_NO_CLEANUP'] = '1' if no_cleanup else '0'
1388
1557
 
1389
1558
  if not test:
1390
1559
  if tasks:
@@ -1397,11 +1566,42 @@ def integration(tasks, workflows, scans, test):
1397
1566
  import shutil
1398
1567
  shutil.rmtree('/tmp/.secator', ignore_errors=True)
1399
1568
 
1400
- cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.integration -m pytest -s -v tests/integration' # noqa: E501
1569
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.integration -m pytest -s -vv tests/integration --durations=5' # noqa: E501
1401
1570
  if test:
1402
1571
  test_str = ' or '.join(test.split(','))
1403
1572
  cmd += f' -k "{test_str}"'
1404
- run_test(cmd, 'integration', verbose=True)
1573
+ run_test(cmd, 'integration', verbose=True, use_os_system=True)
1574
+
1575
+
1576
+ @test.command()
1577
+ @click.option('--tasks', type=str, default='', help='Secator tasks to test (comma-separated)')
1578
+ @click.option('--workflows', type=str, default='', help='Secator workflows to test (comma-separated)')
1579
+ @click.option('--scans', type=str, default='', help='Secator scans to test (comma-separated)')
1580
+ @click.option('--test', '-t', type=str, help='Secator test to run')
1581
+ def template(tasks, workflows, scans, test):
1582
+ """Run integration tests."""
1583
+ os.environ['TEST_TASKS'] = tasks or ''
1584
+ os.environ['TEST_WORKFLOWS'] = workflows or ''
1585
+ os.environ['TEST_SCANS'] = scans or ''
1586
+ os.environ['SECATOR_DIRS_DATA'] = '/tmp/.secator'
1587
+ os.environ['SECATOR_RUNNERS_SKIP_CVE_SEARCH'] = '1'
1588
+
1589
+ if not test:
1590
+ if tasks:
1591
+ test = 'test_tasks'
1592
+ elif workflows:
1593
+ test = 'test_workflows'
1594
+ elif scans:
1595
+ test = 'test_scans'
1596
+
1597
+ import shutil
1598
+ shutil.rmtree('/tmp/.secator', ignore_errors=True)
1599
+
1600
+ cmd = f'{sys.executable} -m coverage run --omit="*test*" --data-file=.coverage.templates -m pytest -s -vv tests/template --durations=5' # noqa: E501
1601
+ if test:
1602
+ test_str = ' or '.join(test.split(','))
1603
+ cmd += f' -k "{test_str}"'
1604
+ run_test(cmd, 'template', verbose=True)
1405
1605
 
1406
1606
 
1407
1607
  @test.command()
@@ -1424,22 +1624,35 @@ def performance(tasks, workflows, scans, test):
1424
1624
  if test:
1425
1625
  test_str = ' or '.join(test.split(','))
1426
1626
  cmd += f' -k "{test_str}"'
1427
- run_test(cmd, 'performance', verbose=True)
1627
+ run_test(cmd, 'performance', verbose=True, use_os_system=True)
1428
1628
 
1429
1629
 
1430
1630
  @test.command()
1431
1631
  @click.argument('name', type=str)
1432
1632
  @click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
1433
1633
  @click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
1434
- def task(name, verbose, check):
1634
+ @click.option('--system-exit', '-e', is_flag=True, default=True, help='Exit with system exit code')
1635
+ def task(name, verbose, check, system_exit):
1435
1636
  """Test a single task for semantics errors, and run unit + integration tests."""
1436
1637
  console.print(f'[bold gold3]:wrench: Testing task {name} ...[/]')
1437
- task = [task for task in ALL_TASKS if task.__name__ == name]
1638
+ task = [task for task in discover_tasks() if task.__name__ == name.strip()]
1438
1639
  warnings = []
1439
1640
  errors = []
1440
1641
  exit_code = 0
1441
1642
 
1442
1643
  # Check if task is correctly registered
1644
+ check_test(
1645
+ len(task) == 1,
1646
+ 'Check task is registered',
1647
+ 'Task is not registered. Please check your task name.',
1648
+ errors
1649
+ )
1650
+ if errors:
1651
+ if system_exit:
1652
+ sys.exit(1)
1653
+ else:
1654
+ return False
1655
+
1443
1656
  task = task[0]
1444
1657
  task_name = task.__name__
1445
1658
 
@@ -1451,7 +1664,10 @@ def task(name, verbose, check):
1451
1664
  errors
1452
1665
  )
1453
1666
  if errors:
1454
- sys.exit(0)
1667
+ if system_exit:
1668
+ sys.exit(1)
1669
+ else:
1670
+ return False
1455
1671
 
1456
1672
  # Run install
1457
1673
  cmd = f'secator install tools {task_name}'
@@ -1561,7 +1777,28 @@ def task(name, verbose, check):
1561
1777
  console.print(warning)
1562
1778
 
1563
1779
  console.print("\n")
1564
- sys.exit(exit_code)
1780
+ if system_exit:
1781
+ sys.exit(exit_code)
1782
+ else:
1783
+ return True if exit_code == 0 else False
1784
+
1785
+
1786
+ @test.command()
1787
+ @click.pass_context
1788
+ @click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)') # noqa: E501
1789
+ @click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
1790
+ def tasks(ctx, check, verbose):
1791
+ """Test all tasks for semantics errors, and run unit + integration tests."""
1792
+ results = []
1793
+ for cls in discover_tasks():
1794
+ success = ctx.invoke(task, name=cls.__name__, verbose=verbose, check=check, system_exit=False)
1795
+ results.append(success)
1796
+
1797
+ if any(not success for success in results):
1798
+ console.print(Error(message='Tasks checks failed. Please check the output for more details.'))
1799
+ sys.exit(1)
1800
+ console.print(Info(message='All tasks checks passed.'))
1801
+ sys.exit(0)
1565
1802
 
1566
1803
 
1567
1804
  def check_test(condition, message, fail_message, results=[], warn=False):
@@ -1583,13 +1820,16 @@ def check_test(condition, message, fail_message, results=[], warn=False):
1583
1820
  @test.command()
1584
1821
  @click.option('--unit-only', '-u', is_flag=True, default=False, help='Only generate coverage for unit tests')
1585
1822
  @click.option('--integration-only', '-i', is_flag=True, default=False, help='Only generate coverage for integration tests') # noqa: E501
1586
- def coverage(unit_only, integration_only):
1823
+ @click.option('--template-only', '-t', is_flag=True, default=False, help='Only generate coverage for template tests') # noqa: E501
1824
+ def coverage(unit_only, integration_only, template_only):
1587
1825
  """Run coverage combine + coverage report."""
1588
1826
  cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*,*/templates/*'
1589
1827
  if unit_only:
1590
1828
  cmd += ' --data-file=.coverage.unit'
1591
1829
  elif integration_only:
1592
1830
  cmd += ' --data-file=.coverage.integration'
1831
+ elif template_only:
1832
+ cmd += ' --data-file=.coverage.template'
1593
1833
  else:
1594
1834
  Command.execute(f'{sys.executable} -m coverage combine --keep', name='coverage combine', cwd=ROOT_FOLDER)
1595
- run_test(cmd, 'coverage')
1835
+ run_test(cmd, 'coverage', use_os_system=True)