secator 0.11.1__py3-none-any.whl → 0.13.0__py3-none-any.whl

secator/cli.py

@@ -37,7 +37,7 @@ ALL_TASKS = discover_tasks()
 ALL_WORKFLOWS = [t for t in TEMPLATES if t.type == 'workflow']
 ALL_SCANS = [t for t in TEMPLATES if t.type == 'scan']
 FINDING_TYPES_LOWER = [c.__name__.lower() for c in FINDING_TYPES]
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '-help', '--help'])
 
 
 #-----#
@@ -46,15 +46,16 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 
 @click.group(cls=OrderedGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
-@click.option('--version', '-version', is_flag=True, default=False)
+@click.option('--version', '-version', '-v', is_flag=True, default=False)
+@click.option('--quiet', '-quiet', '-q', is_flag=True, default=False)
 @click.pass_context
-def cli(ctx, version):
+def cli(ctx, version, quiet):
 	"""Secator CLI."""
 	ctx.obj = {
 		'piped_input': S_ISFIFO(os.fstat(0).st_mode),
 		'piped_output': not sys.stdout.isatty()
 	}
-	if not ctx.obj['piped_output']:
+	if not ctx.obj['piped_output'] and not quiet:
 		console.print(ASCII, highlight=False)
 	if ctx.invoked_subcommand is None:
 		if version:
@@ -67,11 +68,16 @@ def cli(ctx, version):
 # TASK #
 #------#
 
-@cli.group(aliases=['x', 't'])
+@cli.group(aliases=['x', 't'], invoke_without_command=True)
+@click.option('--list', '-list', is_flag=True, default=False)
 @click.pass_context
-def task(ctx):
+def task(ctx, list=False):
 	"""Run a task."""
-	pass
+	if list:
+		print("\n".join(sorted([t.__name__ for t in ALL_TASKS])))
+		return
+	if ctx.invoked_subcommand is None:
+		ctx.get_help()
 
 
 for cls in ALL_TASKS:
@@ -83,11 +89,16 @@ for cls in ALL_TASKS:
 #----------#
 
 
-@cli.group(cls=OrderedGroup, aliases=['w'])
+@cli.group(cls=OrderedGroup, aliases=['w'], invoke_without_command=True)
+@click.option('--list', '-list', is_flag=True, default=False)
 @click.pass_context
-def workflow(ctx):
+def workflow(ctx, list=False):
 	"""Run a workflow."""
-	pass
+	if list:
+		print("\n".join(sorted([t.name for t in ALL_WORKFLOWS])))
+		return
+	if ctx.invoked_subcommand is None:
+		ctx.get_help()
 
 
 for config in sorted(ALL_WORKFLOWS, key=lambda x: x['name']):
@@ -98,11 +109,16 @@ for config in sorted(ALL_WORKFLOWS, key=lambda x: x['name']):
 # SCAN #
 #------#
 
-@cli.group(cls=OrderedGroup, aliases=['s'])
+@cli.group(cls=OrderedGroup, aliases=['s'], invoke_without_command=True)
+@click.option('--list', '-list', is_flag=True, default=False)
 @click.pass_context
-def scan(ctx):
+def scan(ctx, list=False):
 	"""Run a scan."""
-	pass
+	if list:
+		print("\n".join(sorted([t.name for t in ALL_SCANS])))
+		return
+	if ctx.invoked_subcommand is None:
+		ctx.get_help()
 
 
 for config in sorted(ALL_SCANS, key=lambda x: x['name']):
@@ -119,7 +135,7 @@ for config in sorted(ALL_SCANS, key=lambda x: x['name']):
 @click.option('-r', '--reload', is_flag=True, help='Autoreload Celery on code changes.')
 @click.option('-Q', '--queue', type=str, default='', help='Listen to a specific queue.')
 @click.option('-P', '--pool', type=str, default='eventlet', help='Pool implementation.')
-@click.option('--quiet', is_flag=True, help='Quiet mode.')
+@click.option('--quiet', is_flag=True, default=False, help='Quiet mode.')
 @click.option('--loglevel', type=str, default='INFO', help='Log level.')
 @click.option('--check', is_flag=True, help='Check if Celery worker is alive.')
 @click.option('--dev', is_flag=True, help='Start a worker in dev mode (celery multi).')
@@ -860,13 +876,21 @@ def health(json, debug, strict):
 		import json as _json
 		print(_json.dumps(status))
 
+	# Print errors and warnings
+	error = False
+	for tool, info in status['tools'].items():
+		if not info['installed']:
+			console.print(Warning(message=f'{tool} is not installed.'))
+			error = True
+		elif info['outdated']:
+			message = (
+				f'{tool} is outdated (current:{info["version"]}, latest:{info["latest_version"]}).'
+				f' Run `secator install tools {tool}` to update it.'
+			)
+			console.print(Warning(message=message))
+
 	# Strict mode
 	if strict:
-		error = False
-		for tool, info in status['tools'].items():
-			if not info['installed']:
-				console.print(Error(message=f'{tool} is not installed.'))
-				error = True
 		if error:
 			sys.exit(1)
 		console.print(Info(message='Strict healthcheck passed !'))
@@ -1061,16 +1085,27 @@ def install_tools(cmds, cleanup, fail_fast):
 	if CONFIG.offline_mode:
 		console.print(Error(message='Cannot run this command in offline mode.'))
 		return
+	tools = []
 	if cmds is not None:
 		cmds = cmds.split(',')
-		tools = [cls for cls in ALL_TASKS if cls.__name__ in cmds]
+		for cmd in cmds:
+			if '==' in cmd:
+				cmd, version = tuple(cmd.split('=='))
+			else:
+				cmd, version = cmd, None
+			cls = next((cls for cls in ALL_TASKS if cls.__name__ == cmd), None)
+			if cls:
+				if version:
+					cls.install_version = version
+				tools.append(cls)
+			else:
+				console.print(Warning(message=f'Tool {cmd} is not supported or inexistent.'))
 	else:
 		tools = ALL_TASKS
 	tools.sort(key=lambda x: x.__name__)
 	return_code = 0
 	if not tools:
-		cmd_str = ' '.join(cmds)
-		console.print(Error(message=f'No tools found for {cmd_str}.'))
+		console.print(Error(message='No tools found for installing.'))
 		return
 	for ix, cls in enumerate(tools):
 		# with console.status(f'[bold yellow][{ix + 1}/{len(tools)}] Installing {cls.__name__} ...'):
@@ -1140,7 +1175,7 @@ def update(all):
 			cmd = cls.cmd.split(' ')[0]
 			version_flag = cls.get_version_flag()
 			info = get_version_info(cmd, version_flag, cls.install_github_handle)
-			if not info['installed'] or info['status'] == 'outdated' or not info['latest_version']:
+			if not info['installed'] or info['outdated'] or not info['latest_version']:
 				# with console.status(f'[bold yellow]Installing {cls.__name__} ...'):
 				status = ToolInstaller.install(cls)
 				if not status.is_ok():
@@ -1375,73 +1410,147 @@ def performance(tasks, workflows, scans, test):
 @test.command()
 @click.argument('name', type=str)
 @click.option('--verbose', '-v', is_flag=True, default=False, help='Print verbose output')
-def task(name, verbose):
-	"""Test task."""
+@click.option('--check', '-c', is_flag=True, default=False, help='Check task semantics only (no unit + integration tests)')  # noqa: E501
+def task(name, verbose, check):
+	"""Test a single task for semantics errors, and run unit + integration tests."""
+	console.print(f'[bold gold3]:wrench: Testing task {name} ...[/]')
 	task = [task for task in ALL_TASKS if task.__name__ == name]
 	warnings = []
+	errors = []
 	exit_code = 0
 
 	# Check if task is correctly registered
-	check_error(task, 'Check task is registered', 'Task is not registered. Make sure there is no syntax errors in the task class definition.', warnings)  # noqa: E501
 	task = task[0]
 	task_name = task.__name__
 
 	# Run install
-	console.print(f'\n[bold gold3]:wrench: Running install tests for task {name} ...[/]') if verbose else None
 	cmd = f'secator install tools {task_name}'
 	ret_code = Command.execute(cmd, name='install', quiet=not verbose, cwd=ROOT_FOLDER)
 	version_info = task.get_version_info()
-	check_error(version_info['installed'], 'Check task is installed', 'Failed to install command. Fix your installation command.', warnings)  # noqa: E501
-	check_error(any(cmd for cmd in [task.install_cmd, task.install_github_handle]), 'Check task installation command is defined', 'Task has no installation command. Please define a `install_cmd` or `install_github_handle` class attribute.', warnings)  # noqa: E501
-	check_error(version_info['version'], 'Check task version can be fetched', 'Failed to detect version info. Fix your `version_flag` class attribute.', warnings)  # noqa: E501
+	if verbose:
+		console.print(f'Version info:\n{version_info}')
+	status = version_info['status']
+	check_test(
+		version_info['installed'],
+		'Check task is installed',
+		'Failed to install command. Fix your installation command.',
+		errors
+	)
+	check_test(
+		any(cmd for cmd in [task.install_cmd, task.install_github_handle]),
+		'Check task installation command is defined',
+		'Task has no installation command. Please define one or more of the following class attributes: `install_pre`, `install_cmd`, `install_post`, `install_github_handle`.',  # noqa: E501
+		errors
+	)
+	check_test(
+		version_info['version'],
+		'Check task version can be fetched',
+		'Failed to detect current version. Consider updating your `version_flag` class attribute.',
+		warnings,
+		warn=True
+	)
+	check_test(
+		status != 'latest unknown',
+		'Check latest version',
+		'Failed to detect latest version.',
+		warnings,
+		warn=True
+	)
+	check_test(
+		not version_info['outdated'],
+		'Check task version is up to date',
+		f'Task is not up to date (current version: {version_info["version"]}, latest: {version_info["latest_version"]}). Consider updating your `install_version` class attribute.',  # noqa: E501
+		warnings,
+		warn=True
+	)
 
 	# Run task-specific tests
-	console.print(f'\n[bold gold3]:wrench: Running task-specific tests for {name} ...[/]') if verbose else None
-	check_error(task.__doc__, 'Check task description is set (cls.__doc__)', 'Task has no description (class docstring).', warnings)  # noqa: E501
-	check_error(task.cmd, 'Check task command is set (cls.cmd)', 'Task has no cmd attribute.', warnings)
-	check_error(task.input_type, 'Check task input type is set (cls.input_type)', 'Task has no input_type attribute.', warnings)  # noqa: E501
-	check_error(task.output_types, 'Check task output types is set (cls.output_types)', 'Task has no output_types attribute.', warnings)  # noqa: E501
-
-	# Print all warnings
-	exit_code = 1 if len(warnings) > 0 else 0
-	if exit_code == 1:
-		console.print()
-		console.print("[bold red]Issues:[/]")
-		for warning in warnings:
-			console.print(warning)
-		console.print()
-		console.print(Info(message=f'Skipping unit and integration tests for {name} due to previous errors.'))
-		console.print(Error(message=f'Task {name} tests failed. Please fix the issues above before making a PR.'))
-		sys.exit(exit_code)
-
-	# Run unit tests
-	console.print(f'\n[bold gold3]:wrench: Running unit tests for {name} ...[/]') if verbose else None
-	cmd = f'secator test unit --tasks {name}'
-	ret_code = run_test(cmd, exit=False, verbose=verbose)
-	check_error(ret_code == 0, 'Check unit tests pass', 'Unit tests failed.', warnings)
-
-	# Run integration tests
-	console.print(f'\n[bold gold3]:wrench: Running integration tests for {name} ...[/]') if verbose else None
-	cmd = f'secator test integration --tasks {name}'
-	ret_code = run_test(cmd, exit=False, verbose=verbose)
-	check_error(ret_code == 0, 'Check integration tests pass', 'Integration tests failed.', warnings)
+	check_test(
+		task.__doc__,
+		'Check task description is set (cls.__doc__)',
+		'Task has no description (class docstring).',
+		errors
+	)
+	check_test(
+		task.cmd,
+		'Check task command is set (cls.cmd)',
+		'Task has no cmd attribute.',
+		errors
+	)
+	check_test(
+		task.input_type,
+		'Check task input type is set (cls.input_type)',
+		'Task has no input_type attribute.',
+		warnings,
+		warn=True
+	)
+	check_test(
+		task.output_types,
+		'Check task output types is set (cls.output_types)',
+		'Task has no output_types attribute. Consider setting some so that secator can load your task outputs.',
+		warnings,
+		warn=True
+	)
+	check_test(
+		task.install_version,
+		'Check task install_version is set (cls.install_version)',
+		'Task has no install_version attribute. Consider setting it to pin the tool version and ensure it does not break in the future.',  # noqa: E501
+		warnings,
+		warn=True
+	)
+
+	if not check:
+
+		# Run unit tests
+		cmd = f'secator test unit --tasks {name}'
+		ret_code = run_test(cmd, exit=False, verbose=verbose)
+		check_test(
+			ret_code == 0,
+			'Check unit tests pass',
+			'Unit tests failed.',
+			errors
+		)
+
+		# Run integration tests
+		cmd = f'secator test integration --tasks {name}'
+		ret_code = run_test(cmd, exit=False, verbose=verbose)
+		check_test(
+			ret_code == 0,
+			'Check integration tests pass',
+			'Integration tests failed.',
+			errors
+		)
 
 	# Exit with exit code
-	exit_code = 1 if len(warnings) > 0 else 0
+	exit_code = 1 if len(errors) > 0 else 0
 	if exit_code == 0:
-		console.print(f':tada: Task {name} tests passed ! You are free to make a PR.', style='bold green')
+		console.print(f':tada: Task {name} tests passed !', style='bold green')
 	else:
-		console.print(Error(message=f'Task {name} tests failed. Please fix the issues above before making a PR.'))
+		console.print('\n[bold gold3]Errors:[/]')
+		for error in errors:
+			console.print(error)
+		console.print(Error(message=f'Task {name} tests failed. Please fix the issues above.'))
 
+	if warnings:
+		console.print('\n[bold gold3]Warnings:[/]')
+		for warning in warnings:
+			console.print(warning)
+
+	console.print("\n")
 	sys.exit(exit_code)
 
 
-def check_error(condition, message, error, warnings=[]):
+def check_test(condition, message, fail_message, results=[], warn=False):
 	console.print(f'[bold magenta]:zap: {message} ...[/]', end='')
 	if not condition:
-		warning = Warning(message=error)
-		warnings.append(warning)
-		console.print(' [bold red]FAILED[/]', style='dim')
+		if not warn:
+			error = Error(message=fail_message)
+			console.print(' [bold red]FAILED[/]', style='dim')
+			results.append(error)
+		else:
+			warning = Warning(message=fail_message)
+			console.print(' [bold yellow]WARNING[/]', style='dim')
+			results.append(warning)
 	else:
 		console.print(' [bold green]OK[/]', style='dim')
 	return True

secator/config.py

@@ -94,6 +94,7 @@ class Runners(StrictModel):
 	skip_cve_low_confidence: bool = False
 	remove_duplicates: bool = False
 	show_chunk_progress: bool = False
+	show_command_output: bool = False
 
 
 class Security(StrictModel):

secator/configs/workflows/host_recon.yaml

@@ -7,10 +7,19 @@ input_types:
   - host
   - cidr_range
 tasks:
+  naabu:
+    description: Find open ports
+    ports: "-"  # scan all ports
   nmap:
     description: Search for vulnerabilities on open ports
-    skip_host_discovery: True
-    ports: "-"  # scan all ports
+    version_detection: True
+    script: vulners
+    targets_:
+    - port.host
+    - target.name
+    ports_:
+    - port.port
+    ports: "-"  # default if no port found by naabu
   _group/1:
     httpx:
       description: Probe HTTP services on open ports

secator/configs/workflows/port_scan.yaml

@@ -0,0 +1,39 @@
+type: workflow
+name: port_scan
+alias: pscan
+description: Port scan
+tags: [recon, network, http, vuln]
+input_types:
+  - host
+  - cidr_range
+tasks:
+  naabu:
+    description: Find open ports
+    skip_host_discovery: True
+    ports: "-"  # scan all ports
+  nmap:
+    description: Search for vulnerabilities on open ports
+    skip_host_discovery: True
+    version_detection: True
+    targets_: port.host
+    ports_: port.port
+  _group:
+    searchsploit:
+      description: Search for related exploits
+      targets_:
+      - type: port
+        field: '{host}~{service_name}'
+        condition: item._source.startswith('nmap') and len(item.service_name.split('/')) > 1
+    httpx:
+      description: Probe HTTP services on open ports
+      targets_:
+        - type: port
+          field: '{host}:{port}'
+          condition: item._source.startswith('nmap')
+results:
+  - type: port
+
+  - type: url
+    condition: item.status_code != 0
+
+  - type: vulnerability

secator/configs/workflows/url_dirsearch.yaml

@@ -14,6 +14,11 @@ tasks:
         field: '{name}/FUZZ'
   cariddi:
     description: Crawl HTTP directories for content
+    info: True
+    secrets: True
+    errors: True
+    juicy_extensions: 1
+    juicy_endpoints: True
     targets_:
       - target.name
       - url.url

secator/configs/workflows/url_params_fuzz.yaml

@@ -11,6 +11,8 @@ tasks:
   ffuf:
     description: Fuzz URL params
     wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/Web-Content/burp-parameter-names.txt
+    auto_calibration: true
+    follow_redirect: true
     targets_:
       - type: url
         field: url

secator/decorators.py

@@ -19,11 +19,11 @@ RUNNER_OPTS = {
 	'print_stat': {'is_flag': True, 'short': 'stat', 'default': False, 'help': 'Print runtime statistics'},
 	'print_format': {'default': '', 'short': 'fmt', 'help': 'Output formatting string'},
 	'enable_profiler': {'is_flag': True, 'short': 'prof', 'default': False, 'help': 'Enable runner profiling'},
-	'show': {'is_flag': True, 'short': 'sh', 'default': False, 'help': 'Show command that will be run (tasks only)'},
 	'no_process': {'is_flag': True, 'short': 'nps', 'default': False, 'help': 'Disable secator processing'},
 	# 'filter': {'default': '', 'short': 'f', 'help': 'Results filter', 'short': 'of'}, # TODO add this
-	'quiet': {'is_flag': True, 'short': 'q', 'default': False, 'help': 'Enable quiet mode'},
+	'quiet': {'is_flag': True, 'short': 'q', 'default': not CONFIG.runners.show_command_output, 'opposite': 'verbose', 'help': 'Enable quiet mode'},  # noqa: E501
 	'dry_run': {'is_flag': True, 'short': 'dr', 'default': False, 'help': 'Enable dry run'},
+	'show': {'is_flag': True, 'short': 'yml', 'default': False, 'help': 'Show runner yaml'},
 }
 
 RUNNER_GLOBAL_OPTS = {
@@ -163,35 +163,39 @@ def get_command_options(config):
 			elif opt in RUNNER_GLOBAL_OPTS:
 				prefix = 'Execution'
 
+			# Get opt value from YAML config
+			opt_conf_value = task_config_opts.get(opt)
+
 			# Get opt conf
 			conf = opt_conf.copy()
+			opt_is_flag = conf.get('is_flag', False)
+			opt_default = conf.get('default', False if opt_is_flag else None)
+			opt_is_required = conf.get('required', False)
 			conf['show_default'] = True
 			conf['prefix'] = prefix
-			opt_default = conf.get('default', None)
-			opt_is_flag = conf.get('is_flag', False)
-			opt_value_in_config = task_config_opts.get(opt)
+			conf['default'] = opt_default
+			conf['reverse'] = False
 
-			# Check if opt already defined in config
-			if opt_value_in_config:
-				if conf.get('required', False):
+			# Change CLI opt defaults if opt was overriden in YAML config
+			if opt_conf_value:
+				if opt_is_required:
 					debug('OPT (skipped: opt is required and defined in config)', obj={'opt': opt}, sub=f'cli.{config.name}', verbose=True)  # noqa: E501
 					continue
 				mapped_value = cls.opt_value_map.get(opt)
 				if callable(mapped_value):
-					opt_value_in_config = mapped_value(opt_value_in_config)
+					opt_conf_value = mapped_value(opt_conf_value)
 				elif mapped_value:
-					opt_value_in_config = mapped_value
-				if opt_value_in_config != opt_default:
+					opt_conf_value = mapped_value
+
+				# Handle option defaults
+				if opt_conf_value != opt_default:
 					if opt in opt_cache:
 						continue
 					if opt_is_flag:
-						conf['reverse'] = True
-						conf['default'] = not conf['default']
-					# print(f'{opt}: change default to {opt_value_in_config}')
-					conf['default'] = opt_value_in_config
+						conf['default'] = opt_default = opt_conf_value
 
-			# If opt is a flag but the default is True, add opposite flag
-			if opt_is_flag and opt_default is True:
+			# Add reverse flag
+			if opt_default is True:
 				conf['reverse'] = True
 
 			# Check if opt already processed before
@@ -205,7 +209,7 @@ def get_command_options(config):
 			all_opts[opt] = conf
 
 			# Debug
-			debug_conf = OrderedDict({'opt': opt, 'config_val': opt_value_in_config or 'N/A', **conf.copy()})
+			debug_conf = OrderedDict({'opt': opt, 'config_val': opt_conf_value or 'N/A', **conf.copy()})
 			debug('OPT', obj=debug_conf, sub=f'cli.{config.name}', verbose=True)
 
 	return all_opts
@@ -236,11 +240,17 @@ def decorate_command_options(opts):
 			conf.pop('process', None)
 			conf.pop('requires_sudo', None)
 			reverse = conf.pop('reverse', False)
+			opposite = conf.pop('opposite', None)
 			long = f'--{opt_name}'
 			short = f'-{short_opt}' if short_opt else f'-{opt_name}'
 			if reverse:
-				long += f'/--no-{opt_name}'
-				short += f'/-n{short_opt}' if short else f'/-n{opt_name}'
+				if opposite:
+					long += f'/--{opposite}'
+					short += f'/-{opposite[0]}'
+					conf['help'] = conf['help'].replace(opt_name, f'{opt_name} / {opposite}')
+				else:
+					long += f'/--no-{opt_name}'
+					short += f'/-n{short_opt}' if short else f'/-n{opt_name}'
 			f = click.option(long, short, **conf)(f)
 		return f
 	return decorator
@@ -321,9 +331,16 @@ def register_runner(cli_endpoint, config):
 		worker = opts.pop('worker')
 		ws = opts.pop('workspace')
 		driver = opts.pop('driver', '')
+		quiet = opts['quiet']
+		dry_run = opts['dry_run']
 		show = opts['show']
 		context = {'workspace_name': ws}
 
+		# Show runner yaml
+		if show:
+			config.print()
+			sys.exit(0)
+
 		# Remove options whose values are default values
 		for k, v in options.items():
 			opt_name = k.replace('-', '_')
@@ -364,7 +381,7 @@ def register_runner(cli_endpoint, config):
 		hooks = deep_merge_dicts(*hooks)
 
 		# Enable sync or not
-		if sync or show:
+		if sync or dry_run:
 			sync = True
 		else:
 			from secator.celery import is_celery_worker_alive
@@ -394,6 +411,7 @@ def register_runner(cli_endpoint, config):
 			'piped_output': ctx.obj['piped_output'],
 			'caller': 'cli',
 			'sync': sync,
+			'quiet': quiet
 		})
 
 		# Start runner

secator/definitions.py

@@ -23,10 +23,6 @@ ASCII = rf"""
 DEBUG = CONFIG.debug.level
 DEBUG_COMPONENT = CONFIG.debug.component.split(',')
 
-# Default tasks settings
-DEFAULT_NUCLEI_FLAGS = os.environ.get('DEFAULT_NUCLEI_FLAGS', '-stats -sj -si 20 -hm -or')
-DEFAULT_FEROXBUSTER_FLAGS = os.environ.get('DEFAULT_FEROXBUSTER_FLAGS', '--auto-bail --no-state')
-
 # Constants
 OPT_NOT_SUPPORTED = -1
 OPT_PIPE_INPUT = -1