secator 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

secator/utils.py

@@ -1,6 +1,6 @@
 import fnmatch
-import inspect
 import importlib
+import ipaddress
 import itertools
 import json
 import logging
@@ -15,9 +15,7 @@ import warnings
 
 from datetime import datetime, timedelta
 from functools import reduce
-from inspect import isclass
 from pathlib import Path
-from pkgutil import iter_modules
 from time import time
 import traceback
 from urllib.parse import urlparse, quote
@@ -26,7 +24,8 @@ import humanize
 import ifaddr
 import yaml
 
-from secator.definitions import (DEBUG_COMPONENT, VERSION, DEV_PACKAGE)
+from secator.definitions import (DEBUG, VERSION, DEV_PACKAGE, IP, HOST, CIDR_RANGE,
+								 MAC_ADDRESS, SLUG, UUID, EMAIL, IBAN, URL, PATH, HOST_PORT)
 from secator.config import CONFIG, ROOT_FOLDER, LIB_FOLDER, download_file
 from secator.rich import console
 
@@ -73,18 +72,21 @@ def expand_input(input, ctx):
 	Returns:
 		str: Input.
 	"""
+	piped_input = ctx.obj['piped_input']
+	dry_run = ctx.obj['dry_run']
 	if input is None:  # read from stdin
-		if not ctx.obj['piped_input']:
-			console.print('Waiting for input on stdin ...', style='bold yellow')
-		rlist, _, _ = select.select([sys.stdin], [], [], CONFIG.cli.stdin_timeout)
-		if rlist:
-			data = sys.stdin.read().splitlines()
-		else:
-			console.print(
-				'No input passed on stdin. Showing help page.',
-				style='bold red')
-			return None
-		return data
+		if not piped_input and not dry_run:
+			console.print('No input passed on stdin. Showing help page.', style='bold red')
+			ctx.get_help()
+			sys.exit(1)
+		elif piped_input:
+			rlist, _, _ = select.select([sys.stdin], [], [], CONFIG.cli.stdin_timeout)
+			if rlist:
+				data = sys.stdin.read().splitlines()
+				return data
+			else:
+				console.print('No input passed on stdin.', style='bold red')
+				sys.exit(1)
 	elif os.path.exists(input):
 		if os.path.isfile(input):
 			with open(input, 'r') as f:
@@ -99,6 +101,9 @@ def expand_input(input, ctx):
 	if isinstance(input, list) and len(input) == 1:
 		return input[0]
 
+	if ctx.obj['dry_run'] and not input:
+		return ['TARGET']
+
 	return input
 
 
@@ -140,75 +145,6 @@ def deduplicate(array, attr=None):
 	return sorted(list(dict.fromkeys(array)))
 
 
-def discover_internal_tasks():
-	"""Find internal secator tasks."""
-	from secator.runners import Runner
-	package_dir = Path(__file__).resolve().parent / 'tasks'
-	task_classes = []
-	for (_, module_name, _) in iter_modules([str(package_dir)]):
-		if module_name.startswith('_'):
-			continue
-		try:
-			module = importlib.import_module(f'secator.tasks.{module_name}')
-		except ImportError as e:
-			console.print(f'[bold red]Could not import secator.tasks.{module_name}:[/]')
-			console.print(f'\t[bold red]{type(e).__name__}[/]: {str(e)}')
-			continue
-		for attribute_name in dir(module):
-			attribute = getattr(module, attribute_name)
-			if isclass(attribute):
-				bases = inspect.getmro(attribute)
-				if Runner in bases and hasattr(attribute, '__task__'):
-					task_classes.append(attribute)
-
-	# Sort task_classes by category
-	task_classes = sorted(
-		task_classes,
-		# key=lambda x: (get_command_category(x), x.__name__)
-		key=lambda x: x.__name__)
-
-	return task_classes
-
-
-def discover_external_tasks():
-	"""Find external secator tasks."""
-	output = []
-	sys.dont_write_bytecode = True
-	for path in CONFIG.dirs.templates.glob('**/*.py'):
-		try:
-			task_name = path.stem
-			module_name = f'secator.tasks.{task_name}'
-
-			# console.print(f'Importing module {module_name} from {path}')
-			spec = importlib.util.spec_from_file_location(module_name, path)
-			module = importlib.util.module_from_spec(spec)
-			# console.print(f'Adding module "{module_name}" to sys path')
-			sys.modules[module_name] = module
-
-			# console.print(f'Executing module "{module}"')
-			spec.loader.exec_module(module)
-
-			# console.print(f'Checking that {module} contains task {task_name}')
-			if not hasattr(module, task_name):
-				console.print(f'[bold orange1]Could not load external task "{task_name}" from module {path.name}[/] ({path})')
-				continue
-			cls = getattr(module, task_name)
-			console.print(f'[bold green]Successfully loaded external task "{task_name}"[/] ({path})')
-			output.append(cls)
-		except Exception as e:
-			console.print(f'[bold red]Could not load external module {path.name}. Reason: {str(e)}.[/] ({path})')
-	sys.dont_write_bytecode = False
-	return output
-
-
-def discover_tasks():
-	"""Find all secator tasks (internal + external)."""
-	global _tasks
-	if not _tasks:
-		_tasks = discover_internal_tasks() + discover_external_tasks()
-	return _tasks
-
-
 def import_dynamic(path, name=None):
 	"""Import class or module dynamically from path.
 
@@ -238,22 +174,6 @@ def import_dynamic(path, name=None):
 		return None
 
 
-def get_command_cls(cls_name):
-	"""Get secator command by class name.
-
-	Args:
-		cls_name (str): Class name to load.
-
-	Returns:
-		cls: Class.
-	"""
-	tasks_classes = discover_tasks()
-	for task_cls in tasks_classes:
-		if task_cls.__name__ == cls_name:
-			return task_cls
-	return None
-
-
 def get_command_category(command):
 	"""Get the category of a command.
 
@@ -383,7 +303,7 @@ def rich_to_ansi(text):
 			tmp_console.print(text, end='', soft_wrap=True)
 		return capture.get()
 	except Exception:
-		console.print(f'[bold red]Could not convert rich text to ansi: {text}[/]', highlight=False, markup=False)
+		print(f'Could not convert rich text to ansi: {text}[/]', file=sys.stderr)
 		return text
 
 
@@ -397,11 +317,11 @@ def rich_escape(obj):
 		any: Initial object, or escaped Rich string.
 	"""
 	if isinstance(obj, str):
-		return obj.replace('[', r'\[').replace(']', r'\]')
+		return obj.replace('[', r'\[').replace(']', r'\]').replace(r'\[/', r'\[\/')
 	return obj
 
 
-def format_object(obj, obj_breaklines=False):
+def format_debug_object(obj, obj_breaklines=False):
 	"""Format the debug object for printing.
 
 	Args:
@@ -413,7 +333,7 @@ def format_object(obj, obj_breaklines=False):
 	"""
 	sep = '\n ' if obj_breaklines else ', '
 	if isinstance(obj, dict):
-		return sep.join(f'[dim cyan]{k}[/] [dim yellow]->[/] [dim green]{v}[/]' for k, v in obj.items() if v is not None)  # noqa: E501
+		return sep.join(f'[bold blue]{k}[/] [yellow]->[/] [blue]{v}[/]' for k, v in obj.items() if v is not None)  # noqa: E501
 	elif isinstance(obj, list):
 		return f'[dim green]{sep.join(obj)}[/]'
 	return ''
@@ -421,21 +341,25 @@ def format_object(obj, obj_breaklines=False):
 
 def debug(msg, sub='', id='', obj=None, lazy=None, obj_after=True, obj_breaklines=False, verbose=False):
 	"""Print debug log if DEBUG >= level."""
-	if not DEBUG_COMPONENT == ['all']:
-		if not DEBUG_COMPONENT or DEBUG_COMPONENT == [""]:
+	if not DEBUG == ['all'] and not DEBUG == ['1']:
+		if not DEBUG or DEBUG == [""]:
 			return
-
 		if sub:
-			if verbose and sub not in DEBUG_COMPONENT:
-				sub = f'debug.{sub}'
-			if not any(sub.startswith(s) for s in DEBUG_COMPONENT):
+			for s in DEBUG:
+				if '*' in s and re.match(s + '$', sub):
+					break
+				elif not verbose and sub.startswith(s):
+					break
+				elif verbose and sub == s:
+					break
+			else:
 				return
 
 	if lazy:
 		msg = lazy(msg)
 
 	formatted_msg = f'[yellow4]{sub:13s}[/] ' if sub else ''
-	obj_str = format_object(obj, obj_breaklines) if obj else ''
+	obj_str = format_debug_object(obj, obj_breaklines) if obj else ''
 
 	# Constructing the message string based on object position
 	if obj_str and not obj_after:
@@ -446,7 +370,12 @@ def debug(msg, sub='', id='', obj=None, lazy=None, obj_after=True, obj_breakline
 	if id:
 		formatted_msg += rf' [italic gray11]\[{id}][/]'
 
-	console.print(rf'[dim]\[[magenta4]DBG[/]] {formatted_msg}[/]')
+	try:
+		console.print(rf'[dim]\[[magenta4]DBG[/]] {formatted_msg}[/]', highlight=False)
+	except Exception:
+		console.print(rf'[dim]\[[magenta4]DBG[/]] <MARKUP_DISABLED>{rich_escape(formatted_msg)}</MARKUP_DISABLED>[/]', highlight=False)  # noqa: E501
+		if 'rich' in DEBUG:
+			raise
 
 
 def escape_mongodb_url(url):
@@ -498,6 +427,7 @@ def extract_domain_info(input, domain_only=False):
 
 	Args:
 		input (str): An URL or FQDN.
+		domain_only (bool): Return only the registered domain name.
 
 	Returns:
 		tldextract.ExtractResult: Extracted info.
@@ -507,9 +437,9 @@ def extract_domain_info(input, domain_only=False):
 	if not result or not result.domain or not result.suffix:
 		return None
 	if domain_only:
-		if not validators.domain(result.registered_domain):
+		if not validators.domain(result.top_domain_under_public_suffix):
 			return None
-		return result.registered_domain
+		return result.top_domain_under_public_suffix
 	return result
 
 
@@ -787,15 +717,12 @@ def process_wordlist(val):
 	if template_wordlist:
 		val = template_wordlist
 
-	if Path(val).exists():
-		return val
-	else:
-		return download_file(
-			val,
-			target_folder=CONFIG.dirs.wordlists,
-			offline_mode=CONFIG.offline_mode,
-			type='wordlist'
-		)
+	return download_file(
+		val,
+		target_folder=CONFIG.dirs.wordlists,
+		offline_mode=CONFIG.offline_mode,
+		type='wordlist'
+	)
 
 
 def convert_functions_to_strings(data):
@@ -815,3 +742,84 @@ def convert_functions_to_strings(data):
 		return json.dumps(data.__name__)  # or use inspect.getsource(data) if you want the actual function code
 	else:
 		return data
+
+
+def headers_to_dict(header_opt):
+	headers = {}
+	for header in header_opt.split(';;'):
+		split = header.strip().split(':')
+		key = split[0].strip()
+		val = ':'.join(split[1:]).strip()
+		headers[key] = val
+	return headers
+
+
+def format_object(obj, color='magenta', skip_keys=[]):
+	if isinstance(obj, list) and obj:
+		return ' [' + ', '.join([f'[{color}]{rich_escape(item)}[/]' for item in obj]) + ']'
+	elif isinstance(obj, dict) and obj.keys():
+		obj = {k: v for k, v in obj.items() if k.lower().replace('-', '_') not in skip_keys}
+		if obj:
+			return ' [' + ', '.join([f'[bold {color}]{rich_escape(k)}[/]: [{color}]{rich_escape(v)}[/]' for k, v in obj.items()]) + ']'  # noqa: E501
+	return ''
+
+
+def autodetect_type(target):
+	"""Autodetect the type of a target.
+
+	Args:
+		target (str): The target to autodetect the type of.
+
+	Returns:
+		str: The type of the target.
+	"""
+	if validators.url(target, simple_host=True):
+		return URL
+	elif validate_cidr_range(target):
+		return CIDR_RANGE
+	elif validators.ipv4(target) or validators.ipv6(target) or target == 'localhost':
+		return IP
+	elif validators.domain(target):
+		return HOST
+	elif validators.domain(target.split(':')[0]):
+		return HOST_PORT
+	elif validators.mac_address(target):
+		return MAC_ADDRESS
+	elif validators.email(target):
+		return EMAIL
+	elif validators.iban(target):
+		return IBAN
+	elif validators.uuid(target):
+		return UUID
+	elif Path(target).exists():
+		return PATH
+	elif validators.slug(target):
+		return SLUG
+
+	return str(type(target).__name__).lower()
+
+
+def validate_cidr_range(target):
+	if '/' not in target:
+		return False
+	try:
+		ipaddress.ip_network(target, False)
+		return True
+	except ValueError:
+		return False
+
+
+def get_versions_from_string(string):
+	"""Get versions from a string.
+
+	Args:
+		string (str): String to get versions from.
+
+	Returns:
+		list[str]: List of versions.
+	"""
+	regex = r'v?[0-9]+\.[0-9]+\.?[0-9]*\.?[a-zA-Z]*'
+	matches = re.findall(regex, string)
+	if not matches:
+		return []
+	return matches

secator/utils_test.py

@@ -1,6 +1,7 @@
 import contextlib
 import json
 import os
+import re
 import sys
 import unittest.mock
 
@@ -11,11 +12,11 @@ from secator.definitions import (CIDR_RANGE, DELAY, DEPTH, EMAIL,
 							   METHOD, PROXY, RATE_LIMIT, RETRIES,
 							   THREADS, TIMEOUT, URL, USER_AGENT, USERNAME, PATH,
 							   DOCKER_IMAGE, GIT_REPOSITORY)
-from secator.cli import ALL_WORKFLOWS, ALL_TASKS, ALL_SCANS
+from secator.loader import get_configs_by_type
 from secator.output_types import EXECUTION_TYPES, STAT_TYPES
-from secator.runners import Command
+from secator.runners import Command, Task
 from secator.rich import console
-from secator.utils import load_fixture, debug
+from secator.utils import load_fixture, debug, traceback_as_string
 
 #---------#
 # GLOBALS #
@@ -24,33 +25,37 @@ USE_PROXY = bool(int(os.environ.get('USE_PROXY', '0')))
 TEST_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/tests/'
 FIXTURES_DIR = f'{TEST_DIR}/fixtures'
 USE_PROXY = bool(int(os.environ.get('USE_PROXY', '0')))
+TASKS = get_configs_by_type('task')
+WORKFLOWS = get_configs_by_type('workflow')
+SCANS = get_configs_by_type('scan')
+
 
 #------------#
 # TEST TASKS #
 #------------#
 TEST_TASKS = os.environ.get('TEST_TASKS', '')
 if TEST_TASKS:
-	TEST_TASKS = [cls for cls in ALL_TASKS if cls.__name__ in TEST_TASKS.split(',')]
+	TEST_TASKS = [config for config in TASKS if config.name in TEST_TASKS.split(',')]
 else:
-	TEST_TASKS = ALL_TASKS
+	TEST_TASKS = TASKS
 
 #----------------#
 # TEST WORKFLOWS #
 #----------------#
 TEST_WORKFLOWS = os.environ.get('TEST_WORKFLOWS', '')
 if TEST_WORKFLOWS:
-	TEST_WORKFLOWS = [config for config in ALL_WORKFLOWS if config.name in TEST_WORKFLOWS.split(',')]
+	TEST_WORKFLOWS = [config for config in WORKFLOWS if config.name in TEST_WORKFLOWS.split(',')]
 else:
-	TEST_WORKFLOWS = ALL_WORKFLOWS
+	TEST_WORKFLOWS = WORKFLOWS
 
 #------------#
 # TEST SCANS #
 #------------#
 TEST_SCANS = os.environ.get('TEST_SCANS', '')
 if TEST_SCANS:
-	TEST_SCANS = [config for config in ALL_SCANS if config.name in TEST_SCANS.split(',')]
+	TEST_SCANS = [config for config in SCANS if config.name in TEST_SCANS.split(',')]
 else:
-	TEST_SCANS = ALL_SCANS
+	TEST_SCANS = SCANS
 
 #-------------------#
 # TEST INPUTS_TASKS #
@@ -71,15 +76,16 @@ INPUTS_TASKS = {
 # TEST FIXTURES_TASKS #
 #---------------------#
 FIXTURES_TASKS = {
-	tool_cls: load_fixture(f'{tool_cls.__name__}_output', FIXTURES_DIR)
-	for tool_cls in TEST_TASKS
+	Task.get_task_class(task.name): load_fixture(f'{task.name}_output', FIXTURES_DIR)
+	for task in TASKS
+	if task.name in [t.name for t in TEST_TASKS]
 }
 
 #-----------#
 # TEST OPTS #
 #-----------#
 META_OPTS = {
-	HEADER: 'User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1',
+	HEADER: 'User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1;; Hello: World',
 	DELAY: 0,
 	DEPTH: 2,
 	FOLLOW_REDIRECT: True,
@@ -159,6 +165,10 @@ def mock_command(cls, inputs=[], opts={}, fixture=None, method=''):
 
 class CommandOutputTester:  # Mixin for unittest.TestCase
 
+	@staticmethod
+	def get_item_str(item):
+		return f"Item: {repr(item)}\nItem dict: {json.dumps(item.toDict(), default=str, indent=2)}"
+
 	def _test_runner_output(
 			self,
 			runner,
@@ -166,10 +176,12 @@ class CommandOutputTester:  # Mixin for unittest.TestCase
 			expected_output_types=[],
 			expected_results=[],
 			expected_status='SUCCESS',
-			empty_results_allowed=False):
+			empty_results_allowed=False,
+			additional_checks=[]):
 
 		console.print(f'\t[dim]Testing {runner.config.type} {runner.name} ...[/]', end='')
 		debug('', sub='unittest')
+		debug('-' * 10 + f' RUNNER {runner.name} STARTING ' + '-' * 10, sub='unittest')
 
 		if not runner.inputs:
 			console.print('[dim gold3] skipped (no inputs defined).[/]')
@@ -183,8 +195,11 @@ class CommandOutputTester:  # Mixin for unittest.TestCase
 
 			# Run runner
 			results = runner.run()
-			for result in results:
-				debug(result.toDict(), sub='unittest')
+			results_str = "\n".join([repr(r) for r in results])
+			debug(f'{runner.name} yielded results\n{results_str}', sub='unittest')
+			debug(f'{runner.name} yielded results\n{json.dumps([r.toDict() for r in results], default=str, indent=2)}', sub='unittest.dict', verbose=True)  # noqa: E501
+
+			debug('-' * 10 + f' RUNNER {runner.name} TESTS ' + '-' * 10, sub='unittest')
 
 			# Add execution types to allowed output types
 			expected_output_types.extend(EXECUTION_TYPES + STAT_TYPES)
@@ -205,13 +220,17 @@ class CommandOutputTester:  # Mixin for unittest.TestCase
 			self.assertEqual(runner.status, expected_status, f'{runner.name} should have the status {expected_status}. Errors: {runner.errors}')  # noqa: E501
 
 			# Check results
+			failures = []
+			debug('-' * 10 + f' RUNNER {runner.name} ITEM TESTS ' + '-' * 10, sub='unittest')
 			for item in results:
-				debug(f'{runner.name} yielded {repr(item)}', sub='unittest')
-				debug(f'{runner.name} yielded (JSON): {json.dumps(item.toDict(), default=str)}', sub='unittest.dict', verbose=True)
+				item_str = self.get_item_str(item)
+				debug('--' * 5, sub='unittest')
+				debug(f'{runner.name} item {repr(item)}', sub='unittest')
+				debug(f'{runner.name} item [{item.toDict()}]', sub='unittest.item', verbose=True)
 
 				if expected_output_types:
 					debug(f'{runner.name} item should have an output type in {[_._type for _ in expected_output_types]}', sub='unittest')  # noqa: E501
-					self.assertIn(type(item), expected_output_types, f'{runner.name}: item has an unexpected output type "{type(item)}"')  # noqa: E501
+					self.assertIn(type(item), expected_output_types, f'{runner.name}: item has an unexpected output type "{type(item)}". Expected types: {expected_output_types}.\n{item_str}')  # noqa: E501
 
 				if expected_output_keys:
 					keys = [k for k in list(item.keys()) if not k.startswith('_')]
@@ -219,7 +238,29 @@ class CommandOutputTester:  # Mixin for unittest.TestCase
 					self.assertEqual(
 						set(keys).difference(set(expected_output_keys)),
 						set(),
-						f'{runner.name}: item is missing expected keys {set(expected_output_keys)}. Item keys: {keys}')  # noqa: E501
+						f'{runner.name}: item is missing expected keys {set(expected_output_keys)}.\nItem keys: {keys}.\n{item_str}')  # noqa: E501
+
+				if additional_checks and item.__class__ in additional_checks.get('output_types', {}):
+					config = additional_checks['output_types'][item.__class__]
+					runner_regex = config.get('runner', '*')
+					if not re.match(runner_regex, runner.name):
+						continue
+					checks = config.get('checks', [])
+					for check in checks:
+						error = check['error']
+						info = check['info']
+						func = check['function']
+						debug(f'{runner.name} item {info}', sub='unittest')
+						try:
+							result = func(item)
+							if not result:
+								failures.append(f'ERROR ({runner.name}): {error}.\n{item_str}')
+						except Exception as e:
+							failures.append(f'ERROR ({runner.name}): {error}.\n{item_str}\n{traceback_as_string(e)}')
+
+			# Additional checks failures
+			if failures:
+				self.fail("\n\n" + "\n\n".join(failures))
 
 			# Check if runner results in expected results
 			if expected_results:

secator/workflows/__init__.py

@@ -1,4 +1,4 @@
-from secator.cli import ALL_WORKFLOWS
+from secator.loader import get_configs_by_type
 from secator.runners import Workflow
 
 
@@ -21,7 +21,7 @@ class DynamicWorkflow(Workflow):
 
 
 DYNAMIC_WORKFLOWS = {}
-for workflow in ALL_WORKFLOWS:
+for workflow in get_configs_by_type('workflow'):
 	instance = DynamicWorkflow(workflow)
 	DYNAMIC_WORKFLOWS[workflow.name] = instance