secator 0.0.1__py3-none-any.whl

secator/runners/task.py

@@ -0,0 +1,106 @@
+from secator.definitions import DEBUG
+from secator.output_types import Target
+from secator.runners import Runner
+from secator.utils import discover_tasks
+
+
+class Task(Runner):
+	default_exporters = []
+	enable_hooks = False
+
+	def delay(cls, *args, **kwargs):
+		from secator.celery import run_task
+		return run_task.apply_async(kwargs={'args': args, 'kwargs': kwargs}, queue='celery')
+
+	def yielder(self):
+		"""Run task.
+
+		Args:
+			sync (bool): Run in sync mode (main thread). If False, run in Celery worker in distributed mode.
+
+		Returns:
+			list: List of results.
+		"""
+		# Get task class
+		task_cls = Task.get_task_class(self.config.name)
+
+		# Task opts
+		run_opts = self.run_opts.copy()
+		run_opts.pop('output', None)
+		dry_run = run_opts.get('show', False)
+		if dry_run:
+			self.print_item_count = False
+
+		# Fmt opts
+		fmt_opts = {
+			'json': run_opts.get('json', False),
+			'print_cmd': True,
+			'print_cmd_prefix': not self.sync,
+			'print_input_file': DEBUG > 0,
+			'print_item': True,
+			'print_item_count': not self.sync and not dry_run,
+			'print_line': not self.output_quiet,
+		}
+		# self.print_item = not self.sync  # enable print_item for base Task only if running remote
+		run_opts.update(fmt_opts)
+
+		# Set task output types
+		self.output_types = task_cls.output_types
+
+		# Get hooks
+		hooks = {task_cls: self.hooks}
+		run_opts['hooks'] = hooks
+		run_opts['context'] = self.context
+
+		# Run task
+		if self.sync:
+			task = task_cls(self.targets, **run_opts)
+			if dry_run:  # don't run
+				return
+		else:
+			result = task_cls.delay(self.targets, **run_opts)
+			task = self.process_live_tasks(
+				result,
+				description=False,
+				results_only=True,
+				print_remote_status=self.print_remote_status)
+
+		# Yield task results
+		yield from task
+
+		# Yield targets
+		for target in self.targets:
+			yield Target(name=target, _source=self.config.name, _type='target', _context=self.context)
+
+	@staticmethod
+	def get_task_class(name):
+		"""Get task class from a name.
+
+		Args:
+			name (str): Task name.
+		"""
+		if '/' in name:
+			name = name.split('/')[0]
+		tasks_classes = discover_tasks()
+		for task_cls in tasks_classes:
+			if task_cls.__name__ == name:
+				return task_cls
+		raise ValueError(f'Task {name} not found. Aborting.')
+
+	@staticmethod
+	def get_tasks_from_conf(config):
+		"""Get task names from config. Ignore hierarchy and keywords.
+
+		TODO: Add hierarchy tree / add make flow diagrams.
+		"""
+		tasks = []
+		for name, opts in config.items():
+			if name == '_group':
+				tasks.extend(Task.get_tasks_from_conf(opts))
+			elif name == '_chain':
+				tasks.extend(Task.get_tasks_from_conf(opts))
+			else:
+				if '/' in name:
+					name = name.split('/')[0]
+				tasks.append(name)
+		return tasks

secator/runners/workflow.py

@@ -0,0 +1,135 @@
+from celery import chain, chord
+
+from secator.definitions import DEBUG
+from secator.exporters import CsvExporter, JsonExporter
+from secator.output_types import Target
+from secator.runners._base import Runner
+from secator.runners.task import Task
+from secator.utils import merge_opts
+
+
+class Workflow(Runner):
+
+	default_exporters = [
+		JsonExporter,
+		CsvExporter
+	]
+
+	@classmethod
+	def delay(cls, *args, **kwargs):
+		from secator.celery import run_workflow
+		return run_workflow.delay(args=args, kwargs=kwargs)
+
+	def yielder(self):
+		"""Run workflow.
+
+		Args:
+			sync (bool): Run in sync mode (main thread). If False, run in Celery worker in distributed mode.
+
+		Returns:
+			list: List of results.
+		"""
+		# Yield targets
+		for target in self.targets:
+			yield Target(name=target, _source=self.config.name, _type='target', _context=self.context)
+
+		# Task fmt opts
+		run_opts = self.run_opts.copy()
+		fmt_opts = {
+			'json': run_opts.get('json', False),
+			'print_cmd': True,
+			'print_cmd_prefix': not self.sync,
+			'print_description': self.sync,
+			'print_input_file': DEBUG,
+			'print_item': True,
+			'print_item_count': True,
+			'print_line': not self.sync,
+		}
+
+		# Construct run opts
+		run_opts['hooks'] = self._hooks.get(Task, {})
+		run_opts.update(fmt_opts)
+
+		# Build Celery workflow
+		workflow = self.build_celery_workflow(run_opts=run_opts, results=self.results)
+
+		# Run Celery workflow and get results
+		if self.sync:
+			results = workflow.apply().get()
+		else:
+			result = workflow()
+			self.result = result
+			results = self.process_live_tasks(result, results_only=True, print_remote_status=self.print_remote_status)
+
+		# Get workflow results
+		yield from results
+
+	def build_celery_workflow(self, run_opts={}, results=[]):
+		""""Build Celery workflow.
+
+		Returns:
+			celery.chain: Celery task chain.
+		"""
+		from secator.celery import forward_results
+		sigs = self.get_tasks(
+			self.config.tasks.toDict(),
+			self.targets,
+			self.config.options,
+			run_opts)
+		sigs = [forward_results.si(results).set(queue='io')] + sigs + [forward_results.s().set(queue='io')]
+		workflow = chain(*sigs)
+		return workflow
+
+	def get_tasks(self, obj, targets, workflow_opts, run_opts):
+		"""Get tasks recursively as Celery chains / chords.
+
+		Args:
+			obj (secator.config.ConfigLoader): Config.
+			targets (list): List of targets.
+			workflow_opts (dict): Workflow options.
+			run_opts (dict): Run options.
+			sync (bool): Synchronous mode (chain of tasks, no chords).
+
+		Returns:
+			list: List of signatures.
+		"""
+		from secator.celery import forward_results
+		sigs = []
+		for task_name, task_opts in obj.items():
+			# Task opts can be None
+			task_opts = task_opts or {}
+
+			# If it's a group, process the sublevel tasks as a Celery chord.
+			if task_name == '_group':
+				tasks = self.get_tasks(
+					task_opts,
+					targets,
+					workflow_opts,
+					run_opts
+				)
+				sig = chord((tasks), forward_results.s().set(queue='io'))
+			elif task_name == '_chain':
+				tasks = self.get_tasks(
+					task_opts,
+					targets,
+					workflow_opts,
+					run_opts
+				)
+				sig = chain(*tasks)
+			else:
+				# Get task class
+				task = Task.get_task_class(task_name)
+
+				# Merge task options (order of priority with overrides)
+				opts = merge_opts(workflow_opts, task_opts, run_opts)
+
+				# Add task context and hooks to options
+				opts['hooks'] = {task: self._hooks.get(Task, {})}
+				opts['context'] = self.context.copy()
+				opts['name'] = task_name
+
+				# Create task signature
+				sig = task.s(targets, **opts).set(queue=task.profile)
+				self.output_types.extend(task.output_types)
+			sigs.append(sig)
+		return sigs

secator/serializers/__init__.py

@@ -0,0 +1,8 @@
+__all__ = [
+    'JSONSerializer',
+    'RegexSerializer',
+    'DataclassEncoder',
+]
+from secator.serializers.json import JSONSerializer
+from secator.serializers.regex import RegexSerializer
+from secator.serializers.dataclass import DataclassEncoder

secator/serializers/dataclass.py

@@ -0,0 +1,33 @@
+import json
+from secator.output_types import OUTPUT_TYPES
+
+
+class DataclassEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if hasattr(obj, 'toDict'):
+            return obj.toDict()
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+
+def get_output_cls(type):
+    try:
+        return [cls for cls in OUTPUT_TYPES if cls.get_name() == type][0]
+    except IndexError:
+        return None
+
+
+def dataclass_decoder(obj):
+    if '_type' in obj:
+        output_cls = get_output_cls(obj['_type'])
+        if output_cls:
+            return output_cls.load(obj)
+    return obj
+
+
+def dumps_dataclass(obj, indent=None):
+    return json.dumps(obj, cls=DataclassEncoder, indent=indent)
+
+
+def loads_dataclass(obj):
+    return json.loads(obj, object_hook=dataclass_decoder)

secator/serializers/json.py

@@ -0,0 +1,15 @@
+import yaml
+
+
+class JSONSerializer:
+
+	def run(self, line):
+		start_index = line.find('{')
+		end_index = line.rfind('}')
+		if start_index == -1 or end_index == -1:
+			return None
+		try:
+			json_obj = line[start_index:end_index+1]
+			return yaml.safe_load(json_obj)
+		except yaml.YAMLError:
+			return None

secator/serializers/regex.py

@@ -0,0 +1,17 @@
+import re
+
+
+class RegexSerializer:
+
+	def __init__(self, regex, fields=[]):
+		self.regex = re.compile(regex)
+		self.fields = fields
+
+	def run(self, line):
+		match = self.regex.match(line)
+		output = {}
+		if not match:
+			return None
+		for field in self.fields:
+			output[field] = match.group(field)
+		return output

secator/tasks/__init__.py

@@ -0,0 +1,10 @@
+from secator.utils import discover_internal_tasks, discover_external_tasks
+INTERNAL_TASKS = discover_internal_tasks()
+EXTERNAL_TASKS = discover_external_tasks()
+ALL_TASKS = INTERNAL_TASKS + EXTERNAL_TASKS
+__all__ = [
+    cls.__name__
+    for cls in ALL_TASKS
+]
+for cls in INTERNAL_TASKS:
+    exec(f'from .{cls.__name__} import {cls.__name__}')

secator/tasks/_categories.py

@@ -0,0 +1,304 @@
+import json
+import logging
+import os
+
+import requests
+from bs4 import BeautifulSoup
+from cpe import CPE
+
+from secator.definitions import (CIDR_RANGE, CONFIDENCE, CVSS_SCORE,
+							   DEFAULT_HTTP_WORDLIST, DELAY, DEPTH, DESCRIPTION,
+							   FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
+							   FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID,
+							   MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
+							   MATCH_WORDS, METHOD, NAME, PATH, PROVIDER,
+							   PROXY, RATE_LIMIT, REFERENCES, RETRIES,
+							   SEVERITY, TAGS, DATA_FOLDER, THREADS, TIMEOUT,
+							   URL, USER_AGENT, USERNAME, WORDLIST)
+from secator.output_types import (Ip, Port, Subdomain, Tag, Url, UserAccount,
+								Vulnerability)
+from secator.runners import Command
+
+logger = logging.getLogger(__name__)
+
+OPTS = {
+	HEADER: {'type': str, 'help': 'Custom header to add to each request in the form "KEY1:VALUE1; KEY2:VALUE2"'},
+	DELAY: {'type': float, 'short': 'd', 'help': 'Delay to add between each requests'},
+	DEPTH: {'type': int, 'help': 'Scan depth', 'default': 2},
+	FILTER_CODES: {'type': str, 'short': 'fc', 'help': 'Filter out responses with HTTP codes'},
+	FILTER_REGEX: {'type': str, 'short': 'fr', 'help': 'Filter out responses with regular expression'},
+	FILTER_SIZE: {'type': str, 'short': 'fs', 'help': 'Filter out responses with size'},
+	FILTER_WORDS: {'type': str, 'short': 'fw', 'help': 'Filter out responses with word count'},
+	FOLLOW_REDIRECT: {'is_flag': True, 'short': 'frd', 'help': 'Follow HTTP redirects'},
+	MATCH_CODES: {'type': str, 'short': 'mc', 'help': 'Match HTTP status codes e.g "201,300,301"'},
+	MATCH_REGEX: {'type': str, 'short': 'mr', 'help': 'Match responses with regular expression'},
+	MATCH_SIZE: {'type': str, 'short': 'ms', 'help': 'Match respones with size'},
+	MATCH_WORDS: {'type': str, 'short': 'mw', 'help': 'Match responses with word count'},
+	METHOD: {'type': str, 'help': 'HTTP method to use for requests'},
+	PROXY: {'type': str, 'help': 'HTTP(s) / SOCKS5 proxy'},
+	RATE_LIMIT: {'type':  int, 'short': 'rl', 'help': 'Rate limit, i.e max number of requests per second'},
+	RETRIES: {'type': int, 'help': 'Retries'},
+	THREADS: {'type': int, 'help': 'Number of threads to run', 'default': 50},
+	TIMEOUT: {'type': int, 'help': 'Request timeout'},
+	USER_AGENT: {'type': str, 'short': 'ua', 'help': 'User agent, e.g "Mozilla Firefox 1.0"'},
+	WORDLIST: {'type': str, 'short': 'w', 'default': DEFAULT_HTTP_WORDLIST, 'help': 'Wordlist to use'}
+}
+
+OPTS_HTTP = [
+	HEADER, DELAY, FOLLOW_REDIRECT, METHOD, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT
+]
+
+OPTS_HTTP_CRAWLERS = OPTS_HTTP + [
+	DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT,
+	MATCH_CODES
+]
+
+OPTS_HTTP_FUZZERS = OPTS_HTTP_CRAWLERS + [WORDLIST]
+
+OPTS_RECON = [
+	DELAY, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT
+]
+
+OPTS_VULN = [
+	HEADER, DELAY, FOLLOW_REDIRECT, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT
+]
+
+OPTS_OSINT = [
+
+]
+
+
+#---------------#
+# HTTP category #
+#---------------#
+
+class Http(Command):
+	meta_opts = {k: OPTS[k] for k in OPTS_HTTP_CRAWLERS}
+	input_type = URL
+	output_types = [Url]
+
+
+class HttpCrawler(Command):
+	meta_opts = {k: OPTS[k] for k in OPTS_HTTP_CRAWLERS}
+	input_type = URL
+	output_types = [Url]
+
+
+class HttpFuzzer(Command):
+	meta_opts = {k: OPTS[k] for k in OPTS_HTTP_FUZZERS}
+	input_type = URL
+	output_types = [Url]
+
+
+#----------------#
+# Recon category #
+#----------------#
+
+class Recon(Command):
+	meta_opts = {k: OPTS[k] for k in OPTS_RECON}
+	output_types = [Subdomain, UserAccount, Ip, Port]
+
+
+class ReconDns(Recon):
+	input_type = HOST
+	output_types = [Subdomain]
+
+
+class ReconUser(Recon):
+	input_type = USERNAME
+	output_types = [UserAccount]
+
+
+class ReconIp(Recon):
+	input_type = CIDR_RANGE
+	output_types = [Ip]
+
+
+class ReconPort(Recon):
+	input_type = HOST
+	output_types = [Port]
+
+
+#---------------#
+# Vuln category #
+#---------------#
+
+class Vuln(Command):
+	meta_opts = {k: OPTS[k] for k in OPTS_VULN}
+	output_types = [Vulnerability]
+
+	@staticmethod
+	def lookup_local_cve(cve_id):
+		cve_path = f'{DATA_FOLDER}/cves/{cve_id}.json'
+		if os.path.exists(cve_path):
+			with open(cve_path, 'r') as f:
+				return json.load(f)
+		return None
+
+	# @staticmethod
+	# def lookup_exploitdb(exploit_id):
+	# 	print('looking up exploit')
+	# 	try:
+	# 		cve_info = requests.get(f'https://exploit-db.com/exploits/{exploit_id}', timeout=5).content
+	# 		print(cve_info)
+	# 	except Exception:
+	# 		logger.error(f'Could not fetch exploit info for exploit {exploit_id}. Skipping.')
+	# 		return None
+	# 	return cve_info
+
+	@staticmethod
+	def lookup_cve(cve_id, cpes=[]):
+		"""Search for a CVE in local db or using cve.circl.lu and return vulnerability data.
+
+		Args:
+			cve_id (str): CVE ID in the form CVE-*
+			cpes (str, Optional): CPEs to match for.
+
+		Returns:
+			dict: vulnerability data.
+		"""
+		cve_info = Vuln.lookup_local_cve(cve_id)
+		if not cve_info:
+			# logger.debug(f'{cve_id} not found locally. Use `secator utils download-cves` to update the local database.')
+			try:
+				cve_info = requests.get(f'https://cve.circl.lu/api/cve/{cve_id}', timeout=5).json()
+				if not cve_info:
+					logger.error(f'Could not fetch CVE info for cve {cve_id}. Skipping.')
+					return
+			except Exception:
+				logger.error(f'Could not fetch CVE info for cve {cve_id}. Skipping.')
+				return None
+
+		# Match the CPE string against the affected products CPE FS strings from the CVE data if a CPE was passed.
+		# This allow to limit the number of False positives (high) that we get from nmap NSE vuln scripts like vulscan
+		# and ensure we keep only right matches.
+		# The check is not executed if no CPE was passed (sometimes nmap cannot properly detect a CPE) or if the CPE
+		# version cannot be determined.
+		cpe_match = False
+		tags = []
+		if cpes:
+			for cpe in cpes:
+				cpe_obj = CPE(cpe)
+				cpe_fs = cpe_obj.as_fs()
+				# cpe_version = cpe_obj.get_version()[0]
+				vulnerable_fs = cve_info['vulnerable_product']
+				# logger.debug(f'Matching CPE {cpe} against {len(vulnerable_fs)} vulnerable products for {cve_id}')
+				for fs in vulnerable_fs:
+					if fs == cpe_fs:
+						# logger.debug(f'Found matching CPE FS {cpe_fs} ! The CPE is vulnerable to CVE {cve_id}')
+						cpe_match = True
+						tags.append('cpe-match')
+			if not cpe_match:
+				return None
+
+		# Parse CVE id and CVSS
+		name = id = cve_info['id']
+		cvss = cve_info.get('cvss') or 0
+		# exploit_ids = cve_info.get('refmap', {}).get('exploit-db', [])
+		# osvdb_ids = cve_info.get('refmap', {}).get('osvdb', [])
+
+		# Get description
+		description = cve_info.get('summary')
+		if description is not None:
+			description = description.replace(id, '').strip()
+
+		# Get references
+		references = cve_info.get(REFERENCES, [])
+		cve_ref_url = f'https://cve.circl.lu/cve/{id}'
+		references.append(cve_ref_url)
+
+		# Get CWE ID
+		vuln_cwe_id = cve_info.get('cwe')
+		if vuln_cwe_id is None:
+			tags.append(vuln_cwe_id)
+
+		# Parse capecs for a better vuln name / type
+		capecs = cve_info.get('capec', [])
+		if capecs and len(capecs) > 0:
+			name = capecs[0]['name']
+
+		# Parse ovals for a better vuln name / type
+		ovals = cve_info.get('oval', [])
+		if ovals:
+			if description == 'none':
+				description = ovals[0]['title']
+			family = ovals[0]['family']
+			tags.append(family)
+
+		# Set vulnerability severity based on CVSS score
+		severity = None
+		if cvss:
+			if cvss < 4:
+				severity = 'low'
+			elif cvss < 7:
+				severity = 'medium'
+			elif cvss < 9:
+				severity = 'high'
+			else:
+				severity = 'critical'
+
+		# Set confidence
+		confidence = 'low' if not cpe_match else 'high'
+		vuln = {
+			ID: id,
+			NAME: name,
+			PROVIDER: 'cve.circl.lu',
+			SEVERITY: severity,
+			CVSS_SCORE: cvss,
+			TAGS: tags,
+			REFERENCES: [f'https://cve.circl.lu/cve/{id}'] + references,
+			DESCRIPTION: description,
+			CONFIDENCE: confidence
+		}
+		return vuln
+
+	@staticmethod
+	def lookup_ghsa(ghsa_id):
+		"""Search for a GHSA on Github and and return associated CVE vulnerability data.
+
+		Args:
+			ghsa (str): CVE ID in the form GHSA-*
+
+		Returns:
+			dict: vulnerability data.
+		"""
+		reference = f'https://github.com/advisories/{ghsa_id}'
+		response = requests.get(reference)
+		soup = BeautifulSoup(response.text, 'lxml')
+		sidebar_items = soup.find_all('div', {'class': 'discussion-sidebar-item'})
+		cve_id = sidebar_items[2].find('div').text.strip()
+		data = Vuln.lookup_cve(cve_id)
+		if data:
+			data[TAGS].append('ghsa')
+			return data
+		return None
+
+
+class VulnHttp(Vuln):
+	input_type = HOST
+
+
+class VulnCode(Vuln):
+	input_type = PATH
+
+
+class VulnMulti(Vuln):
+	input_type = HOST
+	output_types = [Vulnerability]
+
+
+#--------------#
+# Tag category #
+#--------------#
+
+class Tagger(Command):
+	input_type = URL
+	output_types = [Tag]
+
+#----------------#
+# osint category #
+#----------------#
+
+
+class OSInt(Command):
+	output_types = [UserAccount]