secator 0.0.1__py3-none-any.whl

secator/tasks/gospider.py

@@ -0,0 +1,71 @@
+from furl import furl
+
+from secator.decorators import task
+from secator.definitions import (CONTENT_LENGTH, DELAY, DEPTH, FILTER_CODES,
+							   FILTER_REGEX, FILTER_SIZE, FILTER_WORDS,
+							   FOLLOW_REDIRECT, HEADER, MATCH_CODES,
+							   MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, METHOD,
+							   OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT, RETRIES,
+							   STATUS_CODE, THREADS, TIMEOUT, URL, USER_AGENT)
+from secator.output_types import Url
+from secator.tasks._categories import HttpCrawler
+
+
+@task()
+class gospider(HttpCrawler):
+	"""Fast web spider written in Go."""
+	cmd = 'gospider --js'
+	file_flag = '-S'
+	input_flag = '-s'
+	json_flag = '--json'
+	opt_prefix = '--'
+	opt_key_map = {
+		HEADER: 'header',
+		DELAY: 'delay',
+		DEPTH: 'depth',
+		FILTER_CODES: OPT_NOT_SUPPORTED,
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: 'no-redirect',
+		MATCH_CODES: OPT_NOT_SUPPORTED,
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: OPT_NOT_SUPPORTED,
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: 'user-agent',
+	}
+	opt_value_map = {
+		FOLLOW_REDIRECT: lambda x: not x,
+		DELAY: lambda x: round(x) if isinstance(x, float) else x
+	}
+	output_map = {
+		Url: {
+			URL: 'output',
+			STATUS_CODE: 'status',
+			CONTENT_LENGTH: 'length',
+		}
+	}
+	install_cmd = 'go install -v github.com/jaeles-project/gospider@latest'
+	ignore_return_code = True
+	proxychains = False
+	proxy_socks5 = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61
+	proxy_http = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61
+	profile = 'cpu'
+
+	@staticmethod
+	def validate_item(self, item):
+		"""Keep only items that match the same host."""
+		try:
+			netloc_in = furl(item['input']).netloc
+			netloc_out = furl(item['output']).netloc
+			if netloc_in != netloc_out:
+				return False
+		except ValueError:  # gospider returns invalid URLs for output sometimes
+			return False
+		return True

secator/tasks/grype.py

@@ -0,0 +1,79 @@
+
+from secator.decorators import task
+from secator.definitions import (DELAY, FOLLOW_REDIRECT, HEADER,
+							   OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT, RETRIES,
+							   THREADS, TIMEOUT, USER_AGENT)
+from secator.output_types import Vulnerability
+from secator.tasks._categories import VulnCode
+
+
+def grype_item_loader(self, line):
+	"""Load vulnerabilty dicts from grype line output."""
+	split = [i for i in line.split(' ') if i]
+	if not len(split) in [5, 6] or split[0] == 'NAME':
+		return None
+	version_fixed = None
+	if len(split) == 5:  # no version fixed
+		product, version, product_type, vuln_id, severity = tuple(split)
+	elif len(split) == 6:
+		product, version, version_fixed, product_type, vuln_id, severity = tuple(split)
+	extra_data = {
+		'lang': product_type,
+		'product': product,
+		'version': version,
+	}
+	if version_fixed:
+		extra_data['version_fixed'] = version_fixed
+	data = {
+		'id': vuln_id,
+		'name': vuln_id,
+		'matched_at': self.input,
+		'confidence': 'medium',
+		'severity': severity.lower(),
+		'provider': 'grype',
+		'cvss_score': -1,
+		'tags': [],
+	}
+	if vuln_id.startswith('GHSA'):
+		data['provider'] = 'github.com'
+		data['references'] = [f'https://github.com/advisories/{vuln_id}']
+		data['tags'].extend(['cve', 'ghsa'])
+		vuln = VulnCode.lookup_ghsa(vuln_id)
+		if vuln:
+			data.update(vuln)
+			data['severity'] = data['severity'] or severity.lower()
+			extra_data['ghsa_id'] = vuln_id
+	elif vuln_id.startswith('CVE'):
+		vuln = VulnCode.lookup_cve(vuln_id)
+		if vuln:
+			vuln['tags'].append('cve')
+			data.update(vuln)
+			data['severity'] = data['severity'] or severity.lower()
+	data['extra_data'] = extra_data
+	return data
+
+
+@task()
+class grype(VulnCode):
+	"""Vulnerability scanner for container images and filesystems."""
+	cmd = 'grype --quiet'
+	input_flag = ''
+	file_flag = OPT_NOT_SUPPORTED
+	json_flag = None
+	opt_prefix = '--'
+	opt_key_map = {
+		HEADER: OPT_NOT_SUPPORTED,
+		DELAY: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: OPT_NOT_SUPPORTED,
+		PROXY: OPT_NOT_SUPPORTED,
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: OPT_NOT_SUPPORTED,
+		THREADS: OPT_NOT_SUPPORTED,
+		TIMEOUT: OPT_NOT_SUPPORTED,
+		USER_AGENT: OPT_NOT_SUPPORTED
+	}
+	output_types = [Vulnerability]
+	install_cmd = (
+		'curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin'
+	)
+	item_loaders = [grype_item_loader]

secator/tasks/h8mail.py

@@ -0,0 +1,81 @@
+import os
+import json
+
+from secator.decorators import task
+from secator.definitions import (EMAIL, DATA_FOLDER)
+from secator.tasks._categories import OSInt
+from secator.utils import get_file_timestamp
+from secator.output_types import UserAccount
+
+
+@task()
+class h8mail(OSInt):
+	"""Email information and password lookup tool."""
+	cmd = 'h8mail'
+	json_flag = '--json '
+	input_flag = '--targets'
+	input_type = EMAIL
+	file_flag = '-domain'
+	opt_prefix = '--'
+	opt_key_map = {
+
+	}
+	opts = {
+		'config': {'type': str, 'help': 'Configuration file for API keys'},
+		'local_breach': {'type': str, 'short': 'lb', 'help': 'Local breach file'}
+	}
+	output_map = {
+	}
+
+	install_cmd = 'pip3 install h8mail'
+
+	@staticmethod
+	def on_start(self):
+		output_path = self.get_opt_value('output_path')
+		if not output_path:
+			timestr = get_file_timestamp()
+			output_path = f'{DATA_FOLDER}/h8mail_{timestr}.json'
+		self.output_path = output_path
+		self.cmd = self.cmd.replace('--json', f'--json {self.output_path}')
+
+	def yielder(self):
+		prev = self.print_item_count
+		self.print_item_count = False
+		list(super().yielder())
+		if self.return_code != 0:
+			return
+		self.results = []
+		if os.path.exists(self.output_path):
+			with open(self.output_path, 'r') as f:
+				data = json.load(f)
+			if self.orig:  # original h8mail output
+				yield data
+				return
+			targets = data['targets']
+			for target in targets:
+				email = target['target']
+				target_data = target.get('data', [])
+				pwn_num = target['pwn_num']
+				if not pwn_num > 0:
+					continue
+				if len(target_data) > 0:
+					entries = target_data[0]
+					for entry in entries:
+						source, site_name = tuple(entry.split(':'))
+						yield UserAccount(**{
+							"site_name": site_name,
+							"username": email.split('@')[0],
+							"email": email,
+							"extra_data": {
+								'source': source
+							},
+						})
+				else:
+					yield UserAccount(**{
+						"username": email.split('@')[0],
+						"email": email,
+						"extra_data": {
+							'source': self.get_opt_value('local_breach')
+						},
+					})
+		self.print_item_count = prev

secator/tasks/httpx.py

@@ -0,0 +1,99 @@
+import os
+import uuid
+
+from secator.decorators import task
+from secator.definitions import (DEFAULT_HTTPX_FLAGS,
+								 DEFAULT_STORE_HTTP_RESPONSES, DELAY, DEPTH,
+								 FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
+								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER,
+								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
+								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
+								 RATE_LIMIT, RETRIES, TASKS_FOLDER, THREADS,
+								 TIMEOUT, URL, USER_AGENT)
+from secator.tasks._categories import Http
+from secator.utils import sanitize_url
+
+
+@task()
+class httpx(Http):
+	"""Fast and multi-purpose HTTP toolkit."""
+	cmd = f'httpx {DEFAULT_HTTPX_FLAGS}'
+	file_flag = '-l'
+	input_flag = '-u'
+	json_flag = '-json'
+	opts = {
+		# 'silent': {'is_flag': True, 'default': False, 'help': 'Silent mode'},
+		# 'td': {'is_flag': True, 'default': True, 'help': 'Tech detection'},
+		'irr': {'is_flag': True, 'default': False, 'help': 'Include http request / response'},
+		'fep': {'is_flag': True, 'default': False, 'help': 'Error Page Classifier and Filtering'},
+		'favicon': {'is_flag': True, 'default': False, 'help': 'Favicon hash'},
+		'jarm': {'is_flag': True, 'default': False, 'help': 'Jarm fingerprint'},
+		'asn': {'is_flag': True, 'default': False, 'help': 'ASN detection'},
+		'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'},
+		'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'},
+		'screenshot': {'is_flag': True, 'default': False, 'help': 'Screenshot response'}
+	}
+	opt_key_map = {
+		HEADER: 'header',
+		DELAY: 'delay',
+		DEPTH: OPT_NOT_SUPPORTED,
+		FILTER_CODES: 'filter-code',
+		FILTER_REGEX: 'filter-regex',
+		FILTER_SIZE: 'filter-length',
+		FILTER_WORDS: 'filter-word-count',
+		FOLLOW_REDIRECT: 'follow-redirects',
+		MATCH_CODES: 'match-code',
+		MATCH_REGEX: 'match-regex',
+		MATCH_SIZE: 'match-length',
+		MATCH_WORDS: 'match-word-count',
+		METHOD: 'x',
+		PROXY: 'proxy',
+		RATE_LIMIT: 'rate-limit',
+		RETRIES: 'retries',
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED,
+	}
+	opt_value_map = {
+		DELAY: lambda x: str(x) + 's' if x else None,
+	}
+	install_cmd = 'go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest'
+	proxychains = False
+	proxy_socks5 = True
+	proxy_http = True
+	profile = 'cpu'
+
+	@staticmethod
+	def on_item_pre_convert(self, item):
+		for k, v in item.items():
+			if k == 'time':
+				response_time = float(''.join(ch for ch in v if not ch.isalpha()))
+				if v[-2:] == 'ms':
+					response_time = response_time / 1000
+				item[k] = response_time
+			elif k == URL:
+				item[k] = sanitize_url(v)
+		item[URL] = item.get('final_url') or item[URL]
+		return item
+
+	@staticmethod
+	def on_init(self):
+		debug_resp = self.get_opt_value('debug_resp')
+		if debug_resp:
+			self.cmd = self.cmd.replace('-silent', '')
+		if DEFAULT_STORE_HTTP_RESPONSES:
+			_id = uuid.uuid4()
+			output_path = f'{TASKS_FOLDER}/{_id}'
+			self.output_response_path = f'{output_path}/response'
+			self.output_screenshot_path = f'{output_path}/screenshot'
+			os.makedirs(self.output_response_path, exist_ok=True)
+			os.makedirs(self.output_screenshot_path, exist_ok=True)
+			self.cmd += f' -sr -srd {output_path}'
+
+	@staticmethod
+	def on_end(self):
+		if DEFAULT_STORE_HTTP_RESPONSES:
+			if os.path.exists(self.output_response_path + '/index.txt'):
+				os.remove(self.output_response_path + '/index.txt')
+			if os.path.exists(self.output_screenshot_path + '/index.txt'):
+				os.remove(self.output_screenshot_path + '/index_screenshot.txt')

secator/tasks/katana.py

@@ -0,0 +1,133 @@
+import os
+import json
+import uuid
+from urllib.parse import urlparse
+
+from secator.decorators import task
+from secator.definitions import (CONTENT_TYPE, DEFAULT_KATANA_FLAGS,
+								 DEFAULT_STORE_HTTP_RESPONSES, DELAY, DEPTH,
+								 FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
+								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST,
+								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
+								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
+								 RATE_LIMIT, RETRIES, STATUS_CODE,
+								 STORED_RESPONSE_PATH, TASKS_FOLDER, TECH,
+								 THREADS, TIME, TIMEOUT, URL, USER_AGENT, WEBSERVER, CONTENT_LENGTH)
+from secator.output_types import Url, Tag
+from secator.tasks._categories import HttpCrawler
+
+
+@task()
+class katana(HttpCrawler):
+	"""Next-generation crawling and spidering framework."""
+	# TODO: add -fx for form detection and extract 'forms' from the output with custom item_loader
+	# TODO: add -jsluice for JS parsing
+	cmd = f'katana {DEFAULT_KATANA_FLAGS}'
+	file_flag = '-list'
+	input_flag = '-u'
+	json_flag = '-jsonl'
+	opts = {
+		'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'},
+		'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'},
+		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'}
+	}
+	opt_key_map = {
+		HEADER: 'headers',
+		DELAY: 'delay',
+		DEPTH: 'depth',
+		FILTER_CODES: OPT_NOT_SUPPORTED,
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: OPT_NOT_SUPPORTED,
+		MATCH_CODES: OPT_NOT_SUPPORTED,
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+		RATE_LIMIT: 'rate-limit',
+		RETRIES: 'retry',
+		THREADS: 'concurrency',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED
+	}
+	opt_value_map = {
+		DELAY: lambda x: int(x) if isinstance(x, float) else x
+	}
+	output_map = {
+		Url: {
+			URL: lambda x: x['request']['endpoint'],
+			HOST: lambda x: urlparse(x['request']['endpoint']).netloc,
+			TIME: 'timestamp',
+			METHOD: lambda x: x['request']['method'],
+			STATUS_CODE: lambda x: x['response'].get('status_code'),
+			CONTENT_TYPE: lambda x: x['response'].get('headers', {}).get('content_type', ';').split(';')[0],
+			CONTENT_LENGTH: lambda x: x['response'].get('headers', {}).get('content_length', 0),
+			WEBSERVER: lambda x: x['response'].get('headers', {}).get('server', ''),
+			TECH: lambda x: x['response'].get('technologies', []),
+			STORED_RESPONSE_PATH: lambda x: x['response'].get('stored_response_path', '')
+			# TAGS: lambda x: x['response'].get('server')
+		}
+	}
+	item_loaders = []
+	install_cmd = 'go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
+	proxychains = False
+	proxy_socks5 = True
+	proxy_http = True
+	profile = 'io'
+
+	@staticmethod
+	def item_loader(self, item):
+		try:
+			item = json.loads(item)
+		except json.JSONDecodeError:
+			return None
+
+		# form detection
+		forms = item.get('response', {}).get('forms', [])
+		if forms:
+			for form in forms:
+				method = form['method']
+				yield Url(form['action'], host=urlparse(item['request']['endpoint']).netloc, method=method)
+				yield Tag(
+					name='form',
+					match=form['action'],
+					extra_data={
+						'method': form['method'],
+						'enctype': form.get('enctype', ''),
+						'parameters': ','.join(form.get('parameters', []))
+					}
+				)
+		yield item
+
+	@staticmethod
+	def on_init(self):
+		debug_resp = self.get_opt_value('debug_resp')
+		if debug_resp:
+			self.cmd = self.cmd.replace('-silent', '')
+		if DEFAULT_STORE_HTTP_RESPONSES:
+			_id = uuid.uuid4()
+			output_path = f'{TASKS_FOLDER}/{_id}'
+			self.output_response_path = output_path
+			os.makedirs(self.output_response_path, exist_ok=True)
+			self.cmd += f' -sr -srd {output_path}'
+
+	@staticmethod
+	def on_end(self):
+		if DEFAULT_STORE_HTTP_RESPONSES and os.path.exists(self.output_response_path + '/index.txt'):
+			os.remove(self.output_response_path + '/index.txt')
+
+	@staticmethod
+	def on_item(self, item):
+		if not isinstance(item, Url):
+			return item
+		if DEFAULT_STORE_HTTP_RESPONSES and os.path.exists(item.stored_response_path):
+			with open(item.stored_response_path, 'r') as fin:
+				data = fin.read().splitlines(True)
+				first_line = data[0]
+			with open(item.stored_response_path, 'w') as fout:
+				fout.writelines(data[1:])
+				fout.writelines('\n')
+				fout.writelines(first_line)
+		return item

secator/tasks/maigret.py

@@ -0,0 +1,78 @@
+import json
+import logging
+import os
+import re
+
+from secator.decorators import task
+from secator.definitions import (DELAY, EXTRA_DATA, OPT_NOT_SUPPORTED, PROXY,
+								 RATE_LIMIT, RETRIES, SITE_NAME, THREADS,
+								 TIMEOUT, URL, USERNAME)
+from secator.output_types import UserAccount
+from secator.tasks._categories import ReconUser
+
+logger = logging.getLogger(__name__)
+
+
+@task()
+class maigret(ReconUser):
+	"""Collect a dossier on a person by username."""
+	cmd = 'maigret'
+	file_flag = None
+	input_flag = None
+	json_flag = '--json ndjson'
+	opt_prefix = '--'
+	opts = {
+		'site': {'type': str, 'help': 'Sites to check'},
+	}
+	opt_key_map = {
+		DELAY: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: 'retries',
+		TIMEOUT: 'timeout',
+		THREADS: OPT_NOT_SUPPORTED
+	}
+	input_type = USERNAME
+	output_types = [UserAccount]
+	output_map = {
+		UserAccount: {
+			SITE_NAME: 'sitename',
+			URL: lambda x: x['status']['url'],
+			EXTRA_DATA: lambda x: x['status'].get('ids', {})
+		}
+	}
+	install_cmd = 'pip3 install maigret'
+	socks5_proxy = True
+	profile = 'io'
+
+	def yielder(self):
+		prev = self.print_item_count
+		self.print_item_count = False
+		yield from super().yielder()
+		if self.return_code != 0:
+			return
+		self.results = []
+		if not self.output_path:
+			match = re.search('JSON ndjson report for .* saved in (.*)', self.output)
+			if match is None:
+				logger.warning('JSON output file not found in command output.')
+				return
+			self.output_path = match.group(1)
+		note = f'maigret JSON results saved to {self.output_path}'
+		if self.print_line:
+			self._print(note)
+		if os.path.exists(self.output_path):
+			with open(self.output_path, 'r') as f:
+				data = [json.loads(line) for line in f.read().splitlines()]
+			for item in data:
+				yield item
+		self.print_item_count = prev
+
+	@staticmethod
+	def on_init(self):
+		output_path = self.get_opt_value('output_path')
+		self.output_path = output_path
+
+	@staticmethod
+	def validate_item(self, item):
+		return item['http_status'] == 200

secator/tasks/mapcidr.py

@@ -0,0 +1,32 @@
+import validators
+
+from secator.decorators import task
+from secator.definitions import (CIDR_RANGE, OPT_NOT_SUPPORTED, PROXY,
+							   RATE_LIMIT, RETRIES, THREADS, TIMEOUT)
+from secator.output_types import Ip
+from secator.tasks._categories import ReconIp
+
+
+@task()
+class mapcidr(ReconIp):
+	"""Utility program to perform multiple operations for a given subnet/cidr ranges."""
+	cmd = 'mapcidr -silent'
+	input_flag = '-cidr'
+	file_flag = '-cl'
+	install_cmd = 'go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidr@latest'
+	input_type = CIDR_RANGE
+	output_types = [Ip]
+	opt_key_map = {
+		THREADS: OPT_NOT_SUPPORTED,
+		PROXY: OPT_NOT_SUPPORTED,
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: OPT_NOT_SUPPORTED,
+		TIMEOUT: OPT_NOT_SUPPORTED,
+		THREADS: OPT_NOT_SUPPORTED,
+	}
+
+	@staticmethod
+	def item_loader(self, line):
+		if validators.ipv4(line) or validators.ipv6(line):
+			return {'ip': line, 'alive': False}
+		return None