secator 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

secator/tasks/bbot.py

@@ -0,0 +1,293 @@
+import shutil
+
+from secator.decorators import task
+from secator.runners import Command
+from secator.serializers import RegexSerializer
+from secator.output_types import Vulnerability, Port, Url, Record, Ip, Tag, Error
+from secator.serializers import JSONSerializer
+
+
+BBOT_MODULES = [
+	"affiliates",
+	# "ajaxpro",
+	"anubisdb",
+	"asn",
+	"azure_realm",
+	"azure_tenant",
+	"badsecrets",
+	"bevigil",
+	"binaryedge",
+	# "bucket_aws",
+	"bucket_azure",
+	"bucket_digitalocean",
+	# "bucket_file_enum",
+	"bucket_firebase",
+	"bucket_google",
+	"builtwith",
+	"bypass403",
+	"c99",
+	"censys",
+	"certspotter",
+	# "chaos",
+	"columbus",
+	# "credshed",
+	# "crobat",
+	"crt",
+	# "dastardly",
+	# "dehashed",
+	"digitorus",
+	"dnscommonsrv",
+	"dnsdumpster",
+	# "dnszonetransfer",
+	"emailformat",
+	"ffuf",
+	"ffuf_shortnames",
+	# "filedownload",
+	"fingerprintx",
+	"fullhunt",
+	"generic_ssrf",
+	"git",
+	"telerik",
+	# "github_codesearch",
+	"github_org",
+	"gowitness",
+	"hackertarget",
+	"host_header",
+	"httpx",
+	"hunt",
+	"hunterio",
+	"iis_shortnames",
+	# "internetdb",
+	# "ip2location",
+	"ipneighbor",
+	"ipstack",
+	"leakix",
+	# "masscan",
+	# "massdns",
+	"myssl",
+	# "newsletters",
+	# "nmap",
+	# "nsec",
+	"ntlm",
+	"nuclei",
+	"oauth",
+	"otx",
+	"paramminer_cookies",
+	"paramminer_getparams",
+	"paramminer_headers",
+	"passivetotal",
+	"pgp",
+	# "postman",
+	"rapiddns",
+	# "riddler",
+	"robots",
+	"secretsdb",
+	"securitytrails",
+	"shodan_dns",
+	"sitedossier",
+	"skymem",
+	"smuggler",
+	"social",
+	"sslcert",
+	# "subdomain_hijack",
+	"subdomaincenter",
+	# "sublist3r",
+	"telerik",
+	# "threatminer",
+	"url_manipulation",
+	"urlscan",
+	"vhost",
+	"viewdns",
+	"virustotal",
+	# "wafw00f",
+	"wappalyzer",
+	"wayback",
+	"zoomeye"
+]
+BBOT_PRESETS = [
+	'cloud-enum',
+	'code-enum',
+	'dirbust-heavy',
+	'dirbust-light',
+	'dotnet-audit',
+	'email-enum',
+	'iis-shortnames',
+	'kitchen-sink',
+	'paramminer',
+	'spider',
+	'subdomain-enum',
+	'web-basic',
+	'web-screenshots',
+	'web-thorough'
+]
+BBOT_MODULES_STR = ' '.join(BBOT_MODULES)
+BBOT_MAP_TYPES = {
+	'IP_ADDRESS': Ip,
+	'PROTOCOL': Port,
+	'OPEN_TCP_PORT': Port,
+	'URL': Url,
+	'TECHNOLOGY': Tag,
+	'ASN': Record,
+	'DNS_NAME': Record,
+	'WEBSCREENSHOT': Url,
+	'VULNERABILITY': Vulnerability,
+	'FINDING': Tag
+}
+BBOT_DESCRIPTION_REGEX = RegexSerializer(
+	regex=r'(?P<name>[\w ]+): \[(?P<value>[^\[\]]+)\]',
+	findall=True
+)
+
+
+def output_discriminator(self, item):
+	_type = item.get('type')
+	_message = item.get('message')
+	if not _type and _message:
+		return Error
+	elif _type not in BBOT_MAP_TYPES:
+		return None
+	return BBOT_MAP_TYPES[_type]
+
+
+@task()
+class bbot(Command):
+	cmd = 'bbot -y --allow-deadly --force'
+	json_flag = '--json'
+	input_flag = '-t'
+	file_flag = None
+	opts = {
+		'modules': {'type': str, 'short': 'm', 'default': '', 'help': ','.join(BBOT_MODULES)},
+		'presets': {'type': str, 'short': 'ps', 'default': 'kitchen-sink', 'help': ','.join(BBOT_PRESETS), 'shlex': False},
+	}
+	opt_key_map = {
+		'modules': 'm',
+		'presets': 'p'
+	}
+	opt_value_map = {
+		'presets': lambda x: ' '.join(x.split(','))
+	}
+	item_loaders = [JSONSerializer()]
+	output_types = [Vulnerability, Port, Url, Record, Ip]
+	output_discriminator = output_discriminator
+	output_map = {
+		Ip: {
+			'ip': lambda x: x['data'],
+			'host': lambda x: x['data'],
+			'alive': lambda x: True,
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Tag: {
+			'name': 'name',
+			'match': lambda x: x['data'].get('url') or x['data'].get('host'),
+			'extra_data': 'extra_data',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Url: {
+			'url': lambda x: x['data'].get('url') if isinstance(x['data'], dict) else x['data'],
+			'host': lambda x: x['resolved_hosts'][0] if 'resolved_hosts' in x else '',
+			'status_code': lambda x: bbot.extract_status_code(x),
+			'title': lambda x: bbot.extract_title(x),
+			'screenshot_path': lambda x: x['data']['path'] if isinstance(x['data'], dict) else '',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Port: {
+			'port': lambda x: int(x['data']['port']) if 'port' in x['data'] else x['data'].split(':')[-1],
+			'ip': lambda x: [_ for _ in x['resolved_hosts'] if not _.startswith('::')][0],
+			'state': lambda x: 'OPEN',
+			'service_name': lambda x: x['data']['protocol'] if 'protocol' in x['data'] else '',
+			'cpes': lambda x: [],
+			'host': lambda x: x['data']['host'] if isinstance(x['data'], dict) else x['data'].split(':')[0],
+			'extra_data': 'extra_data',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Vulnerability: {
+			'name': 'name',
+			'match': lambda x: x['data'].get('url') or x['data']['host'],
+			'extra_data': 'extra_data',
+			'severity': lambda x: x['data']['severity'].lower()
+		},
+		Record: {
+			'name': 'name',
+			'type': 'type',
+			'extra_data': 'extra_data'
+		},
+		Error: {
+			'message': 'message'
+		}
+	}
+	install_cmd = 'pipx install bbot && pipx upgrade bbot'
+
+	@staticmethod
+	def on_json_loaded(self, item):
+		_type = item.get('type')
+
+		if not _type:
+			yield item
+			return
+
+		if _type not in BBOT_MAP_TYPES:
+			self._print(f'[bold orange3]Found unsupported bbot type: {_type}.[/] [bold green]Skipping.[/]')
+			return
+
+		if isinstance(item['data'], str):
+			item['name'] = item['data']
+			yield item
+			return
+
+		item['extra_data'] = item['data']
+
+		# Parse bbot description into extra_data
+		description = item['data'].get('description')
+		if description:
+			del item['data']['description']
+			match = BBOT_DESCRIPTION_REGEX.run(description)
+			for chunk in match:
+				key, val = tuple([c.strip() for c in chunk])
+				if ',' in val:
+					val = val.split(',')
+				key = '_'.join(key.split(' ')).lower()
+				item['extra_data'][key] = val
+
+		# Set technology as name for Tag
+		if item['type'] == 'TECHNOLOGY':
+			item['name'] = item['data']['technology']
+			del item['data']['technology']
+
+		# If 'name' key is present in 'data', set it as name
+		elif 'name' in item['data'].keys():
+			item['name'] = item['data']['name']
+			del item['data']['name']
+
+		# If 'name' key is present in 'extra_data', set it as name
+		elif 'extra_data' in item and 'name' in item['extra_data'].keys():
+			item['name'] = item['extra_data']['name']
+			del item['extra_data']['name']
+
+		# If 'discovery_context' and no name set yet, set it as name
+		else:
+			item['name'] = item['discovery_context']
+
+		# If a screenshot was saved, move it to secator output folder
+		if item['type'] == 'WEBSCREENSHOT':
+			path = item['data']['path']
+			name = path.split('/')[-1]
+			secator_path = f'{self.reports_folder}/.outputs/{name}'
+			shutil.copy(path, secator_path)
+			item['data']['path'] = secator_path
+
+		yield item
+
+	@staticmethod
+	def extract_title(item):
+		for tag in item['tags']:
+			if 'http-title' in tag:
+				title = ' '.join(tag.split('-')[2:])
+				return title
+		return ''
+
+	@staticmethod
+	def extract_status_code(item):
+		for tag in item['tags']:
+			if 'status-' in tag:
+				return int([tag.split('-')[-1]][0])
+		return 0

secator/tasks/bup.py

@@ -0,0 +1,98 @@
+import json
+import re
+
+from secator.decorators import task
+from secator.output_types import Url, Progress
+from secator.definitions import (
+	HEADER, DELAY, FOLLOW_REDIRECT, METHOD, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT,
+	DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS,
+	MATCH_CODES, OPT_NOT_SUPPORTED, URL
+)
+from secator.serializers import JSONSerializer
+from secator.tasks._categories import Http
+
+
+@task()
+class bup(Http):
+	cmd = 'bup'
+	input_flag = '-u'
+	input_type = URL
+	json_flag = '--jsonl'
+	opt_prefix = '--'
+	opts = {
+		'spoofport': {'type': int, 'short': 'sp', 'help': 'Port(s) to inject in port-specific headers'},
+		'spoofip': {'type': str, 'short': 'si', 'help': 'IP(s) to inject in ip-specific headers'},
+		'mode': {'type': str, 'help': 'Bypass modes.'},
+	}
+	opt_key_map = {
+		HEADER: 'header',
+		DELAY: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: 'retry',
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED,
+		DEPTH: OPT_NOT_SUPPORTED,
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_CODES: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		MATCH_CODES: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+	}
+	item_loaders = [JSONSerializer()]
+	output_types = [Url, Progress]
+	output_map = {
+		Url: {
+			'url': 'request_url',
+			'method': lambda x: bup.method_extractor(x),
+			'headers': lambda x: bup.headers_extractor(x),
+			'status_code': 'response_status_code',
+			'content_type': 'response_content_type',
+			'content_length': 'response_content_length',
+			'title': 'response_title',
+			'server': 'response_server_type',
+			'lines': 'response_lines_count',
+			'words': 'response_words_count',
+			'stored_response_path': 'response_html_filename',
+		}
+	}
+	install_cmd = 'pipx install bypass-url-parser && pipx upgrade bypass-url-parser'
+
+	@staticmethod
+	def on_init(self):
+		self.cmd += f' -o {self.reports_folder}/.outputs/response'
+
+	@staticmethod
+	def on_line(self, line):
+		if 'Doing' in line:
+			progress_indicator = line.split(':')[-1]
+			current, total = tuple([int(c.strip()) for c in progress_indicator.split('/')])
+			return json.dumps({"duration": "unknown", "percent": int((current / total) * 100)})
+		elif 'batcat' in line:  # ignore batcat lines as they're loaded as JSON
+			return None
+		return line
+
+	@staticmethod
+	def method_extractor(item):
+		payload = item['request_curl_payload']
+		match = re.match(r'-X\s+(\w+)', payload)
+		if match:
+			return match.group(1)
+		return 'GET'
+
+	@staticmethod
+	def headers_extractor(item):
+		headers_list = item['response_headers'].split('\n')[1:]
+		headers = {}
+		for header in headers_list:
+			split_headers = header.split(':')
+			key = split_headers[0]
+			value = ':'.join(split_headers[1:])
+			headers[key] = value
+		return headers

secator/tasks/cariddi.py

@@ -1,5 +1,3 @@
-import json
-
 from secator.decorators import task
 from secator.definitions import (DELAY, DEPTH, FILTER_CODES, FILTER_REGEX,
 							   FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT,
@@ -8,6 +6,7 @@ from secator.definitions import (DELAY, DEPTH, FILTER_CODES, FILTER_REGEX,
 							   OPT_PIPE_INPUT, PROXY, RATE_LIMIT, RETRIES,
 							   THREADS, TIMEOUT, URL, USER_AGENT)
 from secator.output_types import Tag, Url
+from secator.serializers import JSONSerializer
 from secator.tasks._categories import HttpCrawler
 
 
@@ -41,7 +40,7 @@ class cariddi(HttpCrawler):
 		TIMEOUT: 't',
 		USER_AGENT: 'ua'
 	}
-	item_loaders = []
+	item_loaders = [JSONSerializer()]
 	install_cmd = 'go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest'
 	install_github_handle = 'edoardottt/cariddi'
 	encoding = 'ansi'
@@ -51,53 +50,43 @@ class cariddi(HttpCrawler):
 	profile = 'cpu'
 
 	@staticmethod
-	def item_loader(self, line):
-		items = []
-		try:
-			item = json.loads(line)
-			url_item = {k: v for k, v in item.items() if k != 'matches'}
-			url = url_item[URL]
-			items.append(url_item)
-			matches = item.get('matches', {})
-			params = matches.get('parameters', [])
-			errors = matches.get('errors', [])
-			secrets = matches.get('secrets', [])
-			infos = matches.get('infos', [])
-
-			for param in params:
-				param_name = param['name']
-				for attack in param['attacks']:
-					extra_data = {'param': param_name, 'source': 'url'}
-					item = {
-						'name': attack + ' param',
-						'match': url,
-						'extra_data': extra_data
-					}
-					items.append(item)
-
-			for error in errors:
-				match = error['match']
-				match = (match[:1000] + '...TRUNCATED') if len(match) > 1000 else match  # truncate as this can be a very long match
-				error['extra_data'] = {'error': match, 'source': 'body'}
-				error['match'] = url
-				items.append(error)
+	def on_json_loaded(self, item):
+		url_item = {k: v for k, v in item.items() if k != 'matches'}
+		yield Url(**url_item)
+		url = url_item[URL]
+		matches = item.get('matches', {})
+		params = matches.get('parameters', [])
+		errors = matches.get('errors', [])
+		secrets = matches.get('secrets', [])
+		infos = matches.get('infos', [])
 
-			for secret in secrets:
-				match = secret['match']
-				secret['extra_data'] = {'secret': match, 'source': 'body'}
-				secret['match'] = url
-				items.append(secret)
+		for param in params:
+			param_name = param['name']
+			for attack in param['attacks']:
+				extra_data = {'param': param_name, 'source': 'url'}
+				yield Tag(
+					name=f'{attack} param',
+					match=url,
+					extra_data=extra_data
+				)
 
-			for info in infos:
-				CARIDDI_IGNORE_LIST = ['BTC address']  # TODO: make this a config option
-				if info['name'] in CARIDDI_IGNORE_LIST:
-					continue
-				match = info['match']
-				info['extra_data'] = {'info': match, 'source': 'body'}
-				info['match'] = url
-				items.append(info)
+		for error in errors:
+			match = error['match']
+			error['extra_data'] = {'error': match, 'source': 'body'}
+			error['match'] = url
+			yield Tag(**error)
 
-		except json.decoder.JSONDecodeError:
-			pass
+		for secret in secrets:
+			match = secret['match']
+			secret['extra_data'] = {'secret': match, 'source': 'body'}
+			secret['match'] = url
+			yield Tag(**secret)
 
-		return items
+		for info in infos:
+			CARIDDI_IGNORE_LIST = ['BTC address']  # TODO: make this a config option
+			if info['name'] in CARIDDI_IGNORE_LIST:
+				continue
+			match = info['match']
+			info['extra_data'] = {'info': match, 'source': 'body'}
+			info['match'] = url
+			yield Tag(**info)

secator/tasks/dalfox.py

@@ -7,6 +7,7 @@ from secator.definitions import (CONFIDENCE, DELAY, EXTRA_DATA, FOLLOW_REDIRECT,
 							   SEVERITY, TAGS, THREADS, TIMEOUT, URL,
 							   USER_AGENT)
 from secator.output_types import Vulnerability
+from secator.serializers import JSONSerializer
 from secator.tasks._categories import VulnHttp
 
 DALFOX_TYPE_MAP = {
@@ -23,6 +24,7 @@ class dalfox(VulnHttp):
 	input_type = URL
 	input_flag = 'url'
 	file_flag = 'file'
+	# input_chunk_size = 1
 	json_flag = '--format json'
 	version_flag = 'version'
 	opt_prefix = '--'
@@ -37,6 +39,7 @@ class dalfox(VulnHttp):
 		TIMEOUT: 'timeout',
 		USER_AGENT: 'user-agent'
 	}
+	item_loaders = [JSONSerializer()]
 	output_map = {
 		Vulnerability: {
 			ID: lambda x: None,

secator/tasks/dirsearch.py

@@ -10,7 +10,7 @@ from secator.definitions import (CONTENT_LENGTH, CONTENT_TYPE, DELAY, DEPTH,
 							   MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, OUTPUT_PATH, PROXY,
 							   RATE_LIMIT, RETRIES, STATUS_CODE,
 							   THREADS, TIMEOUT, USER_AGENT, WORDLIST)
-from secator.output_types import Url
+from secator.output_types import Url, Info, Error
 from secator.tasks._categories import HttpFuzzer
 
 
@@ -58,31 +58,20 @@ class dirsearch(HttpFuzzer):
 	proxy_http = True
 	profile = 'io'
 
-	def yielder(self):
-		prev = self.print_item_count
-		self.print_item_count = False
-		list(super().yielder())
-		if self.return_code != 0:
-			return
-		self.results = []
-		if not self.output_json:
-			return
-		note = f'dirsearch JSON results saved to {self.output_path}'
-		if self.print_line:
-			self._print(note)
-		if os.path.exists(self.output_path):
-			with open(self.output_path, 'r') as f:
-				results = yaml.safe_load(f.read()).get('results', [])
-			for item in results:
-				item = self._process_item(item)
-				if not item:
-					continue
-				yield item
-		self.print_item_count = prev
-
 	@staticmethod
 	def on_init(self):
 		self.output_path = self.get_opt_value(OUTPUT_PATH)
 		if not self.output_path:
 			self.output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.json'
 		self.cmd += f' -o {self.output_path}'
+
+	@staticmethod
+	def on_cmd_done(self):
+		if not os.path.exists(self.output_path):
+			yield Error(message=f'Could not find JSON results in {self.output_path}')
+			return
+
+		yield Info(message=f'JSON results saved to {self.output_path}')
+		with open(self.output_path, 'r') as f:
+			results = yaml.safe_load(f.read()).get('results', [])
+		yield from results

secator/tasks/dnsx.py

@@ -1,18 +1,20 @@
 from secator.decorators import task
 from secator.definitions import (OPT_PIPE_INPUT, RATE_LIMIT, RETRIES, THREADS)
-from secator.output_types import Record
+from secator.output_types import Record, Ip, Subdomain
+from secator.output_types.ip import IpProtocol
 from secator.tasks._categories import ReconDns
-import json
+from secator.serializers import JSONSerializer
+from secator.utils import extract_domain_info
 
 
 @task()
 class dnsx(ReconDns):
 	"""dnsx is a fast and multi-purpose DNS toolkit designed for running various retryabledns library."""
-	cmd = 'dnsx -resp -a -aaaa -cname -mx -ns -txt -srv -ptr -soa -axfr -caa'
+	cmd = 'dnsx -resp -recon'
 	json_flag = '-json'
 	input_flag = OPT_PIPE_INPUT
 	file_flag = OPT_PIPE_INPUT
-	output_types = [Record]
+	output_types = [Record, Ip, Subdomain]
 	opt_key_map = {
 		RATE_LIMIT: 'rate-limit',
 		RETRIES: 'retry',
@@ -23,35 +25,52 @@ class dnsx(ReconDns):
 		'resolver': {'type': str, 'short': 'r', 'help': 'List of resolvers to use (file or comma separated)'},
 		'wildcard_domain': {'type': str, 'short': 'wd', 'help': 'Domain name for wildcard filtering'},
 	}
-
+	item_loaders = [JSONSerializer()]
 	install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest'
 	install_github_handle = 'projectdiscovery/dnsx'
 	profile = 'io'
 
 	@staticmethod
-	def item_loader(self, line):
-		items = []
-		try:
-			item = json.loads(line)
-			if self.orig:  # original dnsx output
-				return item
-			host = item['host']
-			record_types = ['a', 'aaaa', 'cname', 'mx', 'ns', 'txt', 'srv', 'ptr', 'soa', 'axfr', 'caa']
-			for _type in record_types:
-				values = item.get(_type, [])
-				for value in values:
-					name = value
-					extra_data = {}
-					if isinstance(value, dict):
-						name = value['name']
-						extra_data = {k: v for k, v in value.items() if k != 'name'}
-					items.append({
-						'host': host,
-						'name': name,
-						'type': _type.upper(),
-						'extra_data': extra_data
-					})
-		except json.decoder.JSONDecodeError:
-			pass
+	def on_json_loaded(self, item):
+		# Show full DNS response
+		quiet = self.get_opt_value('quiet')
+		if not quiet:
+			all = item['all']
+			for line in all:
+				yield line
+			yield '\n'
 
-		return items
+		# Loop through record types and yield records
+		record_types = ['a', 'aaaa', 'cname', 'mx', 'ns', 'txt', 'srv', 'ptr', 'soa', 'axfr', 'caa']
+		host = item['host']
+		for _type in record_types:
+			values = item.get(_type, [])
+			for value in values:
+				name = value
+				extra_data = {}
+				if isinstance(value, dict):
+					name = value['name']
+					extra_data = {k: v for k, v in value.items() if k != 'name'}
+				if _type == 'a':
+					yield Ip(
+						host=host,
+						ip=name,
+						protocol=IpProtocol.IPv4
+					)
+				elif _type == 'aaaa':
+					yield Ip(
+						host=host,
+						ip=name,
+						protocol=IpProtocol.IPv6
+					)
+				elif _type == 'ptr':
+					yield Subdomain(
+						host=name,
+						domain=extract_domain_info(name, domain_only=True)
+					)
+				yield Record(
+					host=host,
+					name=name,
+					type=_type.upper(),
+					extra_data=extra_data
+				)