secator 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

secator/tasks/httpx.py

@@ -1,28 +1,25 @@
 import os
 
 from secator.decorators import task
-from secator.definitions import (DEFAULT_HTTPX_FLAGS, DELAY, DEPTH,
-								 FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
-								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER,
-								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
-								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
-								 RATE_LIMIT, RETRIES, THREADS,
-								 TIMEOUT, URL, USER_AGENT)
+from secator.definitions import (DELAY, DEPTH, FILTER_CODES, FILTER_REGEX, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT,
+								 HEADER, MATCH_CODES, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED,
+								 PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, URL, USER_AGENT)
 from secator.config import CONFIG
+from secator.output_types import Url, Subdomain
+from secator.serializers import JSONSerializer
 from secator.tasks._categories import Http
-from secator.utils import sanitize_url
+from secator.utils import (sanitize_url, extract_domain_info, extract_subdomains_from_fqdn)
 
 
 @task()
 class httpx(Http):
 	"""Fast and multi-purpose HTTP toolkit."""
-	cmd = f'httpx {DEFAULT_HTTPX_FLAGS}'
+	cmd = 'httpx'
 	file_flag = '-l'
 	input_flag = '-u'
 	json_flag = '-json'
 	opts = {
 		# 'silent': {'is_flag': True, 'default': False, 'help': 'Silent mode'},
-		# 'td': {'is_flag': True, 'default': True, 'help': 'Tech detection'},
 		# 'irr': {'is_flag': True, 'default': False, 'help': 'Include http request / response'},
 		'fep': {'is_flag': True, 'default': False, 'help': 'Error Page Classifier and Filtering'},
 		'favicon': {'is_flag': True, 'default': False, 'help': 'Favicon hash'},
@@ -35,6 +32,11 @@ class httpx(Http):
 		'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'},
 		'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'},
 		'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'},
+		'follow_host_redirects': {'is_flag': True, 'short': 'fhr', 'default': None, 'help': 'Follow redirects on the same host'},  # noqa: E501
+		'tech_detect': {'is_flag': True, 'short': 'td', 'default': True, 'help': 'Tech detection'},
+		'tls_grab': {'is_flag': True, 'short': 'tlsg', 'default': False, 'help': 'Grab some informations from the tls certificate'},  # noqa: E501
+		'rstr': {'type': int, 'default': CONFIG.http.response_max_size_bytes, 'help': 'Max body size to read (bytes)'},
+		'rsts': {'type': int, 'default': CONFIG.http.response_max_size_bytes, 'help': 'Max body size to save (bytes)'}
 	}
 	opt_key_map = {
 		HEADER: 'header',
@@ -61,6 +63,8 @@ class httpx(Http):
 	opt_value_map = {
 		DELAY: lambda x: str(x) + 's' if x else None,
 	}
+	item_loaders = [JSONSerializer()]
+	output_types = [Url, Subdomain]
 	install_cmd = 'go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest'
 	install_github_handle = 'projectdiscovery/httpx'
 	proxychains = False
@@ -79,19 +83,23 @@ class httpx(Http):
 			self.cmd += f' -srd {self.reports_folder}/.outputs'
 		if screenshot:
 			self.cmd += ' -esb -ehb'
+		self.domains = []
 
 	@staticmethod
-	def on_item_pre_convert(self, item):
-		for k, v in item.items():
-			if k == 'time':
-				response_time = float(''.join(ch for ch in v if not ch.isalpha()))
-				if v[-2:] == 'ms':
-					response_time = response_time / 1000
-				item[k] = response_time
-			elif k == URL:
-				item[k] = sanitize_url(v)
-		item[URL] = item.get('final_url') or item[URL]
-		return item
+	def on_json_loaded(self, item):
+		item = self._preprocess_url(item)
+		yield item
+		tls = item.get('tls', None)
+		if tls:
+			subject_cn = tls.get('subject_cn', None)
+			subject_an = tls.get('subject_an', [])
+			cert_domains = subject_an
+			if subject_cn:
+				cert_domains.append(subject_cn)
+			for cert_domain in cert_domains:
+				subdomain = self._create_subdomain_from_tls_cert(cert_domain, item['url'])
+				if subdomain:
+					yield subdomain
 
 	@staticmethod
 	def on_end(self):
@@ -107,3 +115,32 @@ class httpx(Http):
 				os.remove(index_spath)
 			if os.path.exists(index_spath2):
 				os.remove(index_spath2)
+
+	def _preprocess_url(self, item):
+		"""Replace time string by float, sanitize URL, get final redirect URL."""
+		for k, v in item.items():
+			if k == 'time':
+				response_time = float(''.join(ch for ch in v if not ch.isalpha()))
+				if v[-2:] == 'ms':
+					response_time = response_time / 1000
+				item[k] = response_time
+			elif k == URL:
+				item[k] = sanitize_url(v)
+		item[URL] = item.get('final_url') or item[URL]
+		return item
+
+	def _create_subdomain_from_tls_cert(self, domain, url):
+		"""Extract subdomains from TLS certificate."""
+		if domain.startswith('*.'):
+			domain = domain.lstrip('*.')
+		if domain in self.domains:
+			return None
+		url_domain = extract_domain_info(url)
+		url_domains = extract_subdomains_from_fqdn(url_domain.fqdn, url_domain.domain, url_domain.suffix)
+		if not url_domain or domain not in url_domains:
+			return None
+		self.domains.append(domain)
+		return Subdomain(
+			host=domain,
+			domain=extract_domain_info(domain, domain_only=True)
+		)

secator/tasks/katana.py

@@ -1,28 +1,22 @@
 import os
-import json
 from urllib.parse import urlparse
 
 from secator.decorators import task
-from secator.definitions import (CONTENT_TYPE, DEFAULT_KATANA_FLAGS,
-								 DELAY, DEPTH,
-								 FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
-								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST,
-								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
-								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
-								 RATE_LIMIT, RETRIES, STATUS_CODE,
-								 STORED_RESPONSE_PATH, TECH,
-								 THREADS, TIME, TIMEOUT, URL, USER_AGENT, WEBSERVER, CONTENT_LENGTH)
+from secator.definitions import (CONTENT_TYPE, DELAY, DEPTH, FILTER_CODES, FILTER_REGEX, FILTER_SIZE, FILTER_WORDS,
+								 FOLLOW_REDIRECT, HEADER, HOST, MATCH_CODES, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS,
+								 METHOD, OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT, RETRIES, STATUS_CODE,
+								 STORED_RESPONSE_PATH, TECH, THREADS, TIME, TIMEOUT, URL, USER_AGENT, WEBSERVER,
+								 CONTENT_LENGTH)
 from secator.config import CONFIG
 from secator.output_types import Url, Tag
+from secator.serializers import JSONSerializer
 from secator.tasks._categories import HttpCrawler
 
 
 @task()
 class katana(HttpCrawler):
 	"""Next-generation crawling and spidering framework."""
-	# TODO: add -fx for form detection and extract 'forms' from the output with custom item_loader
-	# TODO: add -jsluice for JS parsing
-	cmd = f'katana {DEFAULT_KATANA_FLAGS}'
+	cmd = 'katana'
 	file_flag = '-list'
 	input_flag = '-u'
 	json_flag = '-jsonl'
@@ -30,7 +24,13 @@ class katana(HttpCrawler):
 		'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'},
 		'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'},
 		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'},
-		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'}
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'},  # noqa: E501
+		'form_fill': {'is_flag': True, 'short': 'ff', 'help': 'Enable form filling'},
+		'js_crawl': {'is_flag': True, 'short': 'jc', 'default': True, 'help': 'Enable endpoint parsing / crawling in javascript file'},  # noqa: E501
+		'jsluice': {'is_flag': True, 'short': 'jsl', 'default': True, 'help': 'Enable jsluice parsing in javascript file (memory intensive)'},  # noqa: E501
+		'known_files': {'type': str, 'short': 'kf', 'default': 'all', 'help': 'Enable crawling of known files (all, robotstxt, sitemapxml)'},  # noqa: E501
+		'omit_raw': {'is_flag': True, 'short': 'or', 'default': True, 'help': 'Omit raw requests/responses from jsonl output'},  # noqa: E501
+		'omit_body': {'is_flag': True, 'short': 'ob', 'default': True, 'help': 'Omit response body from jsonl output'}
 	}
 	opt_key_map = {
 		HEADER: 'headers',
@@ -52,11 +52,13 @@ class katana(HttpCrawler):
 		THREADS: 'concurrency',
 		TIMEOUT: 'timeout',
 		USER_AGENT: OPT_NOT_SUPPORTED,
-		'store_responses': 'sr'
+		'store_responses': 'sr',
+		'form_fill': 'aff'
 	}
 	opt_value_map = {
 		DELAY: lambda x: int(x) if isinstance(x, float) else x
 	}
+	item_loaders = [JSONSerializer()]
 	output_map = {
 		Url: {
 			URL: lambda x: x['request']['endpoint'],
@@ -72,8 +74,7 @@ class katana(HttpCrawler):
 			# TAGS: lambda x: x['response'].get('server')
 		}
 	}
-	item_loaders = []
-	install_cmd = 'sudo apt install build-essential && go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
+	install_cmd = 'go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
 	install_github_handle = 'projectdiscovery/katana'
 	proxychains = False
 	proxy_socks5 = True
@@ -81,12 +82,7 @@ class katana(HttpCrawler):
 	profile = 'io'
 
 	@staticmethod
-	def item_loader(self, item):
-		try:
-			item = json.loads(item)
-		except json.JSONDecodeError:
-			return None
-
+	def on_json_loaded(self, item):
 		# form detection
 		forms = item.get('response', {}).get('forms', [])
 		if forms:

secator/tasks/maigret.py

@@ -7,7 +7,7 @@ from secator.decorators import task
 from secator.definitions import (DELAY, EXTRA_DATA, OPT_NOT_SUPPORTED, OUTPUT_PATH, PROXY,
 								 RATE_LIMIT, RETRIES, SITE_NAME, THREADS,
 								 TIMEOUT, URL, USERNAME)
-from secator.output_types import UserAccount
+from secator.output_types import UserAccount, Info, Error
 from secator.tasks._categories import ReconUser
 
 logger = logging.getLogger(__name__)
@@ -41,38 +41,40 @@ class maigret(ReconUser):
 			EXTRA_DATA: lambda x: x['status'].get('ids', {})
 		}
 	}
-	install_cmd = 'pipx install git+https://github.com/soxoj/maigret@6be2f409e58056b1ca8571a8151e53bef107dedc'
+	install_cmd = 'pipx install git+https://github.com/soxoj/maigret'
 	socks5_proxy = True
 	profile = 'io'
 
-	def yielder(self):
-		prev = self.print_item_count
-		self.print_item_count = False
-		yield from super().yielder()
-		if self.return_code != 0:
-			return
-		self.results = []
+	@staticmethod
+	def on_init(self):
+		self.output_path = self.get_opt_value(OUTPUT_PATH)
+
+	@staticmethod
+	def on_cmd_done(self):
+		# Search output path in cmd output
 		if not self.output_path:
-			match = re.search('JSON ndjson report for .* saved in (.*)', self.output)
-			if match is None:
-				logger.warning('JSON output file not found in command output.')
+			matches = re.findall('JSON ndjson report for .* saved in (.*)', self.output)
+			if not matches:
+				yield Error(message='JSON output file not found in command output.')
 				return
-			self.output_path = match.group(1)
-		note = f'maigret JSON results saved to {self.output_path}'
-		if self.print_line:
-			self._print(note)
-		if os.path.exists(self.output_path):
-			with open(self.output_path, 'r') as f:
+			self.output_path = matches
+
+		if not isinstance(self.output_path, list):
+			self.output_path = [self.output_path]
+
+		for path in self.output_path:
+			if not os.path.exists(path):
+				yield Error(message=f'Could not find JSON results in {path}')
+				return
+
+			yield Info(message=f'JSON results saved to {path}')
+			with open(path, 'r') as f:
 				data = [json.loads(line) for line in f.read().splitlines()]
 			for item in data:
 				yield item
-		self.print_item_count = prev
-
-	@staticmethod
-	def on_init(self):
-		output_path = self.get_opt_value(OUTPUT_PATH)
-		self.output_path = output_path
 
 	@staticmethod
 	def validate_item(self, item):
-		return item['http_status'] == 200
+		if isinstance(item, dict):
+			return item['http_status'] == 200
+		return True

secator/tasks/mapcidr.py

@@ -23,11 +23,10 @@ class mapcidr(ReconIp):
 		RATE_LIMIT: OPT_NOT_SUPPORTED,
 		RETRIES: OPT_NOT_SUPPORTED,
 		TIMEOUT: OPT_NOT_SUPPORTED,
-		THREADS: OPT_NOT_SUPPORTED,
 	}
 
 	@staticmethod
 	def item_loader(self, line):
 		if validators.ipv4(line) or validators.ipv6(line):
-			return {'ip': line, 'alive': False}
-		return None
+			yield {'ip': line, 'alive': False}
+		return

secator/tasks/msfconsole.py

@@ -4,6 +4,7 @@ import logging
 
 from rich.panel import Panel
 
+from secator.config import CONFIG
 from secator.decorators import task
 from secator.definitions import (DELAY, FOLLOW_REDIRECT, HEADER, HOST, OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT, RETRIES,
 								 THREADS, TIMEOUT, USER_AGENT)
@@ -21,7 +22,6 @@ class msfconsole(VulnMulti):
 	input_type = HOST
 	input_chunk_size = 1
 	output_types = []
-	output_return_type = str
 	opt_prefix = '--'
 	opts = {
 		'resource': {'type': str, 'help': 'Metasploit resource script.', 'short': 'r'},
@@ -40,18 +40,24 @@ class msfconsole(VulnMulti):
 		THREADS: OPT_NOT_SUPPORTED,
 		TIMEOUT: OPT_NOT_SUPPORTED,
 		USER_AGENT: OPT_NOT_SUPPORTED,
-		THREADS: OPT_NOT_SUPPORTED,
 	}
 	encoding = 'ansi'
 	ignore_return_code = True
-	# install_cmd = 'wget -O - https://raw.githubusercontent.com/freelabz/secator/main/scripts/msfinstall.sh | sh'
-
-	@staticmethod
-	def validate_input(self, input):
-		"""No list input supported for this command. Pass a single input instead."""
-		if isinstance(input, list):
-			return False
-		return True
+	install_pre = {
+		'apt|apk': ['libpq-dev', 'libpcap-dev', 'libffi-dev'],
+		'pacman': ['ruby-erb', 'postgresql-libs'],
+		'yum|zypper': ['postgresql-devel'],
+	}
+	install_cmd = (
+		f'git clone https://github.com/rapid7/metasploit-framework.git {CONFIG.dirs.share}/metasploit-framework || true && '
+		f'cd {CONFIG.dirs.share}/metasploit-framework && '
+		f'gem install bundler --user-install -n {CONFIG.dirs.bin} && '
+		f'gem install xmlrpc --user-install -n {CONFIG.dirs.bin} && '
+		f'bundle config set --local path "{CONFIG.dirs.share}" && '
+		'bundle update --bundler && '
+		'bundle install && '
+		f'ln -sf $HOME/.local/share/metasploit-framework/msfconsole {CONFIG.dirs.bin}/msfconsole'
+	)
 
 	@staticmethod
 	def on_init(self):
@@ -61,14 +67,14 @@ class msfconsole(VulnMulti):
 		env_vars = {}
 		if environment:
 			env_vars = dict(map(lambda x: x.split('='), environment.strip().split(',')))
-		env_vars['RHOST'] = self.input
-		env_vars['RHOSTS'] = self.input
+		env_vars['RHOST'] = self.inputs[0]
+		env_vars['RHOSTS'] = self.inputs[0]
 
 		# Passing msfconsole command directly, simply add RHOST / RHOSTS from host input and run then exit
 		if command:
 			self.run_opts['msfconsole.execute_command'] = (
-				f'setg RHOST {self.input}; '
-				f'setg RHOSTS {self.input}; '
+				f'setg RHOST {self.inputs[0]}; '
+				f'setg RHOSTS {self.inputs[0]}; '
 				f'{command.format(**env_vars)}; '
 				f'exit;'
 			)
@@ -92,17 +98,14 @@ class msfconsole(VulnMulti):
 				f.write(content)
 
 			script_name = script_path.split('/')[-1]
-			self._print(Panel(content, title=f'[bold magenta]{script_name}', expand=False))
+			self._print(Panel(content, title=f'[bold magenta]{script_name}', expand=False), rich=True)
 
 			# Override original command with new resource script
 			self.run_opts['msfconsole.resource'] = out_path
 
 		# Nothing passed, error out
 		else:
-			raise ValueError('At least one of "inline_script" or "resource_script" must be passed.')
-
-		# Clear host input
-		self.input = ''
+			raise ValueError('At least one of "execute_command" or "resource" must be passed.')
 
 
 # TODO: This is better as it goes through an RPC API to communicate with

secator/tasks/naabu.py

@@ -3,6 +3,7 @@ from secator.definitions import (DELAY, HOST, OPT_NOT_SUPPORTED, PORT, PORTS,
 								 PROXY, RATE_LIMIT, RETRIES, STATE, THREADS,
 								 TIMEOUT, TOP_PORTS)
 from secator.output_types import Port
+from secator.serializers import JSONSerializer
 from secator.tasks._categories import ReconPort
 
 
@@ -16,7 +17,7 @@ class naabu(ReconPort):
 	opts = {
 		PORTS: {'type': str, 'short': 'p', 'help': 'Ports'},
 		TOP_PORTS: {'type': str, 'short': 'tp', 'help': 'Top ports'},
-		'scan_type': {'type': str, 'help': 'Scan type (SYN (s)/CONNECT(c))'},
+		'scan_type': {'type': str, 'short': 'st', 'help': 'Scan type (SYN (s)/CONNECT(c))'},
 		# 'health_check': {'is_flag': True, 'short': 'hc', 'help': 'Health check'}
 	}
 	opt_key_map = {
@@ -37,6 +38,7 @@ class naabu(ReconPort):
 		RETRIES: lambda x: 1 if x == 0 else x,
 		PROXY: lambda x: x.replace('socks5://', '')
 	}
+	item_loaders = [JSONSerializer()]
 	output_map = {
 		Port: {
 			PORT: lambda x: x['port'],
@@ -45,9 +47,23 @@ class naabu(ReconPort):
 		}
 	}
 	output_types = [Port]
-	install_cmd = 'sudo apt install -y build-essential libpcap-dev && go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest'  # noqa: E501
+	install_cmd = 'go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest'
 	install_github_handle = 'projectdiscovery/naabu'
+	install_pre = {'apt|apk': ['libpcap-dev'], 'pacman|brew': ['libpcap']}
+	install_post = {'arch|alpine': 'sudo ln -s /usr/lib/libpcap.so /usr/lib/libpcap.so.0.8'}
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = False
 	profile = 'io'
+
+	@staticmethod
+	def before_init(self):
+		for ix, input in enumerate(self.inputs):
+			if input == 'localhost':
+				self.inputs[ix] = '127.0.0.1'
+
+	@staticmethod
+	def on_item(self, item):
+		if item.host == '127.0.0.1':
+			item.host = 'localhost'
+		return item