secator 0.0.1__py3-none-any.whl → 0.3.5__py3-none-any.whl

secator/runners/task.py

@@ -24,7 +24,7 @@ class Task(Runner):
 		# Get task class
 		task_cls = Task.get_task_class(self.config.name)
 
-		# Task opts
+		# Run opts
 		run_opts = self.run_opts.copy()
 		run_opts.pop('output', None)
 		dry_run = run_opts.get('show', False)
@@ -39,7 +39,8 @@ class Task(Runner):
 			'print_input_file': DEBUG > 0,
 			'print_item': True,
 			'print_item_count': not self.sync and not dry_run,
-			'print_line': not self.output_quiet,
+			'print_line': True
+			# 'print_line': self.sync and not self.output_quiet,
 		}
 		# self.print_item = not self.sync  # enable print_item for base Task only if running remote
 		run_opts.update(fmt_opts)
@@ -51,6 +52,7 @@ class Task(Runner):
 		hooks = {task_cls: self.hooks}
 		run_opts['hooks'] = hooks
 		run_opts['context'] = self.context
+		run_opts['reports_folder'] = self.reports_folder
 
 		# Run task
 		if self.sync:
@@ -58,9 +60,9 @@ class Task(Runner):
 			if dry_run:  # don't run
 				return
 		else:
-			result = task_cls.delay(self.targets, **run_opts)
+			self.celery_result = task_cls.delay(self.targets, **run_opts)
 			task = self.process_live_tasks(
-				result,
+				self.celery_result,
 				description=False,
 				results_only=True,
 				print_remote_status=self.print_remote_status)

secator/runners/workflow.py

@@ -1,5 +1,3 @@
-from celery import chain, chord
-
 from secator.definitions import DEBUG
 from secator.exporters import CsvExporter, JsonExporter
 from secator.output_types import Target
@@ -33,32 +31,34 @@ class Workflow(Runner):
 		for target in self.targets:
 			yield Target(name=target, _source=self.config.name, _type='target', _context=self.context)
 
-		# Task fmt opts
-		run_opts = self.run_opts.copy()
-		fmt_opts = {
-			'json': run_opts.get('json', False),
+		# Task opts
+		task_run_opts = self.run_opts.copy()
+		task_fmt_opts = {
+			'json': task_run_opts.get('json', False),
 			'print_cmd': True,
 			'print_cmd_prefix': not self.sync,
 			'print_description': self.sync,
-			'print_input_file': DEBUG,
+			'print_input_file': DEBUG > 0,
 			'print_item': True,
 			'print_item_count': True,
 			'print_line': not self.sync,
+			'print_progress': self.sync,
 		}
 
 		# Construct run opts
-		run_opts['hooks'] = self._hooks.get(Task, {})
-		run_opts.update(fmt_opts)
+		task_run_opts['hooks'] = self._hooks.get(Task, {})
+		task_run_opts['reports_folder'] = self.reports_folder
+		task_run_opts.update(task_fmt_opts)
 
 		# Build Celery workflow
-		workflow = self.build_celery_workflow(run_opts=run_opts, results=self.results)
+		workflow = self.build_celery_workflow(run_opts=task_run_opts, results=self.results)
 
 		# Run Celery workflow and get results
 		if self.sync:
 			results = workflow.apply().get()
 		else:
 			result = workflow()
-			self.result = result
+			self.celery_result = result
 			results = self.process_live_tasks(result, results_only=True, print_remote_status=self.print_remote_status)
 
 		# Get workflow results
@@ -70,6 +70,7 @@ class Workflow(Runner):
 		Returns:
 			celery.chain: Celery task chain.
 		"""
+		from celery import chain
 		from secator.celery import forward_results
 		sigs = self.get_tasks(
 			self.config.tasks.toDict(),
@@ -93,6 +94,7 @@ class Workflow(Runner):
 		Returns:
 			list: List of signatures.
 		"""
+		from celery import chain, chord
 		from secator.celery import forward_results
 		sigs = []
 		for task_name, task_opts in obj.items():

secator/tasks/_categories.py

@@ -6,17 +6,13 @@ import requests
 from bs4 import BeautifulSoup
 from cpe import CPE
 
-from secator.definitions import (CIDR_RANGE, CONFIDENCE, CVSS_SCORE,
-							   DEFAULT_HTTP_WORDLIST, DELAY, DEPTH, DESCRIPTION,
-							   FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
-							   FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID,
-							   MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
-							   MATCH_WORDS, METHOD, NAME, PATH, PROVIDER,
-							   PROXY, RATE_LIMIT, REFERENCES, RETRIES,
-							   SEVERITY, TAGS, DATA_FOLDER, THREADS, TIMEOUT,
-							   URL, USER_AGENT, USERNAME, WORDLIST)
-from secator.output_types import (Ip, Port, Subdomain, Tag, Url, UserAccount,
-								Vulnerability)
+from secator.definitions import (CIDR_RANGE, CONFIDENCE, CVSS_SCORE, DATA_FOLDER, DEFAULT_HTTP_WORDLIST,
+								 DEFAULT_SKIP_CVE_SEARCH, DELAY, DEPTH, DESCRIPTION, FILTER_CODES, FILTER_REGEX,
+								 FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID, MATCH_CODES, MATCH_REGEX,
+								 MATCH_SIZE, MATCH_WORDS, METHOD, NAME, PATH, PROVIDER, PROXY, RATE_LIMIT, REFERENCES,
+								 RETRIES, SEVERITY, TAGS, THREADS, TIMEOUT, URL, USER_AGENT, USERNAME, WORDLIST)
+from secator.output_types import Ip, Port, Subdomain, Tag, Url, UserAccount, Vulnerability
+from secator.rich import console
 from secator.runners import Command
 
 logger = logging.getLogger(__name__)
@@ -63,10 +59,6 @@ OPTS_VULN = [
 	HEADER, DELAY, FOLLOW_REDIRECT, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT
 ]
 
-OPTS_OSINT = [
-
-]
-
 
 #---------------#
 # HTTP category #
@@ -159,14 +151,17 @@ class Vuln(Command):
 		"""
 		cve_info = Vuln.lookup_local_cve(cve_id)
 		if not cve_info:
-			# logger.debug(f'{cve_id} not found locally. Use `secator utils download-cves` to update the local database.')
+			if DEFAULT_SKIP_CVE_SEARCH:
+				logger.debug(f'{cve_id} not found locally, and DEFAULT_SKIP_CVE_SEARCH is set: ignoring online search.')
+				return None
+			# logger.debug(f'{cve_id} not found locally. Use `secator install cves` to install CVEs locally.')
 			try:
 				cve_info = requests.get(f'https://cve.circl.lu/api/cve/{cve_id}', timeout=5).json()
 				if not cve_info:
-					logger.error(f'Could not fetch CVE info for cve {cve_id}. Skipping.')
-					return
+					console.print(f'Could not fetch CVE info for cve {cve_id}. Skipping.', highlight=False)
+					return None
 			except Exception:
-				logger.error(f'Could not fetch CVE info for cve {cve_id}. Skipping.')
+				console.print(f'Could not fetch CVE info for cve {cve_id}. Skipping.', highlight=False)
 				return None
 
 		# Match the CPE string against the affected products CPE FS strings from the CVE data if a CPE was passed.

secator/tasks/cariddi.py

@@ -43,6 +43,7 @@ class cariddi(HttpCrawler):
 	}
 	item_loaders = []
 	install_cmd = 'go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest'
+	install_github_handle = 'edoardottt/cariddi'
 	encoding = 'ansi'
 	proxychains = False
 	proxy_socks5 = True  # with leaks... https://github.com/edoardottt/cariddi/issues/122
@@ -88,7 +89,7 @@ class cariddi(HttpCrawler):
 				items.append(secret)
 
 			for info in infos:
-				CARIDDI_IGNORE_LIST = ['BTC address']
+				CARIDDI_IGNORE_LIST = ['BTC address']  # TODO: make this a config option
 				if info['name'] in CARIDDI_IGNORE_LIST:
 					continue
 				match = info['match']

secator/tasks/dalfox.py

@@ -24,6 +24,7 @@ class dalfox(VulnHttp):
 	input_flag = 'url'
 	file_flag = 'file'
 	json_flag = '--format json'
+	version_flag = 'version'
 	opt_prefix = '--'
 	opt_key_map = {
 		HEADER: 'header',
@@ -52,6 +53,7 @@ class dalfox(VulnHttp):
 		}
 	}
 	install_cmd = 'go install -v github.com/hahwul/dalfox/v2@latest'
+	install_github_handle = 'hahwul/dalfox'
 	encoding = 'ansi'
 	proxychains = False
 	proxychains_flavor = 'proxychains4'

secator/tasks/dirsearch.py

@@ -7,12 +7,11 @@ from secator.definitions import (CONTENT_LENGTH, CONTENT_TYPE, DELAY, DEPTH,
 							   FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
 							   FILTER_WORDS, FOLLOW_REDIRECT, HEADER,
 							   MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
-							   MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
-							   RATE_LIMIT, RETRIES, STATUS_CODE, DATA_FOLDER,
+							   MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, OUTPUT_PATH, PROXY,
+							   RATE_LIMIT, RETRIES, STATUS_CODE,
 							   THREADS, TIMEOUT, USER_AGENT, WORDLIST)
 from secator.output_types import Url
 from secator.tasks._categories import HttpFuzzer
-from secator.utils import get_file_timestamp
 
 
 @task()
@@ -53,7 +52,7 @@ class dirsearch(HttpFuzzer):
 			STATUS_CODE: 'status'
 		}
 	}
-	install_cmd = 'pip3 install dirsearch'
+	install_cmd = 'pipx install dirsearch'
 	proxychains = True
 	proxy_socks5 = True
 	proxy_http = True
@@ -83,8 +82,7 @@ class dirsearch(HttpFuzzer):
 
 	@staticmethod
 	def on_init(self):
-		self.output_path = self.get_opt_value('output_path')
+		self.output_path = self.get_opt_value(OUTPUT_PATH)
 		if not self.output_path:
-			timestr = get_file_timestamp()
-			self.output_path = f'{DATA_FOLDER}/dirsearch_{timestr}.json'
+			self.output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.json'
 		self.cmd += f' -o {self.output_path}'

secator/tasks/dnsx.py

@@ -25,6 +25,7 @@ class dnsx(ReconDns):
 	}
 
 	install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest'
+	install_github_handle = 'projectdiscovery/dnsx'
 	profile = 'io'
 
 	@staticmethod

secator/tasks/dnsxbrute.py

@@ -31,4 +31,5 @@ class dnsxbrute(ReconDns):
         }
     }
     install_cmd = 'go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest'
+    install_github_handle = 'projectdiscovery/dnsx'
     profile = 'cpu'

secator/tasks/feroxbuster.py

@@ -6,12 +6,11 @@ from secator.definitions import (CONTENT_TYPE, DELAY, DEPTH, FILTER_CODES,
 							   FILTER_REGEX, FILTER_SIZE, FILTER_WORDS,
 							   FOLLOW_REDIRECT, HEADER, LINES, MATCH_CODES,
 							   MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, METHOD,
-							   OPT_NOT_SUPPORTED, OPT_PIPE_INPUT, PROXY,
-							   RATE_LIMIT, RETRIES, STATUS_CODE, DATA_FOLDER,
+							   OPT_NOT_SUPPORTED, OPT_PIPE_INPUT, OUTPUT_PATH, PROXY,
+							   RATE_LIMIT, RETRIES, STATUS_CODE,
 							   THREADS, TIMEOUT, USER_AGENT, WORDLIST, WORDS, DEFAULT_FEROXBUSTER_FLAGS)
 from secator.output_types import Url
 from secator.tasks._categories import HttpFuzzer
-from secator.utils import get_file_timestamp
 
 
 @task()
@@ -61,10 +60,11 @@ class feroxbuster(HttpFuzzer):
 		}
 	}
 	install_cmd = (
-		'sudo apt install -y unzip && '
+		'sudo apt install -y unzip curl && '
 		'curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | '
 		'bash && sudo mv feroxbuster /usr/local/bin'
 	)
+	install_github_handle = 'epi052/feroxbuster'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True
@@ -72,10 +72,9 @@ class feroxbuster(HttpFuzzer):
 
 	@staticmethod
 	def on_init(self):
-		self.output_path = self.get_opt_value('output_path')
+		self.output_path = self.get_opt_value(OUTPUT_PATH)
 		if not self.output_path:
-			timestr = get_file_timestamp()
-			self.output_path = f'{DATA_FOLDER}/feroxbuster_{timestr}.json'
+			self.output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.json'
 		Path(self.output_path).touch()
 		self.cmd += f' --output {self.output_path}'
 

secator/tasks/ffuf.py

@@ -7,7 +7,7 @@ from secator.definitions import (AUTO_CALIBRATION, CONTENT_LENGTH,
 								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED,
 								 PERCENT, PROXY, RATE_LIMIT, RETRIES,
 								 STATUS_CODE, THREADS, TIME, TIMEOUT,
-								 USER_AGENT, WORDLIST)
+								 USER_AGENT, WORDLIST, WORDLISTS_FOLDER)
 from secator.output_types import Progress, Url
 from secator.serializers import JSONSerializer, RegexSerializer
 from secator.tasks._categories import HttpFuzzer
@@ -23,6 +23,7 @@ class ffuf(HttpFuzzer):
 	input_chunk_size = 1
 	file_flag = None
 	json_flag = '-json'
+	version_flag = '-V'
 	item_loaders = [
 		JSONSerializer(),
 		RegexSerializer(FFUF_PROGRESS_REGEX, fields=['count', 'total', 'rps', 'duration', 'errors'])
@@ -69,10 +70,8 @@ class ffuf(HttpFuzzer):
 		},
 	}
 	encoding = 'ansi'
-	install_cmd = (
-		'go install -v github.com/ffuf/ffuf@latest && '
-		'sudo git clone https://github.com/danielmiessler/SecLists /usr/share/seclists || true'
-	)
+	install_cmd = f'go install -v github.com/ffuf/ffuf@latest && sudo git clone https://github.com/danielmiessler/SecLists {WORDLISTS_FOLDER}/seclists || true'  # noqa: E501
+	install_github_handle = 'ffuf/ffuf'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True
@@ -82,5 +81,3 @@ class ffuf(HttpFuzzer):
 	def on_item(self, item):
 		item.method = self.get_opt_value(METHOD) or 'GET'
 		return item
-
-	# TODO: write custom item_loader to pick up Progress items too

secator/tasks/gau.py

@@ -37,11 +37,8 @@ class gau(HttpCrawler):
 		USER_AGENT: OPT_NOT_SUPPORTED,
 	}
 	install_cmd = 'go install -v github.com/lc/gau/v2/cmd/gau@latest'
+	install_github_handle = 'lc/gau'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True
 	profile = 'io'
-
-	# @staticmethod
-	# def validate_item(self, item):
-	# 	return item['url'] == 'response'

secator/tasks/gf.py

@@ -1,5 +1,5 @@
 from secator.decorators import task
-from secator.definitions import OPT_PIPE_INPUT, URL
+from secator.definitions import OPT_PIPE_INPUT, OPT_NOT_SUPPORTED, URL
 from secator.output_types import Tag
 from secator.tasks._categories import Tagger
 
@@ -10,6 +10,7 @@ class gf(Tagger):
 	cmd = 'gf'
 	file_flag = OPT_PIPE_INPUT
 	input_flag = OPT_PIPE_INPUT
+	version_flag = OPT_NOT_SUPPORTED
 	opts = {
 		'pattern': {'type': str, 'help': 'Pattern names to match against (comma-delimited)'}
 	}

secator/tasks/gospider.py

@@ -52,6 +52,7 @@ class gospider(HttpCrawler):
 		}
 	}
 	install_cmd = 'go install -v github.com/jaeles-project/gospider@latest'
+	install_github_handle = 'jaeles-project/gospider'
 	ignore_return_code = True
 	proxychains = False
 	proxy_socks5 = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61

secator/tasks/grype.py

@@ -7,52 +7,6 @@ from secator.output_types import Vulnerability
 from secator.tasks._categories import VulnCode
 
 
-def grype_item_loader(self, line):
-	"""Load vulnerabilty dicts from grype line output."""
-	split = [i for i in line.split(' ') if i]
-	if not len(split) in [5, 6] or split[0] == 'NAME':
-		return None
-	version_fixed = None
-	if len(split) == 5:  # no version fixed
-		product, version, product_type, vuln_id, severity = tuple(split)
-	elif len(split) == 6:
-		product, version, version_fixed, product_type, vuln_id, severity = tuple(split)
-	extra_data = {
-		'lang': product_type,
-		'product': product,
-		'version': version,
-	}
-	if version_fixed:
-		extra_data['version_fixed'] = version_fixed
-	data = {
-		'id': vuln_id,
-		'name': vuln_id,
-		'matched_at': self.input,
-		'confidence': 'medium',
-		'severity': severity.lower(),
-		'provider': 'grype',
-		'cvss_score': -1,
-		'tags': [],
-	}
-	if vuln_id.startswith('GHSA'):
-		data['provider'] = 'github.com'
-		data['references'] = [f'https://github.com/advisories/{vuln_id}']
-		data['tags'].extend(['cve', 'ghsa'])
-		vuln = VulnCode.lookup_ghsa(vuln_id)
-		if vuln:
-			data.update(vuln)
-			data['severity'] = data['severity'] or severity.lower()
-			extra_data['ghsa_id'] = vuln_id
-	elif vuln_id.startswith('CVE'):
-		vuln = VulnCode.lookup_cve(vuln_id)
-		if vuln:
-			vuln['tags'].append('cve')
-			data.update(vuln)
-			data['severity'] = data['severity'] or severity.lower()
-	data['extra_data'] = extra_data
-	return data
-
-
 @task()
 class grype(VulnCode):
 	"""Vulnerability scanner for container images and filesystems."""
@@ -76,4 +30,50 @@ class grype(VulnCode):
 	install_cmd = (
 		'curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin'
 	)
-	item_loaders = [grype_item_loader]
+	install_github_handle = 'anchore/grype'
+
+	@staticmethod
+	def item_loader(self, line):
+		"""Load vulnerabilty dicts from grype line output."""
+		split = [i for i in line.split(' ') if i]
+		if not len(split) in [5, 6] or split[0] == 'NAME':
+			return None
+		version_fixed = None
+		if len(split) == 5:  # no version fixed
+			product, version, product_type, vuln_id, severity = tuple(split)
+		elif len(split) == 6:
+			product, version, version_fixed, product_type, vuln_id, severity = tuple(split)
+		extra_data = {
+			'lang': product_type,
+			'product': product,
+			'version': version,
+		}
+		if version_fixed:
+			extra_data['version_fixed'] = version_fixed
+		data = {
+			'id': vuln_id,
+			'name': vuln_id,
+			'matched_at': self.input,
+			'confidence': 'medium',
+			'severity': severity.lower(),
+			'provider': 'grype',
+			'cvss_score': -1,
+			'tags': [],
+		}
+		if vuln_id.startswith('GHSA'):
+			data['provider'] = 'github.com'
+			data['references'] = [f'https://github.com/advisories/{vuln_id}']
+			data['tags'].extend(['cve', 'ghsa'])
+			vuln = VulnCode.lookup_ghsa(vuln_id)
+			if vuln:
+				data.update(vuln)
+				data['severity'] = data['severity'] or severity.lower()
+				extra_data['ghsa_id'] = vuln_id
+		elif vuln_id.startswith('CVE'):
+			vuln = VulnCode.lookup_cve(vuln_id)
+			if vuln:
+				vuln['tags'].append('cve')
+				data.update(vuln)
+				data['severity'] = data['severity'] or severity.lower()
+		data['extra_data'] = extra_data
+		return data

secator/tasks/h8mail.py

@@ -2,9 +2,8 @@ import os
 import json
 
 from secator.decorators import task
-from secator.definitions import (EMAIL, DATA_FOLDER)
+from secator.definitions import EMAIL, OUTPUT_PATH
 from secator.tasks._categories import OSInt
-from secator.utils import get_file_timestamp
 from secator.output_types import UserAccount
 
 
@@ -16,6 +15,7 @@ class h8mail(OSInt):
 	input_flag = '--targets'
 	input_type = EMAIL
 	file_flag = '-domain'
+	version_flag = '--help'
 	opt_prefix = '--'
 	opt_key_map = {
 
@@ -27,14 +27,13 @@ class h8mail(OSInt):
 	output_map = {
 	}
 
-	install_cmd = 'pip3 install h8mail'
+	install_cmd = 'pipx install h8mail'
 
 	@staticmethod
 	def on_start(self):
-		output_path = self.get_opt_value('output_path')
+		output_path = self.get_opt_value(OUTPUT_PATH)
 		if not output_path:
-			timestr = get_file_timestamp()
-			output_path = f'{DATA_FOLDER}/h8mail_{timestr}.json'
+			output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.json'
 		self.output_path = output_path
 		self.cmd = self.cmd.replace('--json', f'--json {self.output_path}')
 

secator/tasks/httpx.py

@@ -1,5 +1,4 @@
 import os
-import uuid
 
 from secator.decorators import task
 from secator.definitions import (DEFAULT_HTTPX_FLAGS,
@@ -8,7 +7,7 @@ from secator.definitions import (DEFAULT_HTTPX_FLAGS,
 								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER,
 								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
 								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
-								 RATE_LIMIT, RETRIES, TASKS_FOLDER, THREADS,
+								 RATE_LIMIT, RETRIES, THREADS,
 								 TIMEOUT, URL, USER_AGENT)
 from secator.tasks._categories import Http
 from secator.utils import sanitize_url
@@ -24,14 +23,17 @@ class httpx(Http):
 	opts = {
 		# 'silent': {'is_flag': True, 'default': False, 'help': 'Silent mode'},
 		# 'td': {'is_flag': True, 'default': True, 'help': 'Tech detection'},
-		'irr': {'is_flag': True, 'default': False, 'help': 'Include http request / response'},
+		# 'irr': {'is_flag': True, 'default': False, 'help': 'Include http request / response'},
 		'fep': {'is_flag': True, 'default': False, 'help': 'Error Page Classifier and Filtering'},
 		'favicon': {'is_flag': True, 'default': False, 'help': 'Favicon hash'},
 		'jarm': {'is_flag': True, 'default': False, 'help': 'Jarm fingerprint'},
 		'asn': {'is_flag': True, 'default': False, 'help': 'ASN detection'},
 		'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'},
 		'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'},
-		'screenshot': {'is_flag': True, 'default': False, 'help': 'Screenshot response'}
+		'vhost': {'is_flag': True, 'default': False, 'help': 'Probe and display server supporting VHOST'},
+		'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'},
+		'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'},
+		'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'},
 	}
 	opt_key_map = {
 		HEADER: 'header',
@@ -58,11 +60,29 @@ class httpx(Http):
 		DELAY: lambda x: str(x) + 's' if x else None,
 	}
 	install_cmd = 'go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest'
+	install_github_handle = 'projectdiscovery/httpx'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True
 	profile = 'cpu'
 
+	@staticmethod
+	def on_init(self):
+		debug_resp = self.get_opt_value('debug_resp')
+		if debug_resp:
+			self.cmd = self.cmd.replace('-silent', '')
+		if DEFAULT_STORE_HTTP_RESPONSES:
+			self.output_response_path = f'{self.reports_folder}/response'
+			self.output_screenshot_path = f'{self.reports_folder}/screenshot'
+			os.makedirs(self.output_response_path, exist_ok=True)
+			os.makedirs(self.output_screenshot_path, exist_ok=True)
+			self.cmd += f' -sr -srd {self.reports_folder}'
+
+		# Remove screenshot bytes and body bytes when screenshot
+		screenshot = self.get_opt_value('screenshot')
+		if screenshot:
+			self.cmd += ' -esb -ehb'
+
 	@staticmethod
 	def on_item_pre_convert(self, item):
 		for k, v in item.items():
@@ -76,20 +96,6 @@ class httpx(Http):
 		item[URL] = item.get('final_url') or item[URL]
 		return item
 
-	@staticmethod
-	def on_init(self):
-		debug_resp = self.get_opt_value('debug_resp')
-		if debug_resp:
-			self.cmd = self.cmd.replace('-silent', '')
-		if DEFAULT_STORE_HTTP_RESPONSES:
-			_id = uuid.uuid4()
-			output_path = f'{TASKS_FOLDER}/{_id}'
-			self.output_response_path = f'{output_path}/response'
-			self.output_screenshot_path = f'{output_path}/screenshot'
-			os.makedirs(self.output_response_path, exist_ok=True)
-			os.makedirs(self.output_screenshot_path, exist_ok=True)
-			self.cmd += f' -sr -srd {output_path}'
-
 	@staticmethod
 	def on_end(self):
 		if DEFAULT_STORE_HTTP_RESPONSES:

secator/tasks/katana.py

@@ -1,6 +1,5 @@
 import os
 import json
-import uuid
 from urllib.parse import urlparse
 
 from secator.decorators import task
@@ -11,7 +10,7 @@ from secator.definitions import (CONTENT_TYPE, DEFAULT_KATANA_FLAGS,
 								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
 								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED, PROXY,
 								 RATE_LIMIT, RETRIES, STATUS_CODE,
-								 STORED_RESPONSE_PATH, TASKS_FOLDER, TECH,
+								 STORED_RESPONSE_PATH, TECH,
 								 THREADS, TIME, TIMEOUT, URL, USER_AGENT, WEBSERVER, CONTENT_LENGTH)
 from secator.output_types import Url, Tag
 from secator.tasks._categories import HttpCrawler
@@ -71,7 +70,8 @@ class katana(HttpCrawler):
 		}
 	}
 	item_loaders = []
-	install_cmd = 'go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
+	install_cmd = 'sudo apt install build-essential && go install -v github.com/projectdiscovery/katana/cmd/katana@latest'
+	install_github_handle = 'projectdiscovery/katana'
 	proxychains = False
 	proxy_socks5 = True
 	proxy_http = True
@@ -107,27 +107,23 @@ class katana(HttpCrawler):
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
 		if DEFAULT_STORE_HTTP_RESPONSES:
-			_id = uuid.uuid4()
-			output_path = f'{TASKS_FOLDER}/{_id}'
-			self.output_response_path = output_path
-			os.makedirs(self.output_response_path, exist_ok=True)
-			self.cmd += f' -sr -srd {output_path}'
-
-	@staticmethod
-	def on_end(self):
-		if DEFAULT_STORE_HTTP_RESPONSES and os.path.exists(self.output_response_path + '/index.txt'):
-			os.remove(self.output_response_path + '/index.txt')
+			self.cmd += f' -sr -srd {self.reports_folder}'
 
 	@staticmethod
 	def on_item(self, item):
 		if not isinstance(item, Url):
 			return item
 		if DEFAULT_STORE_HTTP_RESPONSES and os.path.exists(item.stored_response_path):
-			with open(item.stored_response_path, 'r') as fin:
+			with open(item.stored_response_path, 'r', encoding='latin-1') as fin:
 				data = fin.read().splitlines(True)
 				first_line = data[0]
-			with open(item.stored_response_path, 'w') as fout:
+			with open(item.stored_response_path, 'w', encoding='latin-1') as fout:
 				fout.writelines(data[1:])
 				fout.writelines('\n')
 				fout.writelines(first_line)
 		return item
+
+	@staticmethod
+	def on_end(self):
+		if DEFAULT_STORE_HTTP_RESPONSES and os.path.exists(self.reports_folder + '/index.txt'):
+			os.remove(self.reports_folder + '/index.txt')