secator 0.22.0__py3-none-any.whl

secator/tasks/ffuf.py

@@ -0,0 +1,135 @@
+from secator.decorators import task
+from secator.definitions import (AUTO_CALIBRATION, DATA, DELAY, DEPTH, EXTRA_DATA,
+								 FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
+								 FILTER_WORDS, FOLLOW_REDIRECT, HEADER,
+								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
+								 MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED,
+								 PERCENT, PROXY, RATE_LIMIT, RETRIES,
+								 THREADS, TIMEOUT, USER_AGENT, WORDLIST, URL)
+from secator.output_types import Progress, Url, Subdomain, Info, Warning
+from secator.serializers import JSONSerializer, RegexSerializer
+from secator.tasks._categories import HttpFuzzer
+from secator.utils import extract_domain_info
+
+
+FFUF_PROGRESS_REGEX = r':: Progress: \[(?P<count>\d+)/(?P<total>\d+)\] :: Job \[\d/\d\] :: (?P<rps>\d+) req/sec :: Duration: \[(?P<duration>[\d:]+)\] :: Errors: (?P<errors>\d+) ::'  # noqa: E501
+
+
+@task()
+class ffuf(HttpFuzzer):
+	"""Fast web fuzzer written in Go."""
+	cmd = 'ffuf -noninteractive'
+	input_types = [URL]
+	output_types = [Url, Subdomain, Progress]
+	tags = ['url', 'fuzz']
+	input_flag = '-u'
+	input_chunk_size = 1
+	file_flag = None
+	json_flag = '-json'
+	version_flag = '-V'
+	item_loaders = [
+		JSONSerializer(strict=True),
+		RegexSerializer(FFUF_PROGRESS_REGEX, fields=['count', 'total', 'rps', 'duration', 'errors'])
+	]
+	opts = {
+		AUTO_CALIBRATION: {'is_flag': True, 'default': True, 'short': 'ac', 'help': 'Auto-calibration'},
+		'recursion': {'is_flag': True, 'default': False, 'short': 'recursion', 'help': 'Recursion'},
+		'stop_on_error': {'is_flag': True, 'default': False, 'short': 'soe', 'help': 'Stop on error'},
+		'fuzz_host_header': {'is_flag': True, 'default': False, 'internal': True, 'short': 'fhh', 'help': 'Fuzz host header'},
+	}
+	opt_key_map = {
+		HEADER: 'H',
+		DATA: 'd',
+		DELAY: 'p',
+		DEPTH: 'recursion-depth',
+		FILTER_CODES: 'fc',
+		FILTER_REGEX: 'fr',
+		FILTER_SIZE: 'fs',
+		FILTER_WORDS: 'fw',
+		FOLLOW_REDIRECT: 'r',
+		MATCH_CODES: 'mc',
+		MATCH_REGEX: 'mr',
+		MATCH_SIZE: 'ms',
+		MATCH_WORDS: 'mw',
+		METHOD: 'X',
+		PROXY: 'x',
+		RATE_LIMIT: 'rate',
+		RETRIES: OPT_NOT_SUPPORTED,
+		THREADS: 't',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED,
+
+		# ffuf opts
+		WORDLIST: 'w',
+		AUTO_CALIBRATION: 'ac',
+		'stop_on_error': 'sa',
+	}
+	output_map = {
+		Progress: {
+			PERCENT: lambda x: int(int(x['count']) * 100 / int(x['total'])),
+			EXTRA_DATA: lambda x: x
+		},
+	}
+	encoding = 'ansi'
+	install_version = 'v2.1.0'
+	install_cmd = 'go install -v github.com/ffuf/ffuf/v2@[install_version]'
+	github_handle = 'ffuf/ffuf'
+	proxychains = False
+	proxy_socks5 = True
+	proxy_http = True
+
+	@staticmethod
+	def before_init(self):
+		# Add /FUZZ to URL if recursion is enabled
+		if self.get_opt_value('recursion') and not len(self.inputs) > 1 and not self.inputs[0].endswith('FUZZ'):
+			self._print(Info(message='Adding /FUZZ to URL as it is needed when recursion is enabled'), rich=True)
+			self.inputs[0] = self.inputs[0].rstrip('/') + '/FUZZ'
+
+	@staticmethod
+	def on_cmd_opts(self, opts):
+		# Fuzz host header
+		if self.get_opt_value('fuzz_host_header') and len(self.inputs) > 0:
+			host = self.inputs[0].split('://')[1].split('/')[0]
+			opts['header']['value']['Host'] = f'FUZZ.{host}'
+		self.headers = opts['header']['value'].copy()
+
+		# Check FUZZ keyword
+		data = self.get_opt_value('data') or ''
+		fuzz_in_headers = any('FUZZ' in v for v in self.headers.values())
+		if len(self.inputs) > 0 and 'FUZZ' not in self.inputs[0] and not fuzz_in_headers and 'FUZZ' not in data:
+			self.add_result(Warning(message='Keyword FUZZ is not present in the URL, header or body'))
+		return opts
+
+	@staticmethod
+	def on_json_loaded(self, item):
+		if 'host' in item:
+			self.current_host = item['host']
+		headers = self.headers.copy()
+		if 'FUZZ' in headers.get('Host', ''):
+			headers['Host'] = self.current_host
+		yield Url(
+			url=item['url'],
+			host=item['host'],
+			status_code=item['status'],
+			content_length=item['length'],
+			content_type=item['content-type'],
+			time=item['duration'] * 10**-9,
+			method=self.get_opt_value(METHOD) or 'GET',
+			request_headers=headers,
+		)
+		if self.get_opt_value('fuzz_host_header'):
+			yield Subdomain(
+				host=item['host'],
+				verified=False,
+				domain=extract_domain_info(item['host'], domain_only=True),
+				sources=['http_host_header']
+			)
+
+	@staticmethod
+	def on_item(self, item):
+		if isinstance(item, Url):
+			item.method = self.get_opt_value(METHOD) or 'GET'
+			item.request_headers = self.headers.copy()
+			if 'FUZZ' in self.headers.get('Host', ''):
+				item.request_headers['Host'] = self.current_host
+		return item

secator/tasks/fping.py

@@ -0,0 +1,85 @@
+import validators
+
+from secator.decorators import task
+from secator.definitions import (DELAY, IP, HOST, OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT,
+							   RETRIES, THREADS, TIMEOUT, CIDR_RANGE)
+from secator.output_types import Ip
+from secator.tasks._categories import ReconIp
+from secator.utils import validate_cidr_range
+
+
+@task()
+class fping(ReconIp):
+	"""Send ICMP echo probes to network hosts, similar to ping, but much better."""
+	cmd = 'fping -a -A'
+	input_types = [IP, HOST, CIDR_RANGE]
+	output_types = [Ip]
+	tags = ['ip', 'recon']
+	file_flag = '-f'
+	input_flag = None
+	opts = {
+		'count': {'type': int, 'default': None, 'help': 'Number of request packets to send to each target'},
+		'show_name': {'is_flag': True, 'default': False, 'help': 'Show network addresses as well as hostnames'},
+		'use_dns': {'is_flag': True, 'default': False, 'help': 'Use DNS to lookup address of return packet (same as -n but will force reverse-DNS lookup for hostnames)'},  # noqa: E501
+		'summary': {'is_flag': True, 'default': False, 'help': 'Print cumulative statistics upon exit'},
+	}
+	opt_prefix = '--'
+	opt_key_map = {
+		DELAY: 'period',
+		PROXY: OPT_NOT_SUPPORTED,
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: 'retry',
+		TIMEOUT: 'timeout',
+		THREADS: OPT_NOT_SUPPORTED,
+		'count': '-c',
+		'show_name': '-n',
+		'use_dns': '-d',
+		'summary': '-s',
+	}
+	opt_value_map = {
+		DELAY: lambda x: int(x) * 1000,  # convert s to ms
+		TIMEOUT: lambda x: int(x) * 1000  # convert s to ms
+	}
+	github_handle = 'schweikert/fping'
+	install_github_bin = False
+	install_version = 'v5.1'
+	install_pre = {'*': ['fping']}
+	ignore_return_code = True
+
+	@staticmethod
+	def before_init(self):
+		for input in self.inputs:
+			if validate_cidr_range(input):
+				self.file_flag = None
+				self.input_chunk_size = 1
+				self.input_flag = '-g'
+
+	@staticmethod
+	def item_loader(self, line):
+		if '(' in line:
+
+			line_part = line.split(' : ')[0] if ' : ' in line else line    # Removing the stat parts that appears when using -c
+
+			start_paren = line_part.find('(')
+			end_paren = line_part.find(')', start_paren)
+
+			if start_paren != -1 and end_paren != -1:
+				host = line_part[:start_paren].strip()
+				ip = line_part[start_paren+1:end_paren].strip()
+
+				if (validators.ipv4(host) or validators.ipv6(host)):
+					host = ''
+			else:
+				return
+		else:
+			ip = line.strip()
+			host = ''
+		if not (validators.ipv4(ip) or validators.ipv6(ip)):
+			return
+		yield Ip(ip=ip, alive=True, host=host, extra_data={'protocol': 'icmp'})
+
+	@staticmethod
+	def on_line(self, line):
+		if 'Unreachable' in line:
+			return ''  # discard line as it pollutes output
+		return line

secator/tasks/gau.py

@@ -0,0 +1,102 @@
+import json
+from collections import defaultdict
+from urllib.parse import urlparse, urlunparse, parse_qs
+
+from secator.decorators import task
+from secator.definitions import (DELAY, DEPTH, FILTER_CODES, FILTER_REGEX,
+							   FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT,
+							   HEADER, MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
+							   MATCH_WORDS, METHOD, OPT_NOT_SUPPORTED,
+							   OPT_PIPE_INPUT, PROXY, RATE_LIMIT, RETRIES,
+							   THREADS, TIMEOUT, USER_AGENT, URL, HOST)
+from secator.output_types import Subdomain, Url, Warning
+from secator.serializers import JSONSerializer
+from secator.tasks._categories import HttpCrawler
+from secator.utils import extract_domain_info
+
+
+@task()
+class gau(HttpCrawler):
+	"""Fetch known URLs from AlienVault's Open Threat Exchange, the Wayback Machine, Common Crawl, and URLScan."""
+	cmd = 'gau --verbose'
+	input_types = [URL, HOST]
+	output_types = [Url, Subdomain]
+	tags = ['pattern', 'scan']
+	file_flag = OPT_PIPE_INPUT
+	json_flag = '--json'
+	opt_prefix = '--'
+	encoding = 'ansi'
+	opts = {
+		'providers': {'type': str, 'default': None, 'help': 'List of providers to use (wayback,commoncrawl,otx,urlscan)'},
+		'subs': {'is_flag': True, 'default': False, 'help': 'Output subdomains as well asURLs'},
+		'max_param_occurrences': {'type': int, 'help': 'Max occurrences for the same parameter in the same URL before discarding next results', 'required': False, 'default': 10, 'internal': True},  # noqa: E501
+	}
+	opt_key_map = {
+		HEADER: OPT_NOT_SUPPORTED,
+		DELAY: OPT_NOT_SUPPORTED,
+		DEPTH: OPT_NOT_SUPPORTED,
+		FILTER_CODES: 'fc',
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		MATCH_CODES: 'mc',
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: 'retries',
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED,
+	}
+	item_loaders = [JSONSerializer()]
+	install_pre = {'apk': ['libc6-compat']}
+	install_version = 'v2.2.4'
+	install_cmd = 'go install -v github.com/lc/gau/v2/cmd/gau@[install_version]'
+	github_handle = 'lc/gau'
+	proxychains = False
+	proxy_socks5 = True
+	proxy_http = True
+	profile = 'io'
+
+	@staticmethod
+	def on_init(self):
+		self.max_param_occurrences = self.get_opt_value('max_param_occurrences')
+		self.seen_params = defaultdict(lambda: defaultdict(int))
+		self.subdomains = []
+
+	@staticmethod
+	def on_line(self, line):
+		if 'level=warning' in line and 'error reading config' not in line:
+			msg = line.split('msg=')[-1].rstrip('""').lstrip('"')
+			if not msg.startswith('http'):
+				msg = msg.capitalize()
+			return json.dumps({'message': msg, '_type': 'warning'})
+		return line
+
+	@staticmethod
+	def on_json_loaded(self, item):
+		if item.get('message'):
+			yield Warning(message=item['message'])
+			return
+		url = item['url']
+		parsed_url = urlparse(url)
+		base_url = urlunparse(parsed_url._replace(query="", fragment=""))  # Remove query & fragment
+		query_params = parse_qs(parsed_url.query)
+		current_params = set(query_params.keys())
+		for param in current_params:
+			self.seen_params[base_url][param] += 1
+			if self.seen_params[base_url][param] > int(self.max_param_occurrences):
+				return
+		if self.get_opt_value('subs'):
+			domain = extract_domain_info(parsed_url.hostname, domain_only=True)
+			if domain:
+				subdomain = Subdomain(host=parsed_url.hostname, domain=domain)
+				if subdomain not in self.subdomains:
+					self.subdomains.append(subdomain)
+					yield subdomain
+		else:
+			yield Url(url=item['url'], host=parsed_url.hostname)

secator/tasks/getasn.py

@@ -0,0 +1,60 @@
+from secator.decorators import task
+from secator.definitions import (DELAY, OPT_PIPE_INPUT, IP, HOST)
+from secator.output_types import Tag
+from secator.tasks._categories import Command
+
+
+@task()
+class getasn(Command):
+	"""Get ASN information from IP address."""
+	cmd = 'getasn'
+	input_chunk_size = 1
+	input_types = [IP, HOST]
+	input_flag = OPT_PIPE_INPUT
+	file_flag = None
+	output_types = [Tag]
+	tags = ['ip', 'probe']
+	opts = {}
+	opt_key_map = {
+		# HEADER: 'header',
+		# DELAY: 'delay',
+		# DEPTH: OPT_NOT_SUPPORTED,
+		# FILTER_CODES: 'filter-code',
+		# FILTER_REGEX: 'filter-regex',
+		# FILTER_SIZE: 'filter-length',
+		# FILTER_WORDS: 'filter-word-count',
+		# FOLLOW_REDIRECT: 'follow-redirects',
+		# MATCH_CODES: 'match-code',
+		# MATCH_REGEX: 'match-regex',
+		# MATCH_SIZE: 'match-length',
+		# MATCH_WORDS: 'match-word-count',
+		# METHOD: 'x',
+		# PROXY: 'proxy',
+		# RATE_LIMIT: 'rate-limit',
+		# RETRIES: 'retries',
+		# THREADS: 'threads',
+		# TIMEOUT: 'timeout',
+		# USER_AGENT: OPT_NOT_SUPPORTED,
+		# 'store_responses': 'sr',
+	}
+	opt_value_map = {
+		DELAY: lambda x: str(x) + 's' if x else None,
+	}
+	install_version = 'latest'
+	install_cmd = 'go install github.com/Vulnpire/getasn@[install_version]'
+	install_github_bin = False
+	github_handle = 'Vulnpire/getasn'
+	proxychains = False
+	proxy_socks5 = True
+	proxy_http = False
+
+	@staticmethod
+	def item_loader(self, line):
+		tag = Tag(
+			category='info',
+			name='asn',
+			match=self.inputs[0],
+			value=line.strip(),
+		)
+		if tag not in self.self_results:
+			yield tag

secator/tasks/gf.py

@@ -0,0 +1,36 @@
+from secator.decorators import task
+from secator.definitions import OPT_PIPE_INPUT, OPT_NOT_SUPPORTED
+from secator.output_types import Tag
+from secator.tasks._categories import Tagger
+
+
+@task()
+class gf(Tagger):
+	"""Wrapper around grep, to help you grep for things."""
+	cmd = 'gf'
+	input_types = None  # anything
+	output_types = [Tag]
+	tags = ['pattern', 'scan']
+	file_flag = OPT_PIPE_INPUT
+	input_flag = OPT_PIPE_INPUT
+	version_flag = OPT_NOT_SUPPORTED
+	opts = {
+		'pattern': {'type': str, 'help': 'Pattern names to match against (comma-delimited)', 'required': True}
+	}
+	opt_key_map = {
+		'pattern': ''
+	}
+	install_cmd = (
+		'go install -v github.com/tomnomnom/gf@latest && '
+		'git clone https://github.com/1ndianl33t/Gf-Patterns $HOME/.gf || true'
+	)
+
+	@staticmethod
+	def item_loader(self, line):
+		yield {'match': line, 'name': self.get_opt_value('pattern').rstrip(), 'category': 'url_pattern', 'value': line}  # noqa: E731,E501
+
+	@staticmethod
+	def on_item(self, item):
+		if isinstance(item, Tag):
+			item.extra_data = {'source': 'url'}
+		return item

secator/tasks/gitleaks.py

@@ -0,0 +1,96 @@
+import click
+import os
+import shlex
+import yaml
+
+from pathlib import Path
+
+from secator.config import CONFIG
+from secator.decorators import task
+from secator.runners import Command
+from secator.definitions import (OUTPUT_PATH, PATH)
+from secator.utils import caml_to_snake
+from secator.output_types import Tag, Info, Error
+from secator.rich import console
+
+GITLEAKS_MODES = ['git', 'dir']
+
+
+def convert_mode(mode):
+	return 'dir' if mode == 'filesystem' else 'git' if mode == 'git' else mode
+
+
+@task()
+class gitleaks(Command):
+	"""Tool for detecting secrets like passwords, API keys, and tokens in git repos, files, and stdin."""
+	cmd = 'gitleaks'
+	tags = ['secret', 'scan']
+	input_types = [PATH]
+	input_flag = None
+	json_flag = '-f json'
+	opt_prefix = '--'
+	opts = {
+		'ignore_path': {'type': str, 'help': 'Path to .gitleaksignore file or folder containing one'},
+		'mode': {'type': click.Choice(GITLEAKS_MODES), 'help': f'Scan mode ({", ".join(GITLEAKS_MODES)})', 'internal': True},  # noqa: E501
+		'config': {'type': str, 'short': 'config', 'help': 'Config file path'}
+	}
+	opt_key_map = {
+		"ignore_path": "gitleaks-ignore-path"
+	}
+	opt_value_map = {
+		'mode': lambda x: convert_mode(x)
+	}
+	input_type = "folder"
+	output_types = [Tag]
+	install_version = 'v8.29.1'
+	install_cmd_pre = {'*': ['git', 'make']}
+	install_cmd = (
+		f'git clone --single-branch -b [install_version] https://github.com/gitleaks/gitleaks.git {CONFIG.dirs.share}/gitleaks_[install_version] || true &&'  # noqa: E501
+		f'cd {CONFIG.dirs.share}/gitleaks_[install_version] && make build &&'
+		f'mv {CONFIG.dirs.share}/gitleaks_[install_version]/gitleaks {CONFIG.dirs.bin}'
+	)
+	github_handle = 'gitleaks/gitleaks'
+
+	@staticmethod
+	def on_cmd(self):
+		mode = self.cmd_options.get('mode', {}).get('value')
+		if mode and mode not in GITLEAKS_MODES:
+			raise Exception(f'Invalid mode: {mode}')
+		if not mode and len(self.inputs) > 0:
+			git_path = Path(self.inputs[0]).joinpath('.git')
+			if git_path.exists():
+				mode = 'git'
+			else:
+				mode = 'dir'
+			console.print(Info(message=f'Auto mode detected: {mode} for input: {self.inputs[0]}'))
+		self.cmd = self.cmd.replace(f'{gitleaks.cmd} ', f'{gitleaks.cmd} {mode} ')
+
+		# add output path
+		output_path = self.get_opt_value(OUTPUT_PATH)
+		if not output_path:
+			output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.json'
+		self.output_path = output_path
+		self.cmd += f' -r {shlex.quote(self.output_path)}'
+		self.cmd += ' --exit-code 0'
+
+	@staticmethod
+	def on_cmd_done(self):
+		if not os.path.exists(self.output_path):
+			yield Error(message=f'Could not find JSON results in {self.output_path}')
+			return
+
+		yield Info(message=f'JSON results saved to {self.output_path}')
+		with open(self.output_path, 'r') as f:
+			results = yaml.safe_load(f.read())
+		for result in results:
+			extra_data = {
+				caml_to_snake(k): v for k, v in result.items()
+				if k not in ['RuleID', 'File', 'Secret']
+			}
+			yield Tag(
+				category='secret',
+				name=result['RuleID'].replace('-', '_'),
+				value=result.get('Secret', ''),
+				match='{File}:{StartLine}:{StartColumn}'.format(**result),
+				extra_data=extra_data
+			)

secator/tasks/gospider.py

@@ -0,0 +1,84 @@
+from furl import furl
+
+from secator.decorators import task
+from secator.definitions import (CONTENT_LENGTH, DELAY, DEPTH, FILTER_CODES,
+							   FILTER_REGEX, FILTER_SIZE, FILTER_WORDS,
+							   FOLLOW_REDIRECT, HEADER, MATCH_CODES,
+							   MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, METHOD,
+							   OPT_NOT_SUPPORTED, PROXY, RATE_LIMIT, RETRIES,
+							   STATUS_CODE, THREADS, TIMEOUT, URL, USER_AGENT)
+from secator.output_types import Url
+from secator.serializers import JSONSerializer
+from secator.tasks._categories import HttpCrawler
+
+
+@task()
+class gospider(HttpCrawler):
+	"""Fast web spider written in Go."""
+	cmd = 'gospider'
+	input_types = [URL]
+	output_types = [Url]
+	tags = ['url', 'crawl']
+	file_flag = '-S'
+	input_flag = '-s'
+	json_flag = '--json'
+	opt_prefix = '--'
+	opt_key_map = {
+		HEADER: 'header',
+		DELAY: 'delay',
+		DEPTH: 'depth',
+		FILTER_CODES: OPT_NOT_SUPPORTED,
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: 'no-redirect',
+		MATCH_CODES: OPT_NOT_SUPPORTED,
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: OPT_NOT_SUPPORTED,
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: 'user-agent',
+	}
+	opt_value_map = {
+		FOLLOW_REDIRECT: lambda x: not x,
+		DELAY: lambda x: round(x) if isinstance(x, float) else x
+	}
+	item_loaders = [JSONSerializer()]
+	output_map = {
+		Url: {
+			URL: 'output',
+			STATUS_CODE: 'status',
+			CONTENT_LENGTH: 'length',
+		}
+	}
+	install_version = 'v1.1.6'
+	install_cmd = 'go install -v github.com/jaeles-project/gospider@[install_version]'
+	github_handle = 'jaeles-project/gospider'
+	proxychains = False
+	proxy_socks5 = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61
+	proxy_http = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61
+	profile = 'io'
+
+	@staticmethod
+	def validate_item(self, item):
+		"""Keep only items that match the same host."""
+		if not isinstance(item, dict):
+			return False
+		try:
+			netloc_in = furl(item['input']).netloc
+			netloc_out = furl(item['output']).netloc
+			if netloc_in != netloc_out:
+				return False
+		except ValueError:  # gospider returns invalid URLs for output sometimes
+			return False
+		return True
+
+	@staticmethod
+	def on_json_loaded(self, item):
+		item['request_headers'] = self.get_opt_value('header', preprocess=True)
+		yield item