secator 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

secator/tasks/_categories.py

@@ -1,19 +1,22 @@
 import json
 import os
+import re
+
+from functools import cache
 
 import requests
 from bs4 import BeautifulSoup
 from cpe import CPE
 
 from secator.definitions import (CIDR_RANGE, CVSS_SCORE, DELAY, DEPTH, DESCRIPTION, FILTER_CODES,
-								 FILTER_REGEX, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID,
+								 FILTER_REGEX, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID, IP,
 								 MATCH_CODES, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, METHOD, NAME, PATH, PROVIDER, PROXY,
 								 RATE_LIMIT, REFERENCES, RETRIES, SEVERITY, TAGS, THREADS, TIMEOUT, URL, USER_AGENT,
 								 USERNAME, WORDLIST)
 from secator.output_types import Ip, Port, Subdomain, Tag, Url, UserAccount, Vulnerability
 from secator.config import CONFIG
 from secator.runners import Command
-from secator.utils import debug
+from secator.utils import debug, process_wordlist
 
 
 OPTS = {
@@ -36,7 +39,7 @@ OPTS = {
 	THREADS: {'type': int, 'help': 'Number of threads to run', 'default': 50},
 	TIMEOUT: {'type': int, 'help': 'Request timeout'},
 	USER_AGENT: {'type': str, 'short': 'ua', 'help': 'User agent, e.g "Mozilla Firefox 1.0"'},
-	WORDLIST: {'type': str, 'short': 'w', 'default': CONFIG.wordlists.defaults.http, 'help': 'Wordlist to use'}
+	WORDLIST: {'type': str, 'short': 'w', 'default': 'http', 'process': process_wordlist, 'help': 'Wordlist to use'}
 }
 
 OPTS_HTTP = [
@@ -106,7 +109,7 @@ class ReconIp(Recon):
 
 
 class ReconPort(Recon):
-	input_type = HOST
+	input_type = IP
 	output_types = [Port]
 
 
@@ -124,6 +127,7 @@ class Vuln(Command):
 		if os.path.exists(cve_path):
 			with open(cve_path, 'r') as f:
 				return json.load(f)
+		debug(f'CVE {cve_id} not found in cache', sub='cve')
 		return None
 
 	# @staticmethod
@@ -179,12 +183,98 @@ class Vuln(Command):
 		return tup1 == tup2
 
 	@staticmethod
-	def lookup_cve(cve_id, cpes=[]):
-		"""Search for a CVE in local db or using cve.circl.lu and return vulnerability data.
+	def get_cpe_fs(cpe):
+		""""Return formatted string for given CPE.
+
+		Args:
+			cpe (string): Input CPE
+
+		Returns:
+			string: CPE formatted string.
+		"""
+		try:
+			return CPE(cpe).as_fs()
+		except NotImplementedError:
+			return None
+
+	@cache
+	@staticmethod
+	def lookup_cve_from_vulners_exploit(exploit_id, *cpes):
+		"""Search for a CVE corresponding to an exploit by extracting the CVE id from the exploit HTML page.
+
+		Args:
+			exploit_id (str): Exploit ID.
+			cpes (tuple[str], Optional): CPEs to match for.
+
+		Returns:
+			dict: vulnerability data.
+		"""
+		if CONFIG.runners.skip_exploit_search:
+			debug(f'Skip remote query for {exploit_id} since config.runners.skip_exploit_search is set.', sub='cve')
+			return None
+		if CONFIG.offline_mode:
+			debug(f'Skip remote query for {exploit_id} since config.offline_mode is set.', sub='cve')
+			return None
+		try:
+			resp = requests.get(f'https://vulners.com/githubexploit/{exploit_id}', timeout=5)
+			resp.raise_for_status()
+			soup = BeautifulSoup(resp.text, 'lxml')
+			title = soup.title.get_text(strip=True)
+			h1 = [h1.get_text(strip=True) for h1 in soup.find_all('h1')]
+			if '404' in h1:
+				raise requests.RequestException("404 [not found or rate limited]")
+			code = [code.get_text(strip=True) for code in soup.find_all('code')]
+			elems = [title] + h1 + code
+			content = '\n'.join(elems)
+			cve_regex = re.compile(r'(CVE(?:-|_)\d{4}(?:-|_)\d{4,7})', re.IGNORECASE)
+			matches = cve_regex.findall(str(content))
+			if not matches:
+				debug(f'{exploit_id}: No CVE found in https://vulners.com/githubexploit/{exploit_id}.', sub='cve')
+				return None
+			cve_id = matches[0].replace('_', '-').upper()
+			cve_data = Vuln.lookup_cve(cve_id, *cpes)
+			if cve_data:
+				return cve_data
+
+		except requests.RequestException as e:
+			debug(f'Failed remote query for {exploit_id} ({str(e)}).', sub='cve')
+			return None
+
+	@cache
+	@staticmethod
+	def lookup_cve_from_cve_circle(cve_id):
+		"""Get CVE data from vulnerability.circl.lu.
+
+		Args:
+			cve_id (str): CVE id.
+
+		Returns:
+			dict | None: CVE data, None if no response or empty response.
+		"""
+		try:
+			resp = requests.get(f'https://vulnerability.circl.lu/api/cve/{cve_id}', timeout=5)
+			resp.raise_for_status()
+			cve_info = resp.json()
+			if not cve_info:
+				debug(f'Empty response from https://vulnerability.circl.lu/api/cve/{cve_id}', sub='cve')
+				return None
+			cve_path = f'{CONFIG.dirs.data}/cves/{cve_id}.json'
+			with open(cve_path, 'w') as f:
+				f.write(json.dumps(cve_info, indent=2))
+			debug(f'Downloaded {cve_id} to {cve_path}', sub='cve')
+			return cve_info
+		except requests.RequestException as e:
+			debug(f'Failed remote query for {cve_id} ({str(e)}).', sub='cve')
+			return None
+
+	@cache
+	@staticmethod
+	def lookup_cve(cve_id, *cpes):
+		"""Search for a CVE info and return vulnerability data.
 
 		Args:
 			cve_id (str): CVE ID in the form CVE-*
-			cpes (str, Optional): CPEs to match for.
+			cpes (tuple[str], Optional): CPEs to match for.
 
 		Returns:
 			dict: vulnerability data.
@@ -199,73 +289,87 @@ class Vuln(Command):
 			if CONFIG.offline_mode:
 				debug(f'Skip remote query for {cve_id} since config.offline_mode is set.', sub='cve')
 				return None
-			try:
-				resp = requests.get(f'https://cve.circl.lu/api/cve/{cve_id}', timeout=5)
-				resp.raise_for_status()
-				cve_info = resp.json()
-			except requests.RequestException as e:
-				debug(f'Failed remote query for {cve_id} ({str(e)}).', sub='cve')
+			cve_info = Vuln.lookup_cve_from_cve_circle(cve_id)
+			if not cve_info:
 				return None
 
+		# Convert cve info to easy format
+		cve_id = cve_info['cveMetadata']['cveId']
+		cna = cve_info['containers']['cna']
+		metrics = cna.get('metrics', [])
+		cvss_score = 0
+		for metric in metrics:
+			for name, value in metric.items():
+				if 'cvss' in name:
+					cvss_score = metric[name]['baseScore']
+		description = cna.get('descriptions', [{}])[0].get('value')
+		cwe_id = cna.get('problemTypes', [{}])[0].get('descriptions', [{}])[0].get('cweId')
+		cpes_affected = []
+		for product in cna['affected']:
+			cpes_affected.extend(product.get('cpes', []))
+		references = [u['url'] for u in cna['references']]
+		cve_info = {
+			'id': cve_id,
+			'cwe_id': cwe_id,
+			'cvss_score': cvss_score,
+			'description': description,
+			'cpes': cpes_affected,
+			'references': references
+		}
+
 		# Match the CPE string against the affected products CPE FS strings from the CVE data if a CPE was passed.
 		# This allow to limit the number of False positives (high) that we get from nmap NSE vuln scripts like vulscan
 		# and ensure we keep only right matches.
 		# The check is not executed if no CPE was passed (sometimes nmap cannot properly detect a CPE) or if the CPE
 		# version cannot be determined.
 		cpe_match = False
-		tags = []
+		tags = [cve_id]
 		if cpes:
 			for cpe in cpes:
-				cpe_obj = CPE(cpe)
-				cpe_fs = cpe_obj.as_fs()
+				cpe_fs = Vuln.get_cpe_fs(cpe)
+				if not cpe_fs:
+					debug(f'{cve_id}: Failed to parse CPE {cpe} with CPE parser', sub='cve.match', verbose=True)
+					tags.append('cpe-invalid')
+					continue
 				# cpe_version = cpe_obj.get_version()[0]
-				vulnerable_fs = cve_info['vulnerable_product']
-				for fs in vulnerable_fs:
-					# debug(f'{cve_id}: Testing {cpe_fs} against {fs}', sub='cve')  # for hardcore debugging
-					if Vuln.match_cpes(cpe_fs, fs):
+				for cpe_affected in cpes_affected:
+					cpe_affected_fs = Vuln.get_cpe_fs(cpe_affected)
+					if not cpe_affected_fs:
+						debug(f'{cve_id}: Failed to parse CPE {cpe} (from online data) with CPE parser', sub='cve.match', verbose=True)
+						continue
+					debug(f'{cve_id}: Testing {cpe_fs} against {cpe_affected_fs}', sub='cve.match', verbose=True)
+					cpe_match = Vuln.match_cpes(cpe_fs, cpe_affected_fs)
+					if cpe_match:
 						debug(f'{cve_id}: CPE match found for {cpe}.', sub='cve')
-						cpe_match = True
 						tags.append('cpe-match')
 						break
+
 				if not cpe_match:
 					debug(f'{cve_id}: no CPE match found for {cpe}.', sub='cve')
 
 		# Parse CVE id and CVSS
 		name = id = cve_info['id']
-		cvss = cve_info.get('cvss') or 0
 		# exploit_ids = cve_info.get('refmap', {}).get('exploit-db', [])
 		# osvdb_ids = cve_info.get('refmap', {}).get('osvdb', [])
 
 		# Get description
-		description = cve_info.get('summary')
+		description = cve_info['description']
 		if description is not None:
 			description = description.replace(id, '').strip()
 
 		# Get references
 		references = cve_info.get(REFERENCES, [])
-		cve_ref_url = f'https://cve.circl.lu/cve/{id}'
+		cve_ref_url = f'https://vulnerability.circl.lu/cve/{id}'
 		references.append(cve_ref_url)
 
 		# Get CWE ID
-		vuln_cwe_id = cve_info.get('cwe')
-		if vuln_cwe_id is None:
-			tags.append(vuln_cwe_id)
-
-		# Parse capecs for a better vuln name / type
-		capecs = cve_info.get('capec', [])
-		if capecs and len(capecs) > 0:
-			name = capecs[0]['name']
-
-		# Parse ovals for a better vuln name / type
-		ovals = cve_info.get('oval', [])
-		if ovals:
-			if description == 'none':
-				description = ovals[0]['title']
-			family = ovals[0]['family']
-			tags.append(family)
+		cwe_id = cve_info['cwe_id']
+		if cwe_id is not None:
+			tags.append(cwe_id)
 
 		# Set vulnerability severity based on CVSS score
 		severity = None
+		cvss = cve_info['cvss_score']
 		if cvss:
 			severity = Vuln.cvss_to_severity(cvss)
 
@@ -273,15 +377,16 @@ class Vuln(Command):
 		vuln = {
 			ID: id,
 			NAME: name,
-			PROVIDER: 'cve.circl.lu',
+			PROVIDER: 'vulnerability.circl.lu',
 			SEVERITY: severity,
 			CVSS_SCORE: cvss,
 			TAGS: tags,
-			REFERENCES: [f'https://cve.circl.lu/cve/{id}'] + references,
+			REFERENCES: [f'https://vulnerability.circl.lu/cve/{id}'] + references,
 			DESCRIPTION: description,
 		}
 		return vuln
 
+	@cache
 	@staticmethod
 	def lookup_ghsa(ghsa_id):
 		"""Search for a GHSA on Github and and return associated CVE vulnerability data.

secator/tasks/bbot.py

@@ -0,0 +1,295 @@
+import shutil
+
+from secator.decorators import task
+from secator.runners import Command
+from secator.serializers import RegexSerializer
+from secator.output_types import Vulnerability, Port, Url, Record, Ip, Tag, Error
+from secator.serializers import JSONSerializer
+
+
+BBOT_MODULES = [
+	"affiliates",
+	# "ajaxpro",
+	"anubisdb",
+	"asn",
+	"azure_realm",
+	"azure_tenant",
+	"badsecrets",
+	"bevigil",
+	"binaryedge",
+	# "bucket_aws",
+	"bucket_azure",
+	"bucket_digitalocean",
+	# "bucket_file_enum",
+	"bucket_firebase",
+	"bucket_google",
+	"builtwith",
+	"bypass403",
+	"c99",
+	"censys",
+	"certspotter",
+	# "chaos",
+	"columbus",
+	# "credshed",
+	# "crobat",
+	"crt",
+	# "dastardly",
+	# "dehashed",
+	"digitorus",
+	"dnscommonsrv",
+	"dnsdumpster",
+	# "dnszonetransfer",
+	"emailformat",
+	"ffuf",
+	"ffuf_shortnames",
+	# "filedownload",
+	"fingerprintx",
+	"fullhunt",
+	"generic_ssrf",
+	"git",
+	"telerik",
+	# "github_codesearch",
+	"github_org",
+	"gowitness",
+	"hackertarget",
+	"host_header",
+	"httpx",
+	"hunt",
+	"hunterio",
+	"iis_shortnames",
+	# "internetdb",
+	# "ip2location",
+	"ipneighbor",
+	"ipstack",
+	"leakix",
+	# "masscan",
+	# "massdns",
+	"myssl",
+	# "newsletters",
+	# "nmap",
+	# "nsec",
+	"ntlm",
+	"nuclei",
+	"oauth",
+	"otx",
+	"paramminer_cookies",
+	"paramminer_getparams",
+	"paramminer_headers",
+	"passivetotal",
+	"pgp",
+	# "postman",
+	"rapiddns",
+	# "riddler",
+	"robots",
+	"secretsdb",
+	"securitytrails",
+	"shodan_dns",
+	"sitedossier",
+	"skymem",
+	"smuggler",
+	"social",
+	"sslcert",
+	# "subdomain_hijack",
+	"subdomaincenter",
+	# "sublist3r",
+	"telerik",
+	# "threatminer",
+	"url_manipulation",
+	"urlscan",
+	"vhost",
+	"viewdns",
+	"virustotal",
+	# "wafw00f",
+	"wappalyzer",
+	"wayback",
+	"zoomeye"
+]
+BBOT_PRESETS = [
+	'cloud-enum',
+	'code-enum',
+	'dirbust-heavy',
+	'dirbust-light',
+	'dotnet-audit',
+	'email-enum',
+	'iis-shortnames',
+	'kitchen-sink',
+	'paramminer',
+	'spider',
+	'subdomain-enum',
+	'web-basic',
+	'web-screenshots',
+	'web-thorough'
+]
+BBOT_MODULES_STR = ' '.join(BBOT_MODULES)
+BBOT_MAP_TYPES = {
+	'IP_ADDRESS': Ip,
+	'PROTOCOL': Port,
+	'OPEN_TCP_PORT': Port,
+	'URL': Url,
+	'TECHNOLOGY': Tag,
+	'ASN': Record,
+	'DNS_NAME': Record,
+	'WEBSCREENSHOT': Url,
+	'VULNERABILITY': Vulnerability,
+	'FINDING': Tag
+}
+BBOT_DESCRIPTION_REGEX = RegexSerializer(
+	regex=r'(?P<name>[\w ]+): \[(?P<value>[^\[\]]+)\]',
+	findall=True
+)
+
+
+def output_discriminator(self, item):
+	_type = item.get('type')
+	_message = item.get('message')
+	if not _type and _message:
+		return Error
+	elif _type not in BBOT_MAP_TYPES:
+		return None
+	return BBOT_MAP_TYPES[_type]
+
+
+@task()
+class bbot(Command):
+	"""Multipurpose scanner."""
+	cmd = 'bbot -y --allow-deadly --force'
+	json_flag = '--json'
+	input_flag = '-t'
+	file_flag = None
+	version_flag = '--help'
+	opts = {
+		'modules': {'type': str, 'short': 'm', 'default': '', 'help': ','.join(BBOT_MODULES)},
+		'presets': {'type': str, 'short': 'ps', 'default': 'kitchen-sink', 'help': ','.join(BBOT_PRESETS), 'shlex': False},
+	}
+	opt_key_map = {
+		'modules': 'm',
+		'presets': 'p'
+	}
+	opt_value_map = {
+		'presets': lambda x: ' '.join(x.split(','))
+	}
+	item_loaders = [JSONSerializer()]
+	output_types = [Vulnerability, Port, Url, Record, Ip]
+	output_discriminator = output_discriminator
+	output_map = {
+		Ip: {
+			'ip': lambda x: x['data'],
+			'host': lambda x: x['data'],
+			'alive': lambda x: True,
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Tag: {
+			'name': 'name',
+			'match': lambda x: x['data'].get('url') or x['data'].get('host'),
+			'extra_data': 'extra_data',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Url: {
+			'url': lambda x: x['data'].get('url') if isinstance(x['data'], dict) else x['data'],
+			'host': lambda x: x['resolved_hosts'][0] if 'resolved_hosts' in x else '',
+			'status_code': lambda x: bbot.extract_status_code(x),
+			'title': lambda x: bbot.extract_title(x),
+			'screenshot_path': lambda x: x['data']['path'] if isinstance(x['data'], dict) else '',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Port: {
+			'port': lambda x: int(x['data']['port']) if 'port' in x['data'] else x['data'].split(':')[-1],
+			'ip': lambda x: [_ for _ in x['resolved_hosts'] if not _.startswith('::')][0],
+			'state': lambda x: 'OPEN',
+			'service_name': lambda x: x['data']['protocol'] if 'protocol' in x['data'] else '',
+			'cpes': lambda x: [],
+			'host': lambda x: x['data']['host'] if isinstance(x['data'], dict) else x['data'].split(':')[0],
+			'extra_data': 'extra_data',
+			'_source': lambda x: 'bbot-' + x['module']
+		},
+		Vulnerability: {
+			'name': 'name',
+			'match': lambda x: x['data'].get('url') or x['data']['host'],
+			'extra_data': 'extra_data',
+			'severity': lambda x: x['data']['severity'].lower()
+		},
+		Record: {
+			'name': 'name',
+			'type': 'type',
+			'extra_data': 'extra_data'
+		},
+		Error: {
+			'message': 'message'
+		}
+	}
+	install_cmd = 'pipx install bbot && pipx upgrade bbot'
+
+	@staticmethod
+	def on_json_loaded(self, item):
+		_type = item.get('type')
+
+		if not _type:
+			yield item
+			return
+
+		if _type not in BBOT_MAP_TYPES:
+			self._print(f'[bold orange3]Found unsupported bbot type: {_type}.[/] [bold green]Skipping.[/]')
+			return
+
+		if isinstance(item['data'], str):
+			item['name'] = item['data']
+			yield item
+			return
+
+		item['extra_data'] = item['data']
+
+		# Parse bbot description into extra_data
+		description = item['data'].get('description')
+		if description:
+			del item['data']['description']
+			match = BBOT_DESCRIPTION_REGEX.run(description)
+			for chunk in match:
+				key, val = tuple([c.strip() for c in chunk])
+				if ',' in val:
+					val = val.split(',')
+				key = '_'.join(key.split(' ')).lower()
+				item['extra_data'][key] = val
+
+		# Set technology as name for Tag
+		if item['type'] == 'TECHNOLOGY':
+			item['name'] = item['data']['technology']
+			del item['data']['technology']
+
+		# If 'name' key is present in 'data', set it as name
+		elif 'name' in item['data'].keys():
+			item['name'] = item['data']['name']
+			del item['data']['name']
+
+		# If 'name' key is present in 'extra_data', set it as name
+		elif 'extra_data' in item and 'name' in item['extra_data'].keys():
+			item['name'] = item['extra_data']['name']
+			del item['extra_data']['name']
+
+		# If 'discovery_context' and no name set yet, set it as name
+		else:
+			item['name'] = item['discovery_context']
+
+		# If a screenshot was saved, move it to secator output folder
+		if item['type'] == 'WEBSCREENSHOT':
+			path = item['data']['path']
+			name = path.split('/')[-1]
+			secator_path = f'{self.reports_folder}/.outputs/{name}'
+			shutil.copy(path, secator_path)
+			item['data']['path'] = secator_path
+
+		yield item
+
+	@staticmethod
+	def extract_title(item):
+		for tag in item['tags']:
+			if 'http-title' in tag:
+				title = ' '.join(tag.split('-')[2:])
+				return title
+		return ''
+
+	@staticmethod
+	def extract_status_code(item):
+		for tag in item['tags']:
+			if 'status-' in tag:
+				return int([tag.split('-')[-1]][0])
+		return 0

secator/tasks/bup.py

@@ -0,0 +1,99 @@
+import json
+import re
+
+from secator.decorators import task
+from secator.output_types import Url, Progress
+from secator.definitions import (
+	HEADER, DELAY, FOLLOW_REDIRECT, METHOD, PROXY, RATE_LIMIT, RETRIES, THREADS, TIMEOUT, USER_AGENT,
+	DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS,
+	MATCH_CODES, OPT_NOT_SUPPORTED, URL
+)
+from secator.serializers import JSONSerializer
+from secator.tasks._categories import Http
+
+
+@task()
+class bup(Http):
+	"""40X bypasser."""
+	cmd = 'bup'
+	input_flag = '-u'
+	input_type = URL
+	json_flag = '--jsonl'
+	opt_prefix = '--'
+	opts = {
+		'spoofport': {'type': int, 'short': 'sp', 'help': 'Port(s) to inject in port-specific headers'},
+		'spoofip': {'type': str, 'short': 'si', 'help': 'IP(s) to inject in ip-specific headers'},
+		'mode': {'type': str, 'help': 'Bypass modes.'},
+	}
+	opt_key_map = {
+		HEADER: 'header',
+		DELAY: OPT_NOT_SUPPORTED,
+		FOLLOW_REDIRECT: OPT_NOT_SUPPORTED,
+		METHOD: OPT_NOT_SUPPORTED,
+		RATE_LIMIT: OPT_NOT_SUPPORTED,
+		RETRIES: 'retry',
+		THREADS: 'threads',
+		TIMEOUT: 'timeout',
+		USER_AGENT: OPT_NOT_SUPPORTED,
+		DEPTH: OPT_NOT_SUPPORTED,
+		MATCH_REGEX: OPT_NOT_SUPPORTED,
+		MATCH_SIZE: OPT_NOT_SUPPORTED,
+		MATCH_WORDS: OPT_NOT_SUPPORTED,
+		FILTER_REGEX: OPT_NOT_SUPPORTED,
+		FILTER_CODES: OPT_NOT_SUPPORTED,
+		FILTER_SIZE: OPT_NOT_SUPPORTED,
+		FILTER_WORDS: OPT_NOT_SUPPORTED,
+		MATCH_CODES: OPT_NOT_SUPPORTED,
+		PROXY: 'proxy',
+	}
+	item_loaders = [JSONSerializer()]
+	output_types = [Url, Progress]
+	output_map = {
+		Url: {
+			'url': 'request_url',
+			'method': lambda x: bup.method_extractor(x),
+			'headers': lambda x: bup.headers_extractor(x),
+			'status_code': 'response_status_code',
+			'content_type': 'response_content_type',
+			'content_length': 'response_content_length',
+			'title': 'response_title',
+			'server': 'response_server_type',
+			'lines': 'response_lines_count',
+			'words': 'response_words_count',
+			'stored_response_path': 'response_html_filename',
+		}
+	}
+	install_cmd = 'pipx install bypass-url-parser && pipx upgrade bypass-url-parser'
+
+	@staticmethod
+	def on_init(self):
+		self.cmd += f' -o {self.reports_folder}/.outputs/response'
+
+	@staticmethod
+	def on_line(self, line):
+		if 'Doing' in line:
+			progress_indicator = line.split(':')[-1]
+			current, total = tuple([int(c.strip()) for c in progress_indicator.split('/')])
+			return json.dumps({"duration": "unknown", "percent": int((current / total) * 100)})
+		elif 'batcat' in line:  # ignore batcat lines as they're loaded as JSON
+			return None
+		return line
+
+	@staticmethod
+	def method_extractor(item):
+		payload = item['request_curl_payload']
+		match = re.match(r'-X\s+(\w+)', payload)
+		if match:
+			return match.group(1)
+		return 'GET'
+
+	@staticmethod
+	def headers_extractor(item):
+		headers_list = item['response_headers'].split('\n')[1:]
+		headers = {}
+		for header in headers_list:
+			split_headers = header.split(':')
+			key = split_headers[0]
+			value = ':'.join(split_headers[1:])
+			headers[key] = value
+		return headers