secator 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

secator/celery.py

@@ -1,11 +1,10 @@
 import gc
 import logging
 import traceback
-from time import sleep
 
 from celery import Celery, chain, chord, signals
 from celery.app import trace
-from celery.result import AsyncResult, allow_join_result
+from celery.result import allow_join_result
 # from pyinstrument import Profiler  # TODO: make pyinstrument optional
 from rich.logging import RichHandler
 
@@ -354,56 +353,9 @@ def forward_results(results):
 	results = deduplicate(results, attr='_uuid')
 	return results
 
-
-#---------------------#
-# Celery result utils #
-#---------------------#
-
-
-def poll_task(result, seen=[]):
-	"""Poll Celery result tree recursively to get results live.
-
-	TODO: function is incomplete, as it does not parse all results.
-
-	Args:
-		result (Union[AsyncResult, GroupResult]): Celery result object.
-		seen (list): List of seen results (do not yield again).
-
-	Yields:
-		dict: Result.
-	"""
-	if result is None:
-		return
-
-	if result.children:
-		for child in result.children:
-			yield from poll_task(child, seen=seen)
-	else:
-		res = AsyncResult(result.id)
-		if not res.info:
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
-
-		# Task done running
-		if isinstance(res.info, list):
-			for item in res.info:
-				if item._uuid not in seen:
-					yield res.id, None, item
-					seen.append(item._uuid)
-			return
-
-		# Get task partial results, remove duplicates
-		results = res.info['results']
-		name = res.info['name']
-		for item in results:
-			if item._uuid not in seen:
-				yield res.id, name, item
-				seen.append(item._uuid)
-
-		# Task still running, keep polling
-		if not res.ready():
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
+#--------------#
+# Celery utils #
+#--------------#
 
 
 def is_celery_worker_alive():

secator/config.py

@@ -74,6 +74,7 @@ class Runners(StrictModel):
 	progress_update_frequency: int = 60
 	skip_cve_search: bool = False
 	skip_cve_low_confidence: bool = True
+	remove_duplicates: bool = False
 
 
 class HTTP(StrictModel):

secator/output_types/url.py

@@ -23,6 +23,7 @@ class Url(OutputType):
 	lines: int = field(default=0, compare=False)
 	screenshot_path: str = field(default='', compare=False)
 	stored_response_path: str = field(default='', compare=False)
+	headers: dict = field(default_factory=dict, repr=True, compare=False)
 	_source: str = field(default='', repr=True, compare=False)
 	_type: str = field(default='url', repr=True)
 	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
@@ -55,6 +56,8 @@ class Url(OutputType):
 
 	def __repr__(self):
 		s = f'🔗 [white]{self.url}'
+		if self.method and self.method != 'GET':
+			s += f' \[[turquoise4]{self.method}[/]]'
 		if self.status_code and self.status_code != 0:
 			if self.status_code < 400:
 				s += f' \[[green]{self.status_code}[/]]'

secator/report.py

@@ -1,5 +1,6 @@
 import operator
 
+from secator.config import CONFIG
 from secator.output_types import OUTPUT_TYPES, OutputType
 from secator.utils import merge_opts, get_file_timestamp
 from secator.rich import console
@@ -64,8 +65,10 @@ class Report:
 			sort_by, _ = get_table_fields(output_type)
 			items = [
 				item for item in self.runner.results
-				if isinstance(item, OutputType) and item._type == output_name and not item._duplicate
+				if isinstance(item, OutputType) and item._type == output_name
 			]
+			if CONFIG.runners.remove_duplicates:
+				items = [item for item in items if not item._duplicate]
 			if items:
 				if sort_by and all(sort_by):
 					items = sorted(items, key=operator.attrgetter(*sort_by))

secator/tasks/httpx.py

@@ -31,6 +31,7 @@ class httpx(Http):
 		'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'},
 		'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'},
 		'vhost': {'is_flag': True, 'default': False, 'help': 'Probe and display server supporting VHOST'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Save HTTP responses'},  # noqa: E501
 		'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'},
 		'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'},
 		'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'},
@@ -55,6 +56,7 @@ class httpx(Http):
 		THREADS: 'threads',
 		TIMEOUT: 'timeout',
 		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr',
 	}
 	opt_value_map = {
 		DELAY: lambda x: str(x) + 's' if x else None,
@@ -71,15 +73,10 @@ class httpx(Http):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.output_response_path = f'{self.reports_folder}/response'
-			self.output_screenshot_path = f'{self.reports_folder}/screenshot'
-			os.makedirs(self.output_response_path, exist_ok=True)
-			os.makedirs(self.output_screenshot_path, exist_ok=True)
-			self.cmd += f' -sr -srd {self.reports_folder}'
-
-		# Remove screenshot bytes and body bytes when screenshot
 		screenshot = self.get_opt_value('screenshot')
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses or screenshot:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 		if screenshot:
 			self.cmd += ' -esb -ehb'
 
@@ -98,8 +95,15 @@ class httpx(Http):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses:
-			if os.path.exists(self.output_response_path + '/index.txt'):
-				os.remove(self.output_response_path + '/index.txt')
-			if os.path.exists(self.output_screenshot_path + '/index.txt'):
-				os.remove(self.output_screenshot_path + '/index_screenshot.txt')
+		store_responses = self.get_opt_value('store_responses')
+		response_dir = f'{self.reports_folder}/.outputs'
+		if store_responses:
+			index_rpath = f'{response_dir}/response/index.txt'
+			index_spath = f'{response_dir}/screenshot/index_screenshot.txt'
+			index_spath2 = f'{response_dir}/screenshot/screenshot.html'
+			if os.path.exists(index_rpath):
+				os.remove(index_rpath)
+			if os.path.exists(index_spath):
+				os.remove(index_spath)
+			if os.path.exists(index_spath2):
+				os.remove(index_spath2)

secator/tasks/katana.py

@@ -29,7 +29,8 @@ class katana(HttpCrawler):
 	opts = {
 		'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'},
 		'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'},
-		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'}
+		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'}
 	}
 	opt_key_map = {
 		HEADER: 'headers',
@@ -50,7 +51,8 @@ class katana(HttpCrawler):
 		RETRIES: 'retry',
 		THREADS: 'concurrency',
 		TIMEOUT: 'timeout',
-		USER_AGENT: OPT_NOT_SUPPORTED
+		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr'
 	}
 	opt_value_map = {
 		DELAY: lambda x: int(x) if isinstance(x, float) else x
@@ -107,14 +109,16 @@ class katana(HttpCrawler):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.cmd += f' -sr -srd {self.reports_folder}'
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 
 	@staticmethod
 	def on_item(self, item):
 		if not isinstance(item, Url):
 			return item
-		if CONFIG.http.store_responses and os.path.exists(item.stored_response_path):
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses and os.path.exists(item.stored_response_path):
 			with open(item.stored_response_path, 'r', encoding='latin-1') as fin:
 				data = fin.read().splitlines(True)
 				first_line = data[0]
@@ -126,5 +130,7 @@ class katana(HttpCrawler):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses and os.path.exists(self.reports_folder + '/index.txt'):
-			os.remove(self.reports_folder + '/index.txt')
+		store_responses = self.get_opt_value('store_responses')
+		index_rpath = f'{self.reports_folder}/.outputs/index.txt'
+		if store_responses and os.path.exists(index_rpath):
+			os.remove(index_rpath)

secator/utils.py

@@ -343,8 +343,10 @@ def print_results_table(results, title=None, exclude_fields=[], log=False):
 		if output_type.__name__ == 'Progress':
 			continue
 		items = [
-			item for item in results if item._type == output_type.get_name() and not item._duplicate
+			item for item in results if item._type == output_type.get_name()
 		]
+		if CONFIG.runners.remove_duplicates:
+			items = [item for item in items if not item._duplicate]
 		if items:
 			_table = build_table(
 				items,

{secator-0.5.0.dist-info → secator-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: secator
-Version: 0.5.0
+Version: 0.5.2
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.5.0.dist-info → secator-0.5.2.dist-info}/RECORD

@@ -1,15 +1,15 @@
 secator/.gitignore,sha256=da8MUc3hdb6Mo0WjZu2upn5uZMbXcBGvhdhTQ1L89HI,3093
 secator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/celery.py,sha256=5Raua1rDFJACdmP4b1HLS15kx3ObcUrVConQ0UcopTc,12135
+secator/celery.py,sha256=8BPgxnxBWe018tZL6-0WaZQdzHV8rdKS_cxYGuPStiw,10999
 secator/cli.py,sha256=f959ENOt4C107hcnG_z4bEVgnNS4y6VqkyybtNk356c,35996
-secator/config.py,sha256=Mzubha9pSzmpJnwA6f87OUScKrYo-szf8i9PJvKkz8w,17501
+secator/config.py,sha256=Yzb7429RweDCB9Qz88710FKnHiFvg9o0tU2Gu_JFzyE,17534
 secator/decorators.py,sha256=SIS_32SbeN9iTx82mvy9F9vLpjofRYspOOLCXytIO2g,10764
 secator/definitions.py,sha256=Spr62Nc0TweBAa84iRGSwNkIvXlKofPxtIi795gqTjc,3047
 secator/installer.py,sha256=fpjf5fbA5M5zDQVP4Fdr51sLoMwtoGZTe3mXh7DvD6s,9466
-secator/report.py,sha256=73FlUZ6lDyJjQ_lBwFcPRARWta_s6rlBJRx1dZ_MJ4g,2505
+secator/report.py,sha256=RCNowMcGDalB51hCoB0aO1adTmDrcnVbKZnpZPjfCCo,2615
 secator/rich.py,sha256=W4PipeZfIVnERfW3ySeWSvnZ90jhCFiABBoERYy_6kM,3177
 secator/template.py,sha256=MRCzvn8FJ7D4n8V4ceBwAjPsdLhTWjDRu5VLefmLb6M,3705
-secator/utils.py,sha256=CeZ0GnjFSKEMnKwItoh92qVu37HwJYGEMri4qNQF_8g,11607
+secator/utils.py,sha256=cN1xi1hB_Wmi30TUgx_HztsC5hosS327cf1szGAzkgU,11681
 secator/utils_test.py,sha256=95KNjEePk39SrDPL9XXV_L1GAdkf_LwBKU02Vuj-tZA,5387
 secator/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 secator/configs/profiles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -54,7 +54,7 @@ secator/output_types/record.py,sha256=WnI0yvwzrO2Wt7OWciHMOuIRRLbuSOAJczdNshV7tY
 secator/output_types/subdomain.py,sha256=lmCoK7_8I4FXWgl9kToRvDn3gr3E3uBTaQzFAOHbswE,1343
 secator/output_types/tag.py,sha256=8AlT0VigsYP04GN8sPCTM07IlL5uMUmFgsNa9IDCoyY,1431
 secator/output_types/target.py,sha256=gJWzzqhal34Cnl9oAKf0m1MSaGxRtUGdA2XbkhD_yd0,848
-secator/output_types/url.py,sha256=yDozBXCuPfuybH1iX_xGmbCJPXO6Ei14C8Hp5CnzNbE,2535
+secator/output_types/url.py,sha256=G_28pWwwLe52rWQGz58GH5vTYYN9sTpv-1AgqmCHECw,2693
 secator/output_types/user_account.py,sha256=EiT2BFl2LTCdqHF1meoMEKVhjKGroyf8-JoWHPuBOTc,1378
 secator/output_types/vulnerability.py,sha256=cWaz6zzClnrKQmdXr4gOcudcWqnvlp-6SxOLPpO29Z8,2712
 secator/runners/__init__.py,sha256=EBbOk37vkBy9p8Hhrbi-2VtM_rTwQ3b-0ggTyiD22cE,290
@@ -83,8 +83,8 @@ secator/tasks/gf.py,sha256=WlhoEyL6xE79w6nE5XNSXHs-jVeO10njqJxBF8w20sA,945
 secator/tasks/gospider.py,sha256=_UlTb9G5Ss8D68NT53s0_rI6TnG00Ph0yxWyHic7cKs,2172
 secator/tasks/grype.py,sha256=n60Zs9d1NWJFHQ0DwIZib5wu3xH-tV2RzgLYwuQSTo4,2413
 secator/tasks/h8mail.py,sha256=hZBpfV6M1mbpD_PbDHxLI5HMvqAvTeY_W0lbkq3Hugo,2037
-secator/tasks/httpx.py,sha256=ugk4AOIqhvT5-HVhlRsQw_oF2BGovgHlS-_D9wav8wo,3972
-secator/tasks/katana.py,sha256=3JHtvxoSxJtju9xqvQptVaYXKnLLlCkOn6oY0A8h7wM,4371
+secator/tasks/httpx.py,sha256=O4jkee6So9MSyHTb7n_owFiL3OMO0697uxDq7ULbq5s,4108
+secator/tasks/katana.py,sha256=edsjIVj8sh5_ubCB5AFjANwute0Uv4Vdry0XB10xE1Q,4677
 secator/tasks/maigret.py,sha256=PZDTICJ4LZF3joKe-dXu2alffakD_1sxBuNEUBtJDm4,2098
 secator/tasks/mapcidr.py,sha256=7aa2WXQATWgIQo5oA12URjAg80L6MFMGdxScxls8DuA,980
 secator/tasks/msfconsole.py,sha256=Cm0vzOFff17C4M1YjkgU6T7Jc5-ClBK0Qi_529qVRb0,6065
@@ -94,8 +94,8 @@ secator/tasks/nuclei.py,sha256=lKZYPVcnCYomd830-ZCOz4fyc8xAKjNDuKayyz0BPek,3507
 secator/tasks/searchsploit.py,sha256=tIqCwYFIyHIgJbtcTL56PXqd-MCvoXOpvSDgoK_dxzc,2953
 secator/tasks/subfinder.py,sha256=cpFyFCpVaDZ3QAjNId26ezOwntn3CA5Uk-AC2l0mo0E,1087
 secator/tasks/wpscan.py,sha256=UVWnBPOQ1RDB2wzMswWR6vc6cucYgHtuJ8pLZoqCM40,5434
-secator-0.5.0.dist-info/METADATA,sha256=zxLyE2AfVBFlf752cRu5f9WrEfKK9owH1uWwZHFvcJ4,14095
-secator-0.5.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-secator-0.5.0.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
-secator-0.5.0.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
-secator-0.5.0.dist-info/RECORD,,
+secator-0.5.2.dist-info/METADATA,sha256=ZMQMB3C0AM0JpSqfAwOHOMYvUJPytL6pJvOg60ozofg,14095
+secator-0.5.2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+secator-0.5.2.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
+secator-0.5.2.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
+secator-0.5.2.dist-info/RECORD,,