secator 0.5.0__tar.gz → 0.5.2__tar.gz

{secator-0.5.0 → secator-0.5.2}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [0.5.2](https://github.com/freelabz/secator/compare/v0.5.1...v0.5.2) (2024-05-07)
+
+
+### Bug Fixes
+
+* **nuclei,katana:** add -sr flag and write http responses and screenshot to correct folder ([#395](https://github.com/freelabz/secator/issues/395)) ([1a51790](https://github.com/freelabz/secator/commit/1a51790c9231f593631c2780b6d5e0fa89f1aa55))
+
+## [0.5.1](https://github.com/freelabz/secator/compare/v0.5.0...v0.5.1) (2024-05-06)
+
+
+### Bug Fixes
+
+* **output:** add headers to Url and print HTTP method when not GET ([#390](https://github.com/freelabz/secator/issues/390)) ([5a87d7b](https://github.com/freelabz/secator/commit/5a87d7b8bc1dd098999f3864952e98068fd32efc))
+* **report:** do not remove duplicate in reports by default ([#392](https://github.com/freelabz/secator/issues/392)) ([7d74ae8](https://github.com/freelabz/secator/commit/7d74ae80bfd99c31714a5e7e25f2bd1caa642eb4))
+
 ## [0.5.0](https://github.com/freelabz/secator/compare/v0.4.1...v0.5.0) (2024-05-03)
 
 

{secator-0.5.0 → secator-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: secator
-Version: 0.5.0
+Version: 0.5.2
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.5.0 → secator-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "secator"
-version = "0.5.0"
+version = "0.5.2"
 authors = [{ name = "FreeLabz", email = "sales@freelabz.com" }]
 readme = "README.md"
 description = "The pentester's swiss knife."

{secator-0.5.0 → secator-0.5.2}/secator/celery.py

@@ -1,11 +1,10 @@
 import gc
 import logging
 import traceback
-from time import sleep
 
 from celery import Celery, chain, chord, signals
 from celery.app import trace
-from celery.result import AsyncResult, allow_join_result
+from celery.result import allow_join_result
 # from pyinstrument import Profiler  # TODO: make pyinstrument optional
 from rich.logging import RichHandler
 
@@ -354,56 +353,9 @@ def forward_results(results):
 	results = deduplicate(results, attr='_uuid')
 	return results
 
-
-#---------------------#
-# Celery result utils #
-#---------------------#
-
-
-def poll_task(result, seen=[]):
-	"""Poll Celery result tree recursively to get results live.
-
-	TODO: function is incomplete, as it does not parse all results.
-
-	Args:
-		result (Union[AsyncResult, GroupResult]): Celery result object.
-		seen (list): List of seen results (do not yield again).
-
-	Yields:
-		dict: Result.
-	"""
-	if result is None:
-		return
-
-	if result.children:
-		for child in result.children:
-			yield from poll_task(child, seen=seen)
-	else:
-		res = AsyncResult(result.id)
-		if not res.info:
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
-
-		# Task done running
-		if isinstance(res.info, list):
-			for item in res.info:
-				if item._uuid not in seen:
-					yield res.id, None, item
-					seen.append(item._uuid)
-			return
-
-		# Get task partial results, remove duplicates
-		results = res.info['results']
-		name = res.info['name']
-		for item in results:
-			if item._uuid not in seen:
-				yield res.id, name, item
-				seen.append(item._uuid)
-
-		# Task still running, keep polling
-		if not res.ready():
-			sleep(0.1)
-			yield from poll_task(result, seen=seen)
+#--------------#
+# Celery utils #
+#--------------#
 
 
 def is_celery_worker_alive():

{secator-0.5.0 → secator-0.5.2}/secator/config.py

@@ -74,6 +74,7 @@ class Runners(StrictModel):
 	progress_update_frequency: int = 60
 	skip_cve_search: bool = False
 	skip_cve_low_confidence: bool = True
+	remove_duplicates: bool = False
 
 
 class HTTP(StrictModel):

{secator-0.5.0 → secator-0.5.2}/secator/output_types/url.py

@@ -23,6 +23,7 @@ class Url(OutputType):
 	lines: int = field(default=0, compare=False)
 	screenshot_path: str = field(default='', compare=False)
 	stored_response_path: str = field(default='', compare=False)
+	headers: dict = field(default_factory=dict, repr=True, compare=False)
 	_source: str = field(default='', repr=True, compare=False)
 	_type: str = field(default='url', repr=True)
 	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
@@ -55,6 +56,8 @@ class Url(OutputType):
 
 	def __repr__(self):
 		s = f'🔗 [white]{self.url}'
+		if self.method and self.method != 'GET':
+			s += f' \[[turquoise4]{self.method}[/]]'
 		if self.status_code and self.status_code != 0:
 			if self.status_code < 400:
 				s += f' \[[green]{self.status_code}[/]]'

{secator-0.5.0 → secator-0.5.2}/secator/report.py

@@ -1,5 +1,6 @@
 import operator
 
+from secator.config import CONFIG
 from secator.output_types import OUTPUT_TYPES, OutputType
 from secator.utils import merge_opts, get_file_timestamp
 from secator.rich import console
@@ -64,8 +65,10 @@ class Report:
 			sort_by, _ = get_table_fields(output_type)
 			items = [
 				item for item in self.runner.results
-				if isinstance(item, OutputType) and item._type == output_name and not item._duplicate
+				if isinstance(item, OutputType) and item._type == output_name
 			]
+			if CONFIG.runners.remove_duplicates:
+				items = [item for item in items if not item._duplicate]
 			if items:
 				if sort_by and all(sort_by):
 					items = sorted(items, key=operator.attrgetter(*sort_by))

{secator-0.5.0 → secator-0.5.2}/secator/tasks/httpx.py

@@ -31,6 +31,7 @@ class httpx(Http):
 		'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'},
 		'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'},
 		'vhost': {'is_flag': True, 'default': False, 'help': 'Probe and display server supporting VHOST'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Save HTTP responses'},  # noqa: E501
 		'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'},
 		'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'},
 		'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'},
@@ -55,6 +56,7 @@ class httpx(Http):
 		THREADS: 'threads',
 		TIMEOUT: 'timeout',
 		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr',
 	}
 	opt_value_map = {
 		DELAY: lambda x: str(x) + 's' if x else None,
@@ -71,15 +73,10 @@ class httpx(Http):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.output_response_path = f'{self.reports_folder}/response'
-			self.output_screenshot_path = f'{self.reports_folder}/screenshot'
-			os.makedirs(self.output_response_path, exist_ok=True)
-			os.makedirs(self.output_screenshot_path, exist_ok=True)
-			self.cmd += f' -sr -srd {self.reports_folder}'
-
-		# Remove screenshot bytes and body bytes when screenshot
 		screenshot = self.get_opt_value('screenshot')
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses or screenshot:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 		if screenshot:
 			self.cmd += ' -esb -ehb'
 
@@ -98,8 +95,15 @@ class httpx(Http):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses:
-			if os.path.exists(self.output_response_path + '/index.txt'):
-				os.remove(self.output_response_path + '/index.txt')
-			if os.path.exists(self.output_screenshot_path + '/index.txt'):
-				os.remove(self.output_screenshot_path + '/index_screenshot.txt')
+		store_responses = self.get_opt_value('store_responses')
+		response_dir = f'{self.reports_folder}/.outputs'
+		if store_responses:
+			index_rpath = f'{response_dir}/response/index.txt'
+			index_spath = f'{response_dir}/screenshot/index_screenshot.txt'
+			index_spath2 = f'{response_dir}/screenshot/screenshot.html'
+			if os.path.exists(index_rpath):
+				os.remove(index_rpath)
+			if os.path.exists(index_spath):
+				os.remove(index_spath)
+			if os.path.exists(index_spath2):
+				os.remove(index_spath2)

{secator-0.5.0 → secator-0.5.2}/secator/tasks/katana.py

@@ -29,7 +29,8 @@ class katana(HttpCrawler):
 	opts = {
 		'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'},
 		'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'},
-		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'}
+		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'}
 	}
 	opt_key_map = {
 		HEADER: 'headers',
@@ -50,7 +51,8 @@ class katana(HttpCrawler):
 		RETRIES: 'retry',
 		THREADS: 'concurrency',
 		TIMEOUT: 'timeout',
-		USER_AGENT: OPT_NOT_SUPPORTED
+		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr'
 	}
 	opt_value_map = {
 		DELAY: lambda x: int(x) if isinstance(x, float) else x
@@ -107,14 +109,16 @@ class katana(HttpCrawler):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.cmd += f' -sr -srd {self.reports_folder}'
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 
 	@staticmethod
 	def on_item(self, item):
 		if not isinstance(item, Url):
 			return item
-		if CONFIG.http.store_responses and os.path.exists(item.stored_response_path):
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses and os.path.exists(item.stored_response_path):
 			with open(item.stored_response_path, 'r', encoding='latin-1') as fin:
 				data = fin.read().splitlines(True)
 				first_line = data[0]
@@ -126,5 +130,7 @@ class katana(HttpCrawler):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses and os.path.exists(self.reports_folder + '/index.txt'):
-			os.remove(self.reports_folder + '/index.txt')
+		store_responses = self.get_opt_value('store_responses')
+		index_rpath = f'{self.reports_folder}/.outputs/index.txt'
+		if store_responses and os.path.exists(index_rpath):
+			os.remove(index_rpath)

{secator-0.5.0 → secator-0.5.2}/secator/utils.py

@@ -343,8 +343,10 @@ def print_results_table(results, title=None, exclude_fields=[], log=False):
 		if output_type.__name__ == 'Progress':
 			continue
 		items = [
-			item for item in results if item._type == output_type.get_name() and not item._duplicate
+			item for item in results if item._type == output_type.get_name()
 		]
+		if CONFIG.runners.remove_duplicates:
+			items = [item for item in items if not item._duplicate]
 		if items:
 			_table = build_table(
 				items,