secator 0.5.1__py3-none-any.whl → 0.6.0__py3-none-any.whl

secator/hooks/mongodb.py

@@ -206,6 +206,6 @@ MONGODB_HOOKS = {
 		'on_item': [update_finding],
 		'on_duplicate': [update_finding],
 		'on_iter': [update_runner],
-		'on_end': [update_runner]
+		'on_end': [update_runner, find_duplicates]
 	}
 }

secator/output_types/progress.py

@@ -8,7 +8,7 @@ from secator.utils import rich_to_ansi
 @dataclass
 class Progress(OutputType):
 	duration: str
-	percent: int
+	percent: int = 0
 	errors: list = field(default_factory=list)
 	extra_data: dict = field(default_factory=dict)
 	_source: str = field(default='', repr=True)

secator/tasks/_categories.py

@@ -44,7 +44,7 @@ OPTS_HTTP = [
 ]
 
 OPTS_HTTP_CRAWLERS = OPTS_HTTP + [
-	DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT,
+	DEPTH, MATCH_REGEX, MATCH_SIZE, MATCH_WORDS, FILTER_REGEX, FILTER_CODES, FILTER_SIZE, FILTER_WORDS,
 	MATCH_CODES
 ]
 

secator/tasks/httpx.py

@@ -31,6 +31,7 @@ class httpx(Http):
 		'cdn': {'is_flag': True, 'default': False, 'help': 'CDN detection'},
 		'debug_resp': {'is_flag': True, 'default': False, 'help': 'Debug response'},
 		'vhost': {'is_flag': True, 'default': False, 'help': 'Probe and display server supporting VHOST'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Save HTTP responses'},  # noqa: E501
 		'screenshot': {'is_flag': True, 'short': 'ss', 'default': False, 'help': 'Screenshot response'},
 		'system_chrome': {'is_flag': True, 'default': False, 'help': 'Use local installed Chrome for screenshot'},
 		'headless_options': {'is_flag': False, 'short': 'ho', 'default': None, 'help': 'Headless Chrome additional options'},
@@ -55,6 +56,7 @@ class httpx(Http):
 		THREADS: 'threads',
 		TIMEOUT: 'timeout',
 		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr',
 	}
 	opt_value_map = {
 		DELAY: lambda x: str(x) + 's' if x else None,
@@ -71,15 +73,10 @@ class httpx(Http):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.output_response_path = f'{self.reports_folder}/response'
-			self.output_screenshot_path = f'{self.reports_folder}/screenshot'
-			os.makedirs(self.output_response_path, exist_ok=True)
-			os.makedirs(self.output_screenshot_path, exist_ok=True)
-			self.cmd += f' -sr -srd {self.reports_folder}'
-
-		# Remove screenshot bytes and body bytes when screenshot
 		screenshot = self.get_opt_value('screenshot')
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses or screenshot:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 		if screenshot:
 			self.cmd += ' -esb -ehb'
 
@@ -98,8 +95,15 @@ class httpx(Http):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses:
-			if os.path.exists(self.output_response_path + '/index.txt'):
-				os.remove(self.output_response_path + '/index.txt')
-			if os.path.exists(self.output_screenshot_path + '/index.txt'):
-				os.remove(self.output_screenshot_path + '/index_screenshot.txt')
+		store_responses = self.get_opt_value('store_responses')
+		response_dir = f'{self.reports_folder}/.outputs'
+		if store_responses:
+			index_rpath = f'{response_dir}/response/index.txt'
+			index_spath = f'{response_dir}/screenshot/index_screenshot.txt'
+			index_spath2 = f'{response_dir}/screenshot/screenshot.html'
+			if os.path.exists(index_rpath):
+				os.remove(index_rpath)
+			if os.path.exists(index_spath):
+				os.remove(index_spath)
+			if os.path.exists(index_spath2):
+				os.remove(index_spath2)

secator/tasks/katana.py

@@ -29,7 +29,8 @@ class katana(HttpCrawler):
 	opts = {
 		'headless': {'is_flag': True, 'short': 'hl', 'help': 'Headless mode'},
 		'system_chrome': {'is_flag': True, 'short': 'sc', 'help': 'Use local installed chrome browser'},
-		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'}
+		'form_extraction': {'is_flag': True, 'short': 'fx', 'help': 'Detect forms'},
+		'store_responses': {'is_flag': True, 'short': 'sr', 'default': CONFIG.http.store_responses, 'help': 'Store responses'}
 	}
 	opt_key_map = {
 		HEADER: 'headers',
@@ -50,7 +51,8 @@ class katana(HttpCrawler):
 		RETRIES: 'retry',
 		THREADS: 'concurrency',
 		TIMEOUT: 'timeout',
-		USER_AGENT: OPT_NOT_SUPPORTED
+		USER_AGENT: OPT_NOT_SUPPORTED,
+		'store_responses': 'sr'
 	}
 	opt_value_map = {
 		DELAY: lambda x: int(x) if isinstance(x, float) else x
@@ -107,14 +109,16 @@ class katana(HttpCrawler):
 		debug_resp = self.get_opt_value('debug_resp')
 		if debug_resp:
 			self.cmd = self.cmd.replace('-silent', '')
-		if CONFIG.http.store_responses:
-			self.cmd += f' -sr -srd {self.reports_folder}'
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses:
+			self.cmd += f' -srd {self.reports_folder}/.outputs'
 
 	@staticmethod
 	def on_item(self, item):
 		if not isinstance(item, Url):
 			return item
-		if CONFIG.http.store_responses and os.path.exists(item.stored_response_path):
+		store_responses = self.get_opt_value('store_responses')
+		if store_responses and os.path.exists(item.stored_response_path):
 			with open(item.stored_response_path, 'r', encoding='latin-1') as fin:
 				data = fin.read().splitlines(True)
 				first_line = data[0]
@@ -126,5 +130,7 @@ class katana(HttpCrawler):
 
 	@staticmethod
 	def on_end(self):
-		if CONFIG.http.store_responses and os.path.exists(self.reports_folder + '/index.txt'):
-			os.remove(self.reports_folder + '/index.txt')
+		store_responses = self.get_opt_value('store_responses')
+		index_rpath = f'{self.reports_folder}/.outputs/index.txt'
+		if store_responses and os.path.exists(index_rpath):
+			os.remove(index_rpath)

{secator-0.5.1.dist-info → secator-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: secator
-Version: 0.5.1
+Version: 0.6.0
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.8
-Requires-Dist: bs4<1
+Requires-Dist: beautifulsoup4<=5
 Requires-Dist: celery<6
 Requires-Dist: cpe<2
 Requires-Dist: dotmap<2

{secator-0.5.1.dist-info → secator-0.6.0.dist-info}/RECORD

@@ -43,13 +43,13 @@ secator/exporters/json.py,sha256=cWkDugUdy-lbcPFKNgBrRFxHspiFhjVbJfdDABjJ9uk,431
 secator/exporters/table.py,sha256=RHQoaFeeyeoBGNucJgrlk2KtmVqe9BGNtAAYee7xJ8Y,210
 secator/exporters/txt.py,sha256=QbiwWYGgHpITGw1sL2TX-S3AfmBdJ-VOWkPJzuBvOu4,785
 secator/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/hooks/mongodb.py,sha256=PGcM6hkTl4bt46-cPlFgcY-PfNWmHpz_eiv77XeV0-A,7036
+secator/hooks/mongodb.py,sha256=BmKLDgGFxZ99Qh2OoECmWR6iZGt7DAP7vYDzZYIHweg,7053
 secator/output_types/__init__.py,sha256=uj6AXDeorECPwhwekNVGjQbGv41jHG_8udkuoc4XzW0,854
 secator/output_types/_base.py,sha256=bld1ED0pN1hOvwBV2canrlKrfBCgawzWKPDH6F3jVQE,2469
 secator/output_types/exploit.py,sha256=tMQcEr4kAZ-na4F6sU2WpVH6Pa6r0oPumXOrxptTrbY,1621
 secator/output_types/ip.py,sha256=ySEqH3Gs7U41I1kS8olZ_p3Mk7JryEbXHLyexqlBQNA,995
 secator/output_types/port.py,sha256=1ZmV4FDvwk1dvFXySnz5yIp13hbaRhnunxnETm66Os0,1607
-secator/output_types/progress.py,sha256=u_-4IiECTSCJf-X_RkFOoFyb8mrff2rMcm8GGqLZ8hs,1231
+secator/output_types/progress.py,sha256=aA2NvHzkPoQCx9gL3ZPDHcdG8qOP5LPz4XYnDx_HmYI,1235
 secator/output_types/record.py,sha256=WnI0yvwzrO2Wt7OWciHMOuIRRLbuSOAJczdNshV7tYU,1192
 secator/output_types/subdomain.py,sha256=lmCoK7_8I4FXWgl9kToRvDn3gr3E3uBTaQzFAOHbswE,1343
 secator/output_types/tag.py,sha256=8AlT0VigsYP04GN8sPCTM07IlL5uMUmFgsNa9IDCoyY,1431
@@ -69,7 +69,7 @@ secator/serializers/dataclass.py,sha256=g5gMT4NwndjhGcGbFuYEs07AZW_Q_m9orov_edVE
 secator/serializers/json.py,sha256=XwuSQOBwrOAs16F5HtY-Q-rAGAxfNvlq3z-Nb2gwigE,304
 secator/serializers/regex.py,sha256=hGJ_1JSOv9xPtfn_umHlsjnR_alnsDFv-UmjYCC3vwU,314
 secator/tasks/__init__.py,sha256=yRIZf9E47aS7o6rpgAJLgJUpX2cug1ofZeq8QsxgyjU,192
-secator/tasks/_categories.py,sha256=2cUsZOdYHA-YXJwryU2FTTT4Y4xXzmDJ92F8ud-MDJQ,10402
+secator/tasks/_categories.py,sha256=LgGqfDn_QUXRa1KgZwlrYqi3tkgR8E4C_Mg_GgAcZAk,10385
 secator/tasks/cariddi.py,sha256=GKVJ8nWtJu9fB_FhAVYA2TX3fMdKYdbMpH2IhCkj_no,3155
 secator/tasks/dalfox.py,sha256=nrLkIbTNz_J7LgUy_3kBgzhTUbQi3RmiSJhc9HWa05c,1744
 secator/tasks/dirsearch.py,sha256=2hJeJZJwaAl3-UAjBwlmjW1w9bxjVWxxwfcaTTxqClc,2387
@@ -83,8 +83,8 @@ secator/tasks/gf.py,sha256=WlhoEyL6xE79w6nE5XNSXHs-jVeO10njqJxBF8w20sA,945
 secator/tasks/gospider.py,sha256=_UlTb9G5Ss8D68NT53s0_rI6TnG00Ph0yxWyHic7cKs,2172
 secator/tasks/grype.py,sha256=n60Zs9d1NWJFHQ0DwIZib5wu3xH-tV2RzgLYwuQSTo4,2413
 secator/tasks/h8mail.py,sha256=hZBpfV6M1mbpD_PbDHxLI5HMvqAvTeY_W0lbkq3Hugo,2037
-secator/tasks/httpx.py,sha256=ugk4AOIqhvT5-HVhlRsQw_oF2BGovgHlS-_D9wav8wo,3972
-secator/tasks/katana.py,sha256=3JHtvxoSxJtju9xqvQptVaYXKnLLlCkOn6oY0A8h7wM,4371
+secator/tasks/httpx.py,sha256=O4jkee6So9MSyHTb7n_owFiL3OMO0697uxDq7ULbq5s,4108
+secator/tasks/katana.py,sha256=edsjIVj8sh5_ubCB5AFjANwute0Uv4Vdry0XB10xE1Q,4677
 secator/tasks/maigret.py,sha256=PZDTICJ4LZF3joKe-dXu2alffakD_1sxBuNEUBtJDm4,2098
 secator/tasks/mapcidr.py,sha256=7aa2WXQATWgIQo5oA12URjAg80L6MFMGdxScxls8DuA,980
 secator/tasks/msfconsole.py,sha256=Cm0vzOFff17C4M1YjkgU6T7Jc5-ClBK0Qi_529qVRb0,6065
@@ -94,8 +94,8 @@ secator/tasks/nuclei.py,sha256=lKZYPVcnCYomd830-ZCOz4fyc8xAKjNDuKayyz0BPek,3507
 secator/tasks/searchsploit.py,sha256=tIqCwYFIyHIgJbtcTL56PXqd-MCvoXOpvSDgoK_dxzc,2953
 secator/tasks/subfinder.py,sha256=cpFyFCpVaDZ3QAjNId26ezOwntn3CA5Uk-AC2l0mo0E,1087
 secator/tasks/wpscan.py,sha256=UVWnBPOQ1RDB2wzMswWR6vc6cucYgHtuJ8pLZoqCM40,5434
-secator-0.5.1.dist-info/METADATA,sha256=wEJx34zkxon4GAdURXZBuEjQTWzeTuz-LvLPXReNgtM,14095
-secator-0.5.1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-secator-0.5.1.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
-secator-0.5.1.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
-secator-0.5.1.dist-info/RECORD,,
+secator-0.6.0.dist-info/METADATA,sha256=KN4Jv-YysvpK61IUZHgrx12ICXj5mt8pDoeZEARi2uE,14107
+secator-0.6.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+secator-0.6.0.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
+secator-0.6.0.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
+secator-0.6.0.dist-info/RECORD,,

{secator-0.5.1.dist-info → secator-0.6.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any