secator 0.16.4__py3-none-any.whl → 0.16.5__py3-none-any.whl

secator/celery.py

@@ -240,7 +240,8 @@ def forward_results(results):
 		console.print(Info(message=f'Deduplicating {len(results)} results'))
 
 	results = flatten(results)
-	if CONFIG.addons.mongodb.enabled:
+	if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
+		console.print(Info(message=f'Extracting uuids from {len(results)} results'))
 		uuids = [r._uuid for r in results if hasattr(r, '_uuid')]
 		uuids.extend([r for r in results if isinstance(r, str)])
 		results = list(set(uuids))
@@ -271,12 +272,14 @@ def mark_runner_started(results, runner, enable_hooks=True):
 	if results:
 		results = forward_results(results)
 	runner.enable_hooks = enable_hooks
-	if CONFIG.addons.mongodb.enabled:
+	if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
 		from secator.hooks.mongodb import get_results
 		results = get_results(results)
 	for item in results:
 		runner.add_result(item, print=False)
 	runner.mark_started()
+	if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
+		return [r._uuid for r in runner.results]
 	return runner.results
 
 
@@ -297,12 +300,14 @@ def mark_runner_completed(results, runner, enable_hooks=True):
 	debug(f'Runner {runner.unique_name} has finished, running mark_completed', sub='celery')
 	results = forward_results(results)
 	runner.enable_hooks = enable_hooks
-	if CONFIG.addons.mongodb.enabled:
+	if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
 		from secator.hooks.mongodb import get_results
 		results = get_results(results)
 	for item in results:
 		runner.add_result(item, print=False)
 	runner.mark_completed()
+	if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
+		return [r._uuid for r in runner.results]
 	return runner.results
 
 

secator/hooks/gcs.py

@@ -42,10 +42,12 @@ def upload_blob(bucket_name, source_file_name, destination_blob_name):
 	storage_client = storage.Client()
 	bucket = storage_client.bucket(bucket_name)
 	blob = bucket.blob(destination_blob_name)
-	blob.upload_from_filename(source_file_name)
+	with open(source_file_name, 'rb') as f:
+		f.seek(0)
+		blob.upload_from_file(f)
 	end_time = time()
 	elapsed = end_time - start_time
-	debug(f'in {elapsed:.4f}s', obj={'blob': 'CREATED', 'blob_name': destination_blob_name, 'bucket': bucket_name}, obj_after=False, sub='hooks.gcs', verbose=True)  # noqa: E501
+	debug(f'in {elapsed:.4f}s', obj={'blob': 'UPLOADED', 'blob_name': destination_blob_name, 'bucket': bucket_name}, obj_after=False, sub='hooks.gcs', verbose=True)  # noqa: E501
 
 
 HOOKS = {

secator/hooks/mongodb.py

@@ -30,7 +30,8 @@ def get_mongodb_client():
 		_mongodb_client = pymongo.MongoClient(
 			escape_mongodb_url(MONGODB_URL),
 			maxPoolSize=MONGODB_MAX_POOL_SIZE,
-			serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT
+			serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT,
+			connect=False
 		)
 	return _mongodb_client
 
@@ -157,19 +158,23 @@ def load_findings(objs):
 
 
 @shared_task
-def tag_duplicates(ws_id: str = None):
+def tag_duplicates(ws_id: str = None, full_scan: bool = False):
 	"""Tag duplicates in workspace.
 
 	Args:
 		ws_id (str): Workspace id.
+		full_scan (bool): If True, scan all findings, otherwise only untagged findings.
 	"""
 	debug(f'running duplicate check on workspace {ws_id}', sub='hooks.mongodb')
 	client = get_mongodb_client()
 	db = client.main
 	workspace_query = list(
 		db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': True}).sort('_timestamp', -1))
+	untagged_query = {'_context.workspace_id': str(ws_id)}
+	if not full_scan:
+		untagged_query['_tagged'] = {'$ne': True}
 	untagged_query = list(
-		db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': {'$ne': True}}).sort('_timestamp', -1))
+		db.findings.find(untagged_query).sort('_timestamp', -1))
 	if not untagged_query:
 		debug('no untagged findings. Skipping.', id=ws_id, sub='hooks.mongodb')
 		return

secator/tasks/_categories.py

@@ -96,6 +96,20 @@ class HttpFuzzer(Command):
 	meta_opts = {k: OPTS[k] for k in OPTS_HTTP_FUZZERS}
 	input_types = [URL]
 	output_types = [Url]
+	profile = lambda opts: HttpFuzzer.dynamic_profile(opts)  # noqa: E731
+
+	@staticmethod
+	def dynamic_profile(opts):
+		wordlist = HttpFuzzer._get_opt_value(
+			opts,
+			'wordlist',
+			opts_conf=dict(HttpFuzzer.opts, **HttpFuzzer.meta_opts),
+			opt_aliases=opts.get('aliases', []),
+			preprocess=True,
+			process=True,
+		)
+		wordlist_size_mb = os.path.getsize(wordlist) / (1024 * 1024)
+		return 'cpu' if wordlist_size_mb > 5 else 'io'
 
 
 #----------------#

{secator-0.16.4.dist-info → secator-0.16.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: secator
-Version: 0.16.4
+Version: 0.16.5
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.16.4.dist-info → secator-0.16.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 secator/.gitignore,sha256=da8MUc3hdb6Mo0WjZu2upn5uZMbXcBGvhdhTQ1L89HI,3093
 secator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/celery.py,sha256=st9wpAwilYk5BCO18wscGt0vqQdCIP4Pynw_9RcvWec,11246
+secator/celery.py,sha256=FAahY_kfjxc3tD2R42kNre_eb-QeSvArKyGpthXeWDg,11626
 secator/celery_signals.py,sha256=R4ZNBPKSxUvesGCvZ7MXoRkWNOTMS5hraZzjLh5sQ0o,4191
 secator/celery_utils.py,sha256=vhL5ZxXDn3ODvyVxMijKyUTJ1dOisMDjF_PhFUyOVSA,9451
 secator/cli.py,sha256=lzgttr8-Hib1X6bGi8PCOfX90incum7ZFR5x46cDZ34,60887
@@ -57,8 +57,8 @@ secator/exporters/json.py,sha256=1ZtDf8RksPO_V0zIvnwDUxMUb630DCElAMM8_RQvyAo,474
 secator/exporters/table.py,sha256=zYNmwNGEyB6dTJ1ATVkrv-AOuPjrW6tvk1_4naLQo8Q,1114
 secator/exporters/txt.py,sha256=t_FykaJOxs4UUlqiH4k6HCccEqYqc8e3iNZndL_CKPg,739
 secator/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/hooks/gcs.py,sha256=CNQmDWbwnIA1mDhzwO75P8907lAFd5-vA2sJVMbhAiY,1514
-secator/hooks/mongodb.py,sha256=fCm_E1hxgSl9C6l4eGW8J-Jjstvbfy9daITWYcmNTe8,8198
+secator/hooks/gcs.py,sha256=v8Q1-ky9e4xaLgyTJaeWWpiVhu0SzWbXLIEHYV9f5h8,1550
+secator/hooks/mongodb.py,sha256=x6gLPdvMOBz0ViWbfaM9Xkv1plLYvXjKWpd83b0VUFU,8393
 secator/output_types/__init__.py,sha256=CJcYy2_Ek-opKiBz4wFlDHQBTm3t0JVwZ4w_2Jxoeuw,1291
 secator/output_types/_base.py,sha256=9iBqPdtlfJBldBiuC729KamHHGbKhwo69P-2UNwz-3Q,2874
 secator/output_types/certificate.py,sha256=IXW3GN0JRmuDgoedr8NV8ccuRQOuoInNZWnAKL8zeqY,3040
@@ -92,7 +92,7 @@ secator/serializers/dataclass.py,sha256=Fo2ZVsVjSF0KMq81JT8mGZxsKsxyueShlDsh9Pgw
 secator/serializers/json.py,sha256=UJwAymRzjF-yBKOgz1MTOyBhQcdQg7fOKRXgmHIu8fo,411
 secator/serializers/regex.py,sha256=fh-fE0RGvKSGKByFtwmKsWriRpZR9PXZQsY9JybHBWI,489
 secator/tasks/__init__.py,sha256=Op0O0Aa8c124AfDG-cEB9VLRsXZ1wXTpVrT3g-wxMNg,184
-secator/tasks/_categories.py,sha256=yns_5PBKStp6TJEeaYB6yFUjkFMmLh7LEuxcNcADNro,14962
+secator/tasks/_categories.py,sha256=ZmUNzeFIZ9-_er9sLJw66PTYIL5nO799JQU3EoW-6nE,15394
 secator/tasks/arjun.py,sha256=WdRZtTCd2Ejbv5HlLS_FoWVKgGpMsR6RCDekV2kR788,3061
 secator/tasks/bbot.py,sha256=moIkwd52jCKaeg1v6Nv4Gfmd4GPObo9c9nwOzQvf-2M,9236
 secator/tasks/bup.py,sha256=9IXsCqMdhOeZcCsQB2L4IJ3Kzm2oQKDE7mflGljm0lM,3867
@@ -125,8 +125,8 @@ secator/tasks/wafw00f.py,sha256=9CnV9F7ZrykO27F3PAb5HtwULDMYEKGSTbz-jh0kc2g,3189
 secator/tasks/wpprobe.py,sha256=1QPJ-7JvhL7LFvjUTAmqpH2Krp-Qmi079lonso16YPQ,3229
 secator/tasks/wpscan.py,sha256=dBkbG9EODHDUBAA8uNVULX4SdVgTCAi_F1T1oCfRbsI,5852
 secator/workflows/__init__.py,sha256=XOviyjSylZ4cuVmmQ76yuqZRdmvOEghqAnuw_4cLmfk,702
-secator-0.16.4.dist-info/METADATA,sha256=E_R6VdalGxZysqe043SuT0M_897S8cT5kXhmwOSWLpY,17253
-secator-0.16.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-secator-0.16.4.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
-secator-0.16.4.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
-secator-0.16.4.dist-info/RECORD,,
+secator-0.16.5.dist-info/METADATA,sha256=Bob8xdFbxnKib4O7WIiG9VAtq9efZ9JLzwpPQIcXhAw,17253
+secator-0.16.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+secator-0.16.5.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
+secator-0.16.5.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
+secator-0.16.5.dist-info/RECORD,,