secator 0.10.1a2__py3-none-any.whl → 0.10.1a3__py3-none-any.whl

secator/celery.py

@@ -2,7 +2,6 @@ import gc
 import json
 import logging
 import os
-import sys
 import uuid
 
 from time import time
@@ -13,14 +12,13 @@ from celery.app import trace
 from rich.logging import RichHandler
 from retry import retry
 
-from secator.celery_signals import setup_handlers
+from secator.celery_signals import IN_CELERY_WORKER_PROCESS, setup_handlers
 from secator.config import CONFIG
 from secator.output_types import Info
 from secator.rich import console
 from secator.runners import Scan, Task, Workflow
 from secator.utils import (debug, deduplicate, flatten, should_update)
 
-IN_CELERY_WORKER_PROCESS = sys.argv and ('secator.celery.app' in sys.argv or 'worker' in sys.argv)
 
 #---------#
 # Logging #
@@ -148,14 +146,14 @@ def handle_runner_error(self, results, runner):
 	return runner.results
 
 
-def break_task(task, task_opts, targets, results=[], chunk_size=1):
+def break_task(task, task_opts, results=[]):
 	"""Break a task into multiple of the same type."""
-	chunks = targets
-	if chunk_size > 1:
-		chunks = list(chunker(targets, chunk_size))
+	chunks = task.inputs
+	if task.input_chunk_size > 1:
+		chunks = list(chunker(task.inputs, task.input_chunk_size))
 	debug(
 		'',
-		obj={task.unique_name: 'CHUNKED', 'chunk_size': chunk_size, 'chunks': len(chunks), 'target_count': len(targets)},
+		obj={task.unique_name: 'CHUNKED', 'chunk_size': task.input_chunk_size, 'chunks': len(chunks), 'target_count': len(task.inputs)},  # noqa: E501
 		obj_after=False,
 		sub='celery.state',
 		verbose=True
@@ -196,6 +194,8 @@ def break_task(task, task_opts, targets, results=[], chunk_size=1):
 def run_task(self, args=[], kwargs={}):
 	print('run task')
 	console.print(Info(message=f'Running task {self.request.id}'))
+	if 'context' not in kwargs:
+		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
 	task = Task(*args, **kwargs)
 	task.run()
@@ -204,6 +204,8 @@ def run_task(self, args=[], kwargs={}):
 @app.task(bind=True)
 def run_workflow(self, args=[], kwargs={}):
 	console.print(Info(message=f'Running workflow {self.request.id}'))
+	if 'context' not in kwargs:
+		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
 	workflow = Workflow(*args, **kwargs)
 	workflow.run()
@@ -244,9 +246,11 @@ def run_command(self, results, name, targets, opts={}):
 	update_state(self, task, force=True)
 
 	# Chunk task if needed
-	if task_cls.needs_chunking(targets, sync):
+	if task.needs_chunking(sync):
 		console.print(Info(message=f'Task {name} requires chunking, breaking into {len(targets)} tasks'))
-		return self.replace(break_task(task, opts, targets, results=results))
+		tasks = break_task(task, opts, results=results)
+		update_state(self, task, force=True)
+		return self.replace(tasks)
 
 	# Update state live
 	[update_state(self, task) for _ in task]
@@ -268,7 +272,8 @@ def forward_results(results):
 		results = results['results']
 	results = flatten(results)
 	results = deduplicate(results, attr='_uuid')
-	console.print(Info(message=f'Forwarding {len(results)} results ...'))
+	if IN_CELERY_WORKER_PROCESS:
+		console.print(Info(message=f'Forwarding {len(results)} results ...'))
 	return results
 
 #--------------#

secator/celery_signals.py

@@ -1,5 +1,6 @@
 import os
 import signal
+import sys
 import threading
 from pathlib import Path
 
@@ -10,6 +11,7 @@ from secator.output_types import Info
 from secator.rich import console
 
 IDLE_TIMEOUT = CONFIG.celery.worker_kill_after_idle_seconds
+IN_CELERY_WORKER_PROCESS = sys.argv and ('secator.celery.app' in sys.argv or 'worker' in sys.argv)
 
 # File-based state management system
 STATE_DIR = Path("/tmp/celery_state")

secator/celery_utils.py

@@ -12,7 +12,7 @@ from rich.padding import Padding
 from rich.progress import Progress as RichProgress, SpinnerColumn, TextColumn, TimeElapsedColumn
 from secator.config import CONFIG
 from secator.definitions import STATE_COLORS
-from secator.output_types import Error
+from secator.output_types import Error, Info
 from secator.rich import console
 from secator.utils import debug, traceback_as_string
 
@@ -76,10 +76,31 @@ class CeleryData(object):
 
 			# Get live results and print progress
 			for data in CeleryData.poll(result, ids_map, refresh_interval):
-				yield from data['results']
+				for result in data['results']:
+
+					# Add dynamic subtask to ids_map
+					if isinstance(result, Info):
+						message = result.message
+						if message.startswith('Celery chunked task created: '):
+							task_id = message.split(' ')[-1]
+							ids_map[task_id] = {
+								'id': task_id,
+								'name': result._source,
+								'full_name': result._source,
+								'descr': '',
+								'state': 'PENDING',
+								'count': 0,
+								'progress': 0
+							}
+					yield result
 
 				if print_remote_info:
 					task_id = data['id']
+					if task_id not in progress_cache:
+						if CONFIG.runners.show_subtasks:
+							progress_cache[task_id] = progress.add_task('', advance=0, **data)
+						else:
+							continue
 					progress_id = progress_cache[task_id]
 					CeleryData.update_progress(progress, progress_id, data)
 

secator/config.py

@@ -93,6 +93,7 @@ class Runners(StrictModel):
 	skip_exploit_search: bool = False
 	skip_cve_low_confidence: bool = False
 	remove_duplicates: bool = False
+	show_chunk_progress: bool = False
 
 
 class Security(StrictModel):

secator/runners/_base.py

@@ -97,6 +97,7 @@ class Runner:
 		self.threads = []
 		self.no_poll = self.run_opts.get('no_poll', False)
 		self.quiet = self.run_opts.get('quiet', False)
+		self.started = False
 
 		# Runner process options
 		self.no_process = self.run_opts.get('no_process', False)
@@ -117,10 +118,21 @@ class Runner:
 		self.raise_on_error = self.run_opts.get('raise_on_error', False)
 		self.print_opts = {k: v for k, v in self.__dict__.items() if k.startswith('print_') if v}
 
+		# Chunks
+		self.has_parent = self.run_opts.get('has_parent', False)
+		self.has_children = self.run_opts.get('has_children', False)
+		self.chunk = self.run_opts.get('chunk', None)
+		self.chunk_count = self.run_opts.get('chunk_count', None)
+		self.unique_name = self.name.replace('/', '_')
+		self.unique_name = f'{self.unique_name}_{self.chunk}' if self.chunk else self.unique_name
+
+		# Add prior results to runner results
+		[self.add_result(result, print=False, output=False) for result in results]
+
 		# Determine inputs
 		inputs = [inputs] if not isinstance(inputs, list) else inputs
-		if results:
-			inputs, run_opts, errors = run_extractors(results, run_opts, inputs)
+		if not self.chunk and self.results:
+			inputs, run_opts, errors = run_extractors(self.results, run_opts, inputs)
 			for error in errors:
 				self.add_result(error, print=True)
 		self.inputs = inputs
@@ -163,18 +175,6 @@ class Runner:
 		self.validators = {name: [] for name in VALIDATORS + getattr(self, 'validators', [])}
 		self.register_validators(validators)
 
-		# Chunks
-		self.has_parent = self.run_opts.get('has_parent', False)
-		self.has_children = self.run_opts.get('has_children', False)
-		self.chunk = self.run_opts.get('chunk', None)
-		self.chunk_count = self.run_opts.get('chunk_count', None)
-		self.unique_name = self.name.replace('/', '_')
-		self.unique_name = f'{self.unique_name}_{self.chunk}' if self.chunk else self.unique_name
-
-		# Process prior results
-		for result in results:
-			list(self._process_item(result, print=False, output=False))
-
 		# Input post-process
 		self.run_hooks('before_init')
 
@@ -238,6 +238,8 @@ class Runner:
 
 	@property
 	def status(self):
+		if not self.started:
+			return 'PENDING'
 		if not self.done:
 			return 'RUNNING'
 		return 'FAILURE' if len(self.self_errors) > 0 else 'SUCCESS'
@@ -326,16 +328,18 @@ class Runner:
 				self.add_result(error, print=True)
 				yield error
 
-	def add_result(self, item, print=False):
+	def add_result(self, item, print=False, output=True):
 		"""Add item to runner results.
 
 		Args:
 			item (OutputType): Item.
 			print (bool): Whether to print it or not.
+			output (bool): Whether to add it to the output or not.
 		"""
 		self.uuids.append(item._uuid)
 		self.results.append(item)
-		self.output += repr(item) + '\n'
+		if output:
+			self.output += repr(item) + '\n'
 		if print:
 			self._print_item(item)
 
@@ -640,6 +644,7 @@ class Runner:
 
 	def log_start(self):
 		"""Log runner start."""
+		self.started = True
 		if not self.print_remote_info:
 			return
 		remote_str = 'starting' if self.sync else 'sent to Celery worker'
@@ -831,14 +836,14 @@ class Runner:
 		# Update item context
 		item._context.update(self.context)
 
-		# Return if already seen
-		if item._uuid in self.uuids:
-			return
-
 		# Add uuid to item
 		if not item._uuid:
 			item._uuid = str(uuid.uuid4())
 
+		# Return if already seen
+		if item._uuid in self.uuids:
+			return
+
 		# Add source to item
 		if not item._source:
 			item._source = self.unique_name

secator/runners/command.py

@@ -193,11 +193,10 @@ class Command(Runner):
 		})
 		return res
 
-	@classmethod
-	def needs_chunking(cls, targets, sync):
-		many_targets = len(targets) > 1
-		targets_over_chunk_size = cls.input_chunk_size and len(targets) > cls.input_chunk_size
-		has_file_flag = cls.file_flag is not None
+	def needs_chunking(self, sync):
+		many_targets = len(self.inputs) > 1
+		targets_over_chunk_size = self.input_chunk_size and len(self.inputs) > self.input_chunk_size
+		has_file_flag = self.file_flag is not None
 		chunk_it = (sync and many_targets and not has_file_flag) or (not sync and many_targets and targets_over_chunk_size)
 		return chunk_it
 

secator/runners/scan.py

@@ -3,8 +3,8 @@ import logging
 from secator.template import TemplateLoader
 from secator.config import CONFIG
 from secator.runners._base import Runner
-from secator.runners._helpers import run_extractors
 from secator.runners.workflow import Workflow
+from secator.output_types import Error
 from secator.utils import merge_opts
 
 logger = logging.getLogger(__name__)
@@ -28,16 +28,16 @@ class Scan(Runner):
 		scan_opts = self.config.options
 		self.print_item = False
 		for name, workflow_opts in self.config.workflows.items():
-			# Run workflow
 			run_opts = self.run_opts.copy()
 			opts = merge_opts(scan_opts, workflow_opts, run_opts)
 			workflow = Workflow(
 				TemplateLoader(name=f'workflows/{name}'),
 				self.inputs,
-				results=[],
+				results=self.results,
 				run_opts=opts,
 				hooks=self._hooks,
 				context=self.context.copy())
-
-			# Get results
 			yield from workflow
+			if len(self.errors) > 0:
+				self.add_result(Error(message=f'Stopping scan since workflow {name} has errors'))
+				return

secator/scans/__init__.py

@@ -2,9 +2,9 @@ from secator.cli import ALL_SCANS
 
 
 def generate_class(config):
-	from secator.runners import Workflow
+	from secator.runners import Scan
 
-	class workflow(Workflow):
+	class scan(Scan):
 		def __init__(self, inputs=[], **run_opts):
 			hooks = run_opts.pop('hooks', {})
 			results = run_opts.pop('results', [])
@@ -16,12 +16,12 @@ def generate_class(config):
 				run_opts=run_opts,
 				hooks=hooks,
 				context=context)
-	return workflow, config.name
+	return scan, config.name
 
 
 DYNAMIC_SCANS = {}
-for workflow in ALL_SCANS:
-	cls, name = generate_class(workflow)
+for scan in ALL_SCANS:
+	cls, name = generate_class(scan)
 	DYNAMIC_SCANS[name] = cls
 
 globals().update(DYNAMIC_SCANS)

{secator-0.10.1a2.dist-info → secator-0.10.1a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: secator
-Version: 0.10.1a2
+Version: 0.10.1a3
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.10.1a2.dist-info → secator-0.10.1a3.dist-info}/RECORD

@@ -1,10 +1,10 @@
 secator/.gitignore,sha256=da8MUc3hdb6Mo0WjZu2upn5uZMbXcBGvhdhTQ1L89HI,3093
 secator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/celery.py,sha256=9KXKv4EamJYJrHt_Ppn7aIp1AiFaTn2V0J_tZBwtWK0,8802
-secator/celery_signals.py,sha256=HobT7hCbVKPEHvCNwxCvQxFVUyocU1kkrTXi67b1DDw,4346
-secator/celery_utils.py,sha256=UWqLZpUaOXcztC_GD6uEDLiP8bGmD3WiTQN-u3lialg,7712
+secator/celery.py,sha256=wEvhoBrnpkHuj_sFjW-NnWyPdvraBhgJ9ecF8gwlZ20,8926
+secator/celery_signals.py,sha256=iumfx7tTeoavAbHijBtij0JzeIqElxQldNZtuZmFY_U,4456
+secator/celery_utils.py,sha256=2Mw08To1DzWIlnW7TBeoX5suPMjm3XYbKT7NWPOV5nI,8346
 secator/cli.py,sha256=3_tTTusW12MCejFgtOeYjiedjrJpyQj_gsCK8FkTMJA,43922
-secator/config.py,sha256=xItKM29yvMqzNZZygSNZXZ2V9vJbTdRuLTfIoRfP3XE,19653
+secator/config.py,sha256=CdVBh6d4k13SpkQKyHQfMFHgkLypUH07kAKLmCJJO1w,19688
 secator/decorators.py,sha256=3kYadCz6haIZtnjkFHSRfenTdc6Yu7bHd-0IVjhD72w,13902
 secator/definitions.py,sha256=gFtLT9fjNtX_1qkiCjNfQyCvYq07IhScsQzX4o20_SE,3084
 secator/installer.py,sha256=Q5qmGbxGmuhysEA9YovTpy-YY2TxxFskhrzSX44c42E,17971
@@ -68,14 +68,14 @@ secator/output_types/user_account.py,sha256=rm10somxyu30JHjj629IkR15Nhahylud_fVO
 secator/output_types/vulnerability.py,sha256=nF7OT9zGez8sZvLrkhjBOORjVi8hCqfCYUFq3eZ_ywo,2870
 secator/output_types/warning.py,sha256=47GtmG083GqGPb_R5JDFmARJ9Mqrme58UxwJhgdGPuI,853
 secator/runners/__init__.py,sha256=EBbOk37vkBy9p8Hhrbi-2VtM_rTwQ3b-0ggTyiD22cE,290
-secator/runners/_base.py,sha256=T9gjOqe-UPDHe5ZdVRBtUtxTefRgDcq9JV08F6UV5ZU,29596
+secator/runners/_base.py,sha256=Luxr6-Nd0CtLjfVBNGNXsUeQdELETKdgdaAac8oqE5U,29801
 secator/runners/_helpers.py,sha256=QhJmdmFdu5XSx3LBFf4Q4Hy2EXS6bLGnJUq8G7C6f68,2410
 secator/runners/celery.py,sha256=bqvDTTdoHiGRCt0FRvlgFHQ_nsjKMP5P0PzGbwfCj_0,425
-secator/runners/command.py,sha256=PqCOHDKJXvG4weB8mXDTElGxc8i8pK2RoyTKUBpHASU,25480
-secator/runners/scan.py,sha256=Pab_o_liI5fhlv2OOwYNmonz5JFYYVqtQFf9eyAQpiE,1071
+secator/runners/command.py,sha256=9AvjZgSXctP8D-ffPCtlnXEiGqTeaD2wVGhiGNuROb0,25469
+secator/runners/scan.py,sha256=AJ7ucBIXXcdUz6pzpsLIdEj_tRRa2IlANhYRKOc2IZQ,1157
 secator/runners/task.py,sha256=f2AduWpIy8JHK-Qitl_2Kh0fia573_YHAyAlV6MsJ50,2068
 secator/runners/workflow.py,sha256=XEhBfL-f3vGH0HgEPnj62d8ITxjH_tPXiNSVkaonuwQ,3862
-secator/scans/__init__.py,sha256=nlNLiRl7Vu--c_iXClFFcagMd_b_OWKitq8tX1-1krQ,641
+secator/scans/__init__.py,sha256=1EEbngbDbvWxmeDYC6uux00WWy1v5qHtSpk6NVz27rM,617
 secator/serializers/__init__.py,sha256=OP5cmFl77ovgSCW_IDcZ21St2mUt5UK4QHfrsK2KvH8,248
 secator/serializers/dataclass.py,sha256=RqICpfsYWGjHAACAA2h2jZ_69CFHim4VZwcBqowGMcQ,1010
 secator/serializers/json.py,sha256=UJwAymRzjF-yBKOgz1MTOyBhQcdQg7fOKRXgmHIu8fo,411
@@ -109,8 +109,8 @@ secator/tasks/searchsploit.py,sha256=gvtLZbL2hzAZ07Cf0cSj2Qs0GvWK94XyHvoPFsetXu8
 secator/tasks/subfinder.py,sha256=C6W5NnXT92OUB1aSS9IYseqdI3wDMAz70TOEl8X-o3U,1213
 secator/tasks/wpscan.py,sha256=036ywiEqZfX_Bt071U7qIm7bi6pNk7vodflmuslJurA,5550
 secator/workflows/__init__.py,sha256=ivpZHiYYlj4JqlXLRmB9cmAPUGdk8QcUrCRL34hIqEA,665
-secator-0.10.1a2.dist-info/METADATA,sha256=YgZWsub4cdLtNiz-2UKkPKhLAi8mx0wFIPwGjnYU794,14726
-secator-0.10.1a2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-secator-0.10.1a2.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
-secator-0.10.1a2.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
-secator-0.10.1a2.dist-info/RECORD,,
+secator-0.10.1a3.dist-info/METADATA,sha256=G7F0k81GVKn7Y9a0mGQ3u96ZOHxxoshrlUxNj12dLao,14726
+secator-0.10.1a3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+secator-0.10.1a3.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
+secator-0.10.1a3.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
+secator-0.10.1a3.dist-info/RECORD,,