secator 0.10.0__py3-none-any.whl → 0.10.1a0__py3-none-any.whl

secator/celery.py

@@ -7,7 +7,7 @@ import uuid
 
 from time import time
 
-from celery import Celery, chain, chord
+from celery import Celery, chord
 from celery.app import trace
 
 from rich.logging import RichHandler
@@ -15,10 +15,9 @@ from retry import retry
 
 from secator.celery_signals import setup_handlers
 from secator.config import CONFIG
-from secator.output_types import Info, Error
+from secator.output_types import Info
 from secator.rich import console
 from secator.runners import Scan, Task, Workflow
-from secator.runners._helpers import run_extractors
 from secator.utils import (debug, deduplicate, flatten, should_update)
 
 IN_CELERY_WORKER_PROCESS = sys.argv and ('secator.celery.app' in sys.argv or 'worker' in sys.argv)
@@ -103,7 +102,7 @@ if IN_CELERY_WORKER_PROCESS:
 @retry(Exception, tries=3, delay=2)
 def update_state(celery_task, task, force=False):
 	"""Update task state to add metadata information."""
-	if task.sync:
+	if not IN_CELERY_WORKER_PROCESS:
 		return
 	if not force and not should_update(CONFIG.runners.backend_update_frequency, task.last_updated_celery):
 		return
@@ -139,6 +138,16 @@ def chunker(seq, size):
 	return (seq[pos:pos + size] for pos in range(0, len(seq), size))
 
 
+@app.task(bind=True)
+def handle_runner_error(self, results, runner):
+	"""Handle errors in Celery workflows (chunked tasks or runners)."""
+	results = forward_results(results)
+	runner.results = results
+	runner.log_results()
+	runner.run_hooks('on_end')
+	return runner.results
+
+
 def break_task(task, task_opts, targets, results=[], chunk_size=1):
 	"""Break a task into multiple of the same type."""
 	chunks = targets
@@ -167,7 +176,8 @@ def break_task(task, task_opts, targets, results=[], chunk_size=1):
 		task_id = str(uuid.uuid4())
 		opts['has_parent'] = True
 		opts['enable_duplicate_check'] = False
-		sig = type(task).s(chunk, **opts).set(queue=type(task).profile, task_id=task_id)
+		opts['results'] = results
+		sig = type(task).si(chunk, **opts).set(queue=type(task).profile, task_id=task_id)
 		full_name = f'{task.name}_{ix + 1}'
 		task.add_subtask(task_id, task.name, f'{task.name}_{ix + 1}')
 		info = Info(message=f'Celery chunked task created: {task_id}', _source=full_name, _uuid=str(uuid.uuid4()))
@@ -175,23 +185,16 @@ def break_task(task, task_opts, targets, results=[], chunk_size=1):
 		sigs.append(sig)
 
 	# Build Celery workflow
-	workflow = chain(
-		forward_results.s(results).set(queue='results'),
-		chord(
-			tuple(sigs),
-			forward_results.s().set(queue='results'),
-		)
+	workflow = chord(
+		tuple(sigs),
+		handle_runner_error.s(runner=task).set(queue='results')
 	)
-	if task.sync:
-		task.print_item = False
-		task.results = workflow.apply().get()
-	else:
-		result = workflow.apply_async()
-		task.celery_result = result
+	return workflow
 
 
 @app.task(bind=True)
 def run_task(self, args=[], kwargs={}):
+	print('run task')
 	console.print(Info(message=f'Running task {self.request.id}'))
 	kwargs['context']['celery_id'] = self.request.id
 	task = Task(*args, **kwargs)
@@ -218,111 +221,41 @@ def run_scan(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_command(self, results, name, targets, opts={}):
-	chunk = opts.get('chunk')
-	sync = opts.get('sync', True)
+	if IN_CELERY_WORKER_PROCESS:
+		opts.update({'print_item': True, 'print_line': True, 'print_cmd': True})
+		routing_key = self.request.delivery_info['routing_key']
+		console.print(Info(message=f'Task "{name}" running with routing key "{routing_key}"'))
+
+	# Flatten + dedupe + filter results
+	results = forward_results(results)
 
 	# Set Celery request id in context
 	context = opts.get('context', {})
 	context['celery_id'] = self.request.id
 	context['worker_name'] = os.environ.get('WORKER_NAME', 'unknown')
 	opts['context'] = context
-	opts['print_remote_info'] = False
 	opts['results'] = results
+	opts['sync'] = True
 
-	# If we are in a Celery worker, print everything, always
-	if IN_CELERY_WORKER_PROCESS:
-		opts.update({
-			'print_item': True,
-			'print_line': True,
-			'print_cmd': True
-		})
-		routing_key = self.request.delivery_info['routing_key']
-		console.print(Info(message=f'Task "{name}" running with routing key "{routing_key}"'))
+	# Initialize task
+	sync = not IN_CELERY_WORKER_PROCESS
+	task_cls = Task.get_task_class(name)
+	task = task_cls(targets, **opts)
+	update_state(self, task, force=True)
 
-	# Flatten + dedupe results
-	results = flatten(results)
-	results = deduplicate(results, attr='_uuid')
+	# Chunk task if needed
+	if task_cls.needs_chunking(targets, sync):
+		console.print(Info(message=f'Task {name} requires chunking, breaking into {len(targets)} tasks'))
+		return self.replace(break_task(task, opts, targets, results=results))
+
+	# Update state live
+	[update_state(self, task) for _ in task]
+	update_state(self, task, force=True)
+
+	# Garbage collection to save RAM
+	gc.collect()
 
-	# Get expanded targets
-	if not chunk and results:
-		targets, opts = run_extractors(results, opts, targets)
-		debug('after extractors', obj={'targets': targets, 'opts': opts}, sub='celery.state')
-
-	task = None
-
-	try:
-
-		# Get task class
-		task_cls = Task.get_task_class(name)
-
-		# Check if chunkable
-		many_targets = len(targets) > 1
-		targets_over_chunk_size = task_cls.input_chunk_size and len(targets) > task_cls.input_chunk_size
-		has_file_flag = task_cls.file_flag is not None
-		chunk_it = (sync and many_targets and not has_file_flag) or (not sync and many_targets and targets_over_chunk_size)
-		task_opts = opts.copy()
-		task_opts.update({
-			'print_remote_info': False,
-			'has_children': chunk_it,
-		})
-
-		if IN_CELERY_WORKER_PROCESS and chunk_it and routing_key != 'poll':
-			console.print(Info(message=f'Task {name} is chunkable but not running on "poll" queue, re-routing to "poll" queue'))
-			raise self.replace(run_command.si(results, name, targets, opts=opts).set(queue='poll', task_id=self.request.id))
-
-		if chunk_it:
-			task_opts['print_cmd'] = False
-
-		task = task_cls(targets, **task_opts)
-		debug(
-			'',
-			obj={
-				f'{task.unique_name}': 'CHUNK STATUS',
-				'chunk_it': chunk_it,
-				'sync': task.sync,
-				'many_targets': many_targets,
-				'targets_over_chunk_size': targets_over_chunk_size,
-			},
-			obj_after=False,
-			id=self.request.id,
-			sub='celery.state',
-			verbose=True
-		)
-
-		# Chunk task if needed
-		if chunk_it:
-			chunk_size = task_cls.input_chunk_size if has_file_flag else 1
-			break_task(
-				task,
-				opts,
-				targets,
-				results=results,
-				chunk_size=chunk_size)
-			console.print(Info(message=f'Task "{name}" starts polling for chunked results'))
-
-		# Update state before starting
-		update_state(self, task)
-
-		# Update state for each item found
-		for _ in task:
-			update_state(self, task)
-
-	except BaseException as e:
-		if not task:
-			raise e
-		error = Error.from_exception(e)
-		error._source = task.unique_name
-		error._uuid = str(uuid.uuid4())
-		task.add_result(error, print=True)
-		task.stop_celery_tasks()
-
-	finally:
-		if not task:
-			raise
-		update_state(self, task, force=True)
-		gc.collect()
-		debug('', obj={task.unique_name: task.status, 'results': task.results}, sub='celery.results', verbose=True)
-		return task.results
+	return task.results
 
 
 @app.task
@@ -335,6 +268,7 @@ def forward_results(results):
 		results = results['results']
 	results = flatten(results)
 	results = deduplicate(results, attr='_uuid')
+	console.print(Info(message=f'Forwarding {len(results)} results ...'))
 	return results
 
 #--------------#

secator/celery_signals.py

@@ -1,6 +1,7 @@
 import os
 import signal
 import threading
+from pathlib import Path
 
 from celery import signals
 
@@ -8,57 +9,75 @@ from secator.config import CONFIG
 from secator.output_types import Info
 from secator.rich import console
 
-
 IDLE_TIMEOUT = CONFIG.celery.worker_kill_after_idle_seconds
-TASK_IN_PROGRESS = False
 
+# File-based state management system
+STATE_DIR = Path("/tmp/celery_state")
+STATE_DIR.mkdir(exist_ok=True, parents=True)
+
+
+def get_lock_file_path():
+    worker_name = os.environ.get("WORKER_NAME", f"unknown_{os.getpid()}")
+    return Path(f"/tmp/celery_worker_{worker_name}.lock")
+
+
+def set_task_running(task_id):
+    """Mark that a task is running in current worker"""
+    with open(get_lock_file_path(), "w") as f:
+        f.write(task_id)
+
+
+def clear_task_running():
+    """Clear the task running state"""
+    lock_file = get_lock_file_path()
+    if lock_file.exists():
+        lock_file.unlink()
 
-def kill_worker():
-	""""Kill current worker using it's pid by sending a SIGTERM to Celery master process."""
-	worker_name = os.environ['WORKER_NAME']
-	if not TASK_IN_PROGRESS:
-		pid = os.getpid()
-		console.print(Info(message=f'Sending SIGTERM to worker {worker_name} with pid {pid}'))
-		os.kill(pid, signal.SIGTERM)
-	else:
-		console.print(Info(message=f'Cancelling worker shutdown of {worker_name} since a task is currently in progress'))
 
+def is_task_running():
+    """Check if a task is currently running"""
+    return get_lock_file_path().exists()
 
-class IdleTimer:
-	def __init__(self, timeout):
-		self.thread = None
-		self.is_started = False
-		self.thread = threading.Timer(timeout, kill_worker)
 
-	def start(self):
-		if self.is_started:
-			self.cancel()
-		self.thread.start()
-		self.is_started = True
+def kill_worker(parent=False):
+    """Kill current worker using its pid by sending a SIGTERM to Celery master process."""
+    worker_name = os.environ.get('WORKER_NAME', 'unknown')
 
-	def cancel(self):
-		self.thread.cancel()
-		self.s_started = False
+    # Check if a task is running via the lock file
+    if not is_task_running():
+        pid = os.getppid() if parent else os.getpid()
+        console.print(Info(message=f'Sending SIGTERM to worker {worker_name} with pid {pid}'))
+        os.kill(pid, signal.SIGTERM)
+    else:
+        console.print(Info(message=f'Cancelling worker shutdown of {worker_name} since a task is running'))
 
 
-IDLE_TIMER = IdleTimer(IDLE_TIMEOUT)
+def setup_idle_timer(timeout):
+    """Setup a timer to kill the worker after being idle"""
+    if timeout == -1:
+        return
+
+    console.print(Info(message=f'Starting inactivity timer for {timeout} seconds ...'))
+    timer = threading.Timer(timeout, kill_worker)
+    timer.daemon = True  # Make sure timer is killed when worker exits
+    timer.start()
 
 
 def maybe_override_logging():
-	def decorator(func):
-		if CONFIG.celery.override_default_logging:
-			return signals.setup_logging.connect(func)
-		else:
-			return func
-	return decorator
+    def decorator(func):
+        if CONFIG.celery.override_default_logging:
+            return signals.setup_logging.connect(func)
+        else:
+            return func
+    return decorator
 
 
 @maybe_override_logging()
 def setup_logging(*args, **kwargs):
-	"""Override celery's logging setup to prevent it from altering our settings.
-	github.com/celery/celery/issues/1867
-	"""
-	pass
+    """Override celery's logging setup to prevent it from altering our settings.
+    github.com/celery/celery/issues/1867
+    """
+    pass
 
 
 def capture_worker_name(sender, instance, **kwargs):
@@ -66,38 +85,57 @@ def capture_worker_name(sender, instance, **kwargs):
 
 
 def worker_init_handler(**kwargs):
-	if IDLE_TIMEOUT != -1:
-		console.print(Info(message=f'Starting inactivity timer for {IDLE_TIMEOUT} seconds ...'))
-		IDLE_TIMER.start()
+    if IDLE_TIMEOUT != -1:
+        setup_idle_timer(IDLE_TIMEOUT)
 
 
-def task_prerun_handler(**kwargs):
-	global TASK_IN_PROGRESS, IDLE_TIMER
-	TASK_IN_PROGRESS = True
-	if IDLE_TIMEOUT != -1:
-		IDLE_TIMER.cancel()
+def task_prerun_handler(task_id, **kwargs):
+    # Mark that a task is running
+    set_task_running(task_id)
 
 
 def task_postrun_handler(**kwargs):
-	global TASK_IN_PROGRESS, IDLE_TIMER
-	TASK_IN_PROGRESS = False
-	sender_name = kwargs['sender'].name
+    # Mark that no task is running
+    clear_task_running()
+
+    # Get sender name from kwargs
+    sender_name = kwargs['sender'].name
+
+    if CONFIG.celery.worker_kill_after_task and sender_name.startswith('secator.'):
+        worker_name = os.environ.get('WORKER_NAME', 'unknown')
+        console.print(Info(message=f'Shutdown worker {worker_name} since config celery.worker_kill_after_task is set.'))
+        kill_worker(parent=True)
+        return
+
+    # Set up a new idle timer
+    if IDLE_TIMEOUT != -1:
+        console.print(Info(message=f'Reset inactivity timer to {IDLE_TIMEOUT} seconds'))
+        setup_idle_timer(IDLE_TIMEOUT)
+
+
+def task_revoked_handler(request=None, **kwargs):
+    """Handle revoked tasks by clearing the task running state"""
+    console.print(Info(message='Task was revoked, clearing running state'))
+    clear_task_running()
+
+    # Set up a new idle timer
+    if IDLE_TIMEOUT != -1:
+        console.print(Info(message=f'Reset inactivity timer to {IDLE_TIMEOUT} seconds after task revocation'))
+        setup_idle_timer(IDLE_TIMEOUT)
 
-	if CONFIG.celery.worker_kill_after_task and sender_name.startswith('secator.'):
-		worker_name = os.environ['WORKER_NAME']
-		console.print(Info(message=f'Shutdown worker {worker_name} since config celery.worker_kill_after_task is set.'))
-		IDLE_TIMER.cancel()
-		kill_worker()
-		return
 
-	if IDLE_TIMEOUT != -1:  # restart timer
-		console.print(Info(message=f'Reset inactivity timer to {IDLE_TIMEOUT} seconds'))
-		IDLE_TIMER.start()
+def worker_shutdown_handler(**kwargs):
+    """Cleanup lock files when worker shuts down"""
+    lock_file = get_lock_file_path()
+    if lock_file.exists():
+        lock_file.unlink()
 
 
 def setup_handlers():
-	signals.celeryd_after_setup.connect(capture_worker_name)
-	signals.setup_logging.connect(setup_logging)
-	signals.task_prerun.connect(task_prerun_handler)
-	signals.task_postrun.connect(task_postrun_handler)
-	signals.worker_ready.connect(worker_init_handler)
+    signals.celeryd_after_setup.connect(capture_worker_name)
+    signals.setup_logging.connect(setup_logging)
+    signals.task_prerun.connect(task_prerun_handler)
+    signals.task_postrun.connect(task_postrun_handler)
+    signals.task_revoked.connect(task_revoked_handler)
+    signals.worker_ready.connect(worker_init_handler)
+    signals.worker_shutdown.connect(worker_shutdown_handler)

secator/celery_utils.py

@@ -264,5 +264,5 @@ class CeleryData(object):
 				CeleryData.get_task_ids(result.parent, ids=ids)
 
 		except kombu.exceptions.DecodeError:
-			debug('kombu decode error', sub='celery.data.get_task_ids')
+			debug('kombu decode error', sub='celery.data')
 			return

secator/cli.py

@@ -171,7 +171,8 @@ def worker(hostname, concurrency, reload, queue, pool, check, dev, stop, show):
 		patterns = "celery.py;tasks/*.py;runners/*.py;serializers/*.py;output_types/*.py;hooks/*.py;exporters/*.py"
 		cmd = f'watchmedo auto-restart --directory=./ --patterns="{patterns}" --recursive -- {cmd}'
 
-	Command.execute(cmd, name='secator_worker')
+	ret = Command.execute(cmd, name='secator_worker')
+	sys.exit(ret.return_code)
 
 
 #-------#

secator/hooks/mongodb.py

@@ -20,11 +20,19 @@ MONGODB_MAX_POOL_SIZE = CONFIG.addons.mongodb.max_pool_size
 
 logger = logging.getLogger(__name__)
 
-client = pymongo.MongoClient(
-	escape_mongodb_url(MONGODB_URL),
-	maxPoolSize=MONGODB_MAX_POOL_SIZE,
-	serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT
-)
+_mongodb_client = None
+
+
+def get_mongodb_client():
+	"""Get or create MongoDB client"""
+	global _mongodb_client
+	if _mongodb_client is None:
+		_mongodb_client = pymongo.MongoClient(
+			escape_mongodb_url(MONGODB_URL),
+			maxPoolSize=MONGODB_MAX_POOL_SIZE,
+			serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT
+		)
+	return _mongodb_client
 
 
 def get_runner_dbg(runner):
@@ -39,6 +47,7 @@ def get_runner_dbg(runner):
 
 
 def update_runner(self):
+	client = get_mongodb_client()
 	db = client.main
 	type = self.config.type
 	collection = f'{type}s'
@@ -72,6 +81,7 @@ def update_finding(self, item):
 	if type(item) not in FINDING_TYPES:
 		return item
 	start_time = time.time()
+	client = get_mongodb_client()
 	db = client.main
 	update = item.toDict()
 	_type = item._type
@@ -97,15 +107,14 @@ def update_finding(self, item):
 
 
 def find_duplicates(self):
+	from secator.celery import IN_CELERY_WORKER_PROCESS
 	ws_id = self.toDict().get('context', {}).get('workspace_id')
 	if not ws_id:
 		return
-	if self.sync:
-		debug(f'running duplicate check on workspace {ws_id}', sub='hooks.mongodb')
+	if not IN_CELERY_WORKER_PROCESS:
 		tag_duplicates(ws_id)
 	else:
-		celery_id = tag_duplicates.delay(ws_id)
-		debug(f'running duplicate check on workspace {ws_id}', id=celery_id, sub='hooks.mongodb')
+		tag_duplicates.delay(ws_id)
 
 
 def load_finding(obj):
@@ -132,6 +141,8 @@ def tag_duplicates(ws_id: str = None):
 	Args:
 		ws_id (str): Workspace id.
 	"""
+	debug(f'running duplicate check on workspace {ws_id}', sub='hooks.mongodb')
+	client = get_mongodb_client()
 	db = client.main
 	workspace_query = list(
 		db.findings.find({'_context.workspace_id': str(ws_id), '_tagged': True}).sort('_timestamp', -1))
@@ -172,19 +183,19 @@ def tag_duplicates(ws_id: str = None):
 				'seen dupes': len(seen_dupes)
 			},
 			id=ws_id,
-			sub='hooks.mongodb.duplicates',
+			sub='hooks.mongodb',
 			verbose=True)
 		tmp_duplicates_ids = list(dict.fromkeys([i._uuid for i in tmp_duplicates]))
-		debug(f'duplicate ids: {tmp_duplicates_ids}', id=ws_id, sub='hooks.mongodb.duplicates', verbose=True)
+		debug(f'duplicate ids: {tmp_duplicates_ids}', id=ws_id, sub='hooks.mongodb', verbose=True)
 
 		# Update latest object as non-duplicate
 		if tmp_duplicates:
 			duplicates.extend([f for f in tmp_duplicates])
 			db.findings.update_one({'_id': ObjectId(item._uuid)}, {'$set': {'_related': tmp_duplicates_ids}})
-			debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb.duplicates', verbose=True)
+			debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb', verbose=True)
 			non_duplicates.append(item)
 		else:
-			debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb.duplicates', verbose=True)
+			debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb', verbose=True)
 			non_duplicates.append(item)
 
 	# debug(f'found {len(duplicates)} total duplicates')
@@ -208,7 +219,7 @@ def tag_duplicates(ws_id: str = None):
 			'duplicates': len(duplicates_ids),
 			'non-duplicates': len(non_duplicates_ids)
 		},
-		sub='hooks.mongodb.duplicates')
+		sub='hooks.mongodb')
 
 
 HOOKS = {
@@ -232,6 +243,6 @@ HOOKS = {
 		'on_item': [update_finding],
 		'on_duplicate': [update_finding],
 		'on_interval': [update_runner],
-		'on_end': [update_runner, find_duplicates]
+		'on_end': [update_runner]
 	}
 }

secator/output_types/error.py

@@ -21,10 +21,13 @@ class Error(OutputType):
 	_sort_by = ('_timestamp',)
 
 	def from_exception(e, **kwargs):
-		message = type(e).__name__
+		errtype = type(e).__name__
+		message = errtype
 		if str(e):
 			message += f': {str(e)}'
-		return Error(message=message, traceback=traceback_as_string(e), **kwargs)
+		traceback = traceback_as_string(e) if errtype not in ['KeyboardInterrupt', 'GreenletExit'] else ''
+		error = Error(message=message, traceback=traceback, **kwargs)
+		return error
 
 	def __str__(self):
 		return self.message

secator/runners/_base.py

@@ -15,7 +15,7 @@ from secator.config import CONFIG
 from secator.output_types import FINDING_TYPES, OutputType, Progress, Info, Warning, Error, Target
 from secator.report import Report
 from secator.rich import console, console_stdout
-from secator.runners._helpers import (get_task_folder_id, process_extractor)
+from secator.runners._helpers import (get_task_folder_id, process_extractor, run_extractors)
 from secator.utils import (debug, import_dynamic, merge_opts, rich_to_ansi, should_update)
 
 logger = logging.getLogger(__name__)
@@ -69,15 +69,14 @@ class Runner:
 	reports_folder = None
 
 	def __init__(self, config, inputs=[], results=[], run_opts={}, hooks={}, validators={}, context={}):
+		self.uuids = []
+		self.results = []
+		self.output = ''
+
+		# Runner config
 		self.config = config
 		self.name = run_opts.get('name', config.name)
 		self.description = run_opts.get('description', config.description)
-		if not isinstance(inputs, list):
-			inputs = [inputs]
-		self.inputs = inputs
-		self.uuids = []
-		self.output = ''
-		self.results = []
 		self.workspace_name = context.get('workspace_name', 'default')
 		self.run_opts = run_opts.copy()
 		self.sync = run_opts.get('sync', True)
@@ -97,6 +96,39 @@ class Runner:
 		self.caller = self.run_opts.get('caller', None)
 		self.threads = []
 		self.no_poll = self.run_opts.get('no_poll', False)
+		self.quiet = self.run_opts.get('quiet', False)
+
+		# Runner process options
+		self.no_process = self.run_opts.get('no_process', False)
+		self.piped_input = self.run_opts.get('piped_input', False)
+		self.piped_output = self.run_opts.get('piped_output', False)
+		self.enable_duplicate_check = self.run_opts.get('enable_duplicate_check', True)
+
+		# Runner print opts
+		self.print_item = self.run_opts.get('print_item', False)
+		self.print_line = self.run_opts.get('print_line', False) and not self.quiet
+		self.print_remote_info = self.run_opts.get('print_remote_info', False) and not self.piped_input and not self.piped_output  # noqa: E501
+		self.print_json = self.run_opts.get('print_json', False)
+		self.print_raw = self.run_opts.get('print_raw', False) or self.piped_output
+		self.print_fmt = self.run_opts.get('fmt', '')
+		self.print_progress = self.run_opts.get('print_progress', False) and not self.quiet and not self.print_raw
+		self.print_target = self.run_opts.get('print_target', False) and not self.quiet and not self.print_raw
+		self.print_stat = self.run_opts.get('print_stat', False) and not self.quiet and not self.print_raw
+		self.raise_on_error = self.run_opts.get('raise_on_error', False)
+		self.print_opts = {k: v for k, v in self.__dict__.items() if k.startswith('print_') if v}
+
+		# Determine inputs
+		inputs = [inputs] if not isinstance(inputs, list) else inputs
+		if results:
+			inputs, run_opts, errors = run_extractors(results, run_opts, inputs)
+			for error in errors:
+				self.add_result(error, print=True)
+		self.inputs = inputs
+
+		# Debug
+		self.debug('Inputs', obj=self.inputs, sub='init')
+		self.debug('Run opts', obj={k: v for k, v in self.run_opts.items() if v is not None}, sub='init')
+		self.debug('Print opts', obj={k: v for k, v in self.print_opts.items() if v is not None}, sub='init')
 
 		# Determine exporters
 		exporters_str = self.run_opts.get('output') or self.default_exporters
@@ -123,31 +155,6 @@ class Runner:
 				self.enable_profiler = False
 				pass
 
-		# Process opts
-		self.quiet = self.run_opts.get('quiet', False)
-		self.no_process = self.run_opts.get('no_process', False)
-		self.piped_input = self.run_opts.get('piped_input', False)
-		self.piped_output = self.run_opts.get('piped_output', False)
-		self.enable_duplicate_check = self.run_opts.get('enable_duplicate_check', True)
-
-		# Print opts
-		self.print_item = self.run_opts.get('print_item', False)
-		self.print_line = self.run_opts.get('print_line', False) and not self.quiet
-		self.print_remote_info = self.run_opts.get('print_remote_info', False) and not self.piped_input and not self.piped_output  # noqa: E501
-		self.print_json = self.run_opts.get('print_json', False)
-		self.print_raw = self.run_opts.get('print_raw', False) or self.piped_output
-		self.print_fmt = self.run_opts.get('fmt', '')
-		self.print_progress = self.run_opts.get('print_progress', False) and not self.quiet and not self.print_raw
-		self.print_target = self.run_opts.get('print_target', False) and not self.quiet and not self.print_raw
-		self.print_stat = self.run_opts.get('print_stat', False) and not self.quiet and not self.print_raw
-		self.raise_on_error = self.run_opts.get('raise_on_error', False)
-		self.print_opts = {k: v for k, v in self.__dict__.items() if k.startswith('print_') if v}
-
-		# Debug
-		self.debug('Inputs', obj=self.inputs, sub='init')
-		self.debug('Run opts', obj={k: v for k, v in self.run_opts.items() if v is not None}, sub='init')
-		self.debug('Print opts', obj={k: v for k, v in self.print_opts.items() if v is not None}, sub='init')
-
 		# Hooks
 		self.hooks = {name: [] for name in HOOKS + getattr(self, 'hooks', [])}
 		self.register_hooks(hooks)
@@ -166,7 +173,7 @@ class Runner:
 
 		# Process prior results
 		for result in results:
-			list(self._process_item(result, print=False))
+			list(self._process_item(result, print=False, output=False))
 
 		# Input post-process
 		self.run_hooks('before_init')
@@ -219,6 +226,12 @@ class Runner:
 	def self_findings(self):
 		return [r for r in self.results if isinstance(r, tuple(FINDING_TYPES)) if r._source.startswith(self.unique_name)]
 
+	@property
+	def self_errors(self):
+		if self.config.type == 'task':
+			return [r for r in self.results if isinstance(r, Error) and r._source.startswith(self.unique_name)]
+		return [r for r in self.results if isinstance(r, Error)]
+
 	@property
 	def self_findings_count(self):
 		return len(self.self_findings)
@@ -227,7 +240,7 @@ class Runner:
 	def status(self):
 		if not self.done:
 			return 'RUNNING'
-		return 'FAILURE' if len(self.errors) > 0 else 'SUCCESS'
+		return 'FAILURE' if len(self.self_errors) > 0 else 'SUCCESS'
 
 	@property
 	def celery_state(self):
@@ -525,7 +538,7 @@ class Runner:
 			fun = self.get_func_path(hook)
 			try:
 				if hook_type == 'on_interval' and not should_update(CONFIG.runners.backend_update_frequency, self.last_updated_db):
-					self.debug('', obj={f'{name} [dim yellow]->[/] {fun}': '[dim gray11]skipped[/]'}, id=_id, sub='hooks.db', verbose=True)  # noqa: E501
+					self.debug('', obj={f'{name} [dim yellow]->[/] {fun}': '[dim gray11]skipped[/]'}, id=_id, sub='hooks', verbose=True)  # noqa: E501
 					return
 				if not self.enable_hooks or self.no_process:
 					self.debug('', obj={f'{name} [dim yellow]->[/] {fun}': '[dim gray11]skipped[/]'}, id=_id, sub='hooks', verbose=True)  # noqa: E501
@@ -783,19 +796,20 @@ class Runner:
 				count_map[name] = count
 		return count_map
 
-	def _process_item(self, item, print=True):
+	def _process_item(self, item, print=True, output=True):
 		"""Process an item yielded by the derived runner.
 
 		Args:
 			item (dict | str): Input item.
 			print (bool): Print item in console.
+			output (bool): Add to runner output.
 
 		Yields:
 			OutputType: Output type.
 		"""
 		# Item is a string, just print it
 		if isinstance(item, str):
-			self.output += item + '\n'
+			self.output += item + '\n' if output else ''
 			self._print_item(item) if item and print else ''
 			return
 

secator/runners/_helpers.py

@@ -1,6 +1,7 @@
 import os
 
-from secator.utils import deduplicate
+from secator.output_types import Error
+from secator.utils import deduplicate, debug
 
 
 def run_extractors(results, opts, inputs=[]):
@@ -12,17 +13,19 @@ def run_extractors(results, opts, inputs=[]):
 		inputs (list): Original inputs.
 
 	Returns:
-		tuple: inputs, options.
+		tuple: inputs, options, errors.
 	"""
 	extractors = {k: v for k, v in opts.items() if k.endswith('_')}
+	errors = []
 	for key, val in extractors.items():
 		key = key.rstrip('_')
-		values = extract_from_results(results, val)
+		values, err = extract_from_results(results, val)
+		errors.extend(err)
 		if key == 'targets':
 			inputs = deduplicate(values)
 		else:
 			opts[key] = deduplicate(values)
-	return inputs, opts
+	return inputs, opts, errors
 
 
 def extract_from_results(results, extractors):
@@ -33,14 +36,19 @@ def extract_from_results(results, extractors):
 		extractors (list): List of extractors to extract from.
 
 	Returns:
-		list: List of extracted results (flat).
+		tuple: List of extracted results (flat), list of errors.
 	"""
-	extracted = []
+	extracted_results = []
+	errors = []
 	if not isinstance(extractors, list):
 		extractors = [extractors]
 	for extractor in extractors:
-		extracted.extend(process_extractor(results, extractor))
-	return extracted
+		try:
+			extracted_results.extend(process_extractor(results, extractor))
+		except Exception as e:
+			error = Error.from_exception(e)
+			errors.append(error)
+	return extracted_results, errors
 
 
 def process_extractor(results, extractor, ctx={}):
@@ -53,6 +61,7 @@ def process_extractor(results, extractor, ctx={}):
 	Returns:
 		list: List of extracted results.
 	"""
+	debug('before extract', obj={'results': results, 'extractor': extractor}, sub='extractor')
 	if isinstance(extractor, dict):
 		_type = extractor['type']
 		_field = extractor.get('field')
@@ -66,6 +75,7 @@ def process_extractor(results, extractor, ctx={}):
 	if _field:
 		_field = '{' + _field + '}' if not _field.startswith('{') else _field
 		items = [_field.format(**item.toDict()) for item in items]
+	debug('after extract', obj={'items': items}, sub='extractor')
 	return items
 
 

secator/runners/command.py

@@ -193,6 +193,14 @@ class Command(Runner):
 		})
 		return res
 
+	@classmethod
+	def needs_chunking(cls, targets, sync):
+		many_targets = len(targets) > 1
+		targets_over_chunk_size = cls.input_chunk_size and len(targets) > cls.input_chunk_size
+		has_file_flag = cls.file_flag is not None
+		chunk_it = (sync and many_targets and not has_file_flag) or (not sync and many_targets and targets_over_chunk_size)
+		return chunk_it
+
 	@classmethod
 	def delay(cls, *args, **kwargs):
 		# TODO: Move this to TaskBase
@@ -858,6 +866,8 @@ class Command(Runner):
 				cmd = f'cat {fpath} | {cmd}'
 			elif self.file_flag:
 				cmd += f' {self.file_flag} {fpath}'
+			else:
+				cmd += f' {fpath}'
 
 			self.inputs_path = fpath
 

secator/tasks/fping.py

@@ -13,7 +13,6 @@ class fping(ReconIp):
 	cmd = 'fping -a'
 	file_flag = '-f'
 	input_flag = None
-	ignore_return_code = True
 	opt_prefix = '--'
 	opt_key_map = {
 		DELAY: 'period',

secator/tasks/gospider.py

@@ -55,7 +55,6 @@ class gospider(HttpCrawler):
 	}
 	install_cmd = 'go install -v github.com/jaeles-project/gospider@latest'
 	install_github_handle = 'jaeles-project/gospider'
-	ignore_return_code = True
 	proxychains = False
 	proxy_socks5 = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61
 	proxy_http = True  # with leaks... https://github.com/jaeles-project/gospider/issues/61

secator/tasks/naabu.py

@@ -47,8 +47,8 @@ class naabu(ReconPort):
 		}
 	}
 	output_types = [Port]
-	install_cmd = 'go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest'
-	install_github_handle = 'projectdiscovery/naabu'
+	install_cmd = 'go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@v2.3.3'
+	# install_github_handle = 'projectdiscovery/naabu'
 	install_pre = {'apt': ['libpcap-dev'], 'apk': ['libpcap-dev', 'libc6-compat'], 'pacman|brew': ['libpcap']}
 	install_post = {'arch|alpine': 'sudo ln -sf /usr/lib/libpcap.so /usr/lib/libpcap.so.0.8'}
 	proxychains = False

secator/tasks/nuclei.py

@@ -73,7 +73,6 @@ class nuclei(VulnMulti):
 			EXTRA_DATA: lambda x: {k: v for k, v in x.items() if k not in ['duration', 'errors', 'percent']}
 		}
 	}
-	ignore_return_code = True
 	install_pre = {
 		'*': ['git']
 	}

secator/tasks/wpscan.py

@@ -82,7 +82,6 @@ class wpscan(VulnHttp):
 	proxychains = False
 	proxy_http = True
 	proxy_socks5 = False
-	ignore_return_code = True
 	profile = 'io'
 
 	@staticmethod

secator/utils.py

@@ -772,6 +772,8 @@ def process_wordlist(val):
 	template_wordlist = getattr(CONFIG.wordlists.templates, val)
 	if template_wordlist:
 		return template_wordlist
+	elif Path(val).exists():
+		return val
 	else:
 		return download_file(
 			val,

{secator-0.10.0.dist-info → secator-0.10.1a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: secator
-Version: 0.10.0
+Version: 0.10.1a0
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.10.0.dist-info → secator-0.10.1a0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 secator/.gitignore,sha256=da8MUc3hdb6Mo0WjZu2upn5uZMbXcBGvhdhTQ1L89HI,3093
 secator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-secator/celery.py,sha256=cc7swFNPAuYMGRdL55Drwyo5RO4sK1l9UxDO-ojLM4Y,10518
-secator/celery_signals.py,sha256=WG9d41CoRIPjHpauIiAE41ekI7j7DPQ01uWU6RHtcL0,2719
-secator/celery_utils.py,sha256=iIuCn_3YkPXCtpnbaYqpppU2TARzSDyTIYHkrRyt54s,7725
-secator/cli.py,sha256=b-Oo_fACToy3pGfIo2Bzci_6rcWR4fONfOP01vnfVt4,43889
+secator/celery.py,sha256=9KXKv4EamJYJrHt_Ppn7aIp1AiFaTn2V0J_tZBwtWK0,8802
+secator/celery_signals.py,sha256=HobT7hCbVKPEHvCNwxCvQxFVUyocU1kkrTXi67b1DDw,4346
+secator/celery_utils.py,sha256=UWqLZpUaOXcztC_GD6uEDLiP8bGmD3WiTQN-u3lialg,7712
+secator/cli.py,sha256=3_tTTusW12MCejFgtOeYjiedjrJpyQj_gsCK8FkTMJA,43922
 secator/config.py,sha256=xItKM29yvMqzNZZygSNZXZ2V9vJbTdRuLTfIoRfP3XE,19653
 secator/decorators.py,sha256=3kYadCz6haIZtnjkFHSRfenTdc6Yu7bHd-0IVjhD72w,13902
 secator/definitions.py,sha256=gFtLT9fjNtX_1qkiCjNfQyCvYq07IhScsQzX4o20_SE,3084
@@ -12,7 +12,7 @@ secator/report.py,sha256=qJkEdCFttDBXIwUNUzZqFU_sG8l0PvyTSTogZVBv1Rs,3628
 secator/rich.py,sha256=owmuLcTTUt8xYBTE3_SqWTkPeAomcU_8bPdW_V-U8VM,3264
 secator/template.py,sha256=Sb6PjCTGIkZ7I0OGWFp5CaXmjt-6VPe_xpcRhWhjGpU,4409
 secator/thread.py,sha256=rgRgEtcMgs2wyfLWVlCTUCLWeg6jsMo5iKpyyrON5rY,655
-secator/utils.py,sha256=FBDa0BWPFLDLXKD_3FwFd8Bmz-fP0inKX8kG8LoivjU,21748
+secator/utils.py,sha256=zlG3-f3KEN9DdiT5kCqHhIASdEudYDgSYPkB76DTLLk,21787
 secator/utils_test.py,sha256=ArHwkWW89t0IDqxO4HjJWd_tm7tp1illP4pu3nLq5yo,6559
 secator/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 secator/configs/profiles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -49,10 +49,10 @@ secator/exporters/table.py,sha256=RY7Tct5kowEx8Oza8QMXFx6fKBARYfLxEbbvjKiE3eQ,11
 secator/exporters/txt.py,sha256=oMtr22di6cqyE_5yJoiWP-KElrI5QgvK1cOUrj7H7js,730
 secator/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 secator/hooks/gcs.py,sha256=MIhntyWYz9BZdTXhWl5JznaczSq1_7fl3TVqPufuTSo,1490
-secator/hooks/mongodb.py,sha256=HyjtpJSoxvSZ6aG8uBf1RFLNKGXCBDQL5eEh4xzDonA,7545
+secator/hooks/mongodb.py,sha256=XKbm_SrcSbQ2koILWvhzSg4tqdvHXgX5aU5x46Edu1s,7716
 secator/output_types/__init__.py,sha256=LxCW0K1f2vdgUapc4pIEsUpBfC0TQVvqo7T57rGuZGk,1159
 secator/output_types/_base.py,sha256=OgS6ICt66TzPsqo1JZwRIIwbng2HRX1i_u5qbUECgNk,2820
-secator/output_types/error.py,sha256=QjiJ5RoN3-utHqAyvgL2jlmZp7-u7emgUQpvLpYammU,1405
+secator/output_types/error.py,sha256=39gpEJfKM2EuyOhD9lSkjjna2QicMvnLdFav6kHmhlg,1529
 secator/output_types/exploit.py,sha256=-BKTqPBg94rVgjw8YSmcYuBCI2x-73WwMd9ITP9qr3Y,1750
 secator/output_types/info.py,sha256=R8xeiD3ocNOsvkJPhrQgsx6q-Ea1G0eTAqyuh5JrAR0,843
 secator/output_types/ip.py,sha256=CyE3qkp55Kmj5YRl0CZGS4XrHX8N5apWrLN3OMzaK0U,1127
@@ -68,10 +68,10 @@ secator/output_types/user_account.py,sha256=rm10somxyu30JHjj629IkR15Nhahylud_fVO
 secator/output_types/vulnerability.py,sha256=nF7OT9zGez8sZvLrkhjBOORjVi8hCqfCYUFq3eZ_ywo,2870
 secator/output_types/warning.py,sha256=47GtmG083GqGPb_R5JDFmARJ9Mqrme58UxwJhgdGPuI,853
 secator/runners/__init__.py,sha256=EBbOk37vkBy9p8Hhrbi-2VtM_rTwQ3b-0ggTyiD22cE,290
-secator/runners/_base.py,sha256=QBYyrYCPMJz0YPP6lE8vkgIHmDLplO6byrsisqVjV5g,29047
-secator/runners/_helpers.py,sha256=FGogmmdHfCWmIyq7wRprwU1oOSxesOu3Y0N4GyAgiGw,2000
+secator/runners/_base.py,sha256=T9gjOqe-UPDHe5ZdVRBtUtxTefRgDcq9JV08F6UV5ZU,29596
+secator/runners/_helpers.py,sha256=QhJmdmFdu5XSx3LBFf4Q4Hy2EXS6bLGnJUq8G7C6f68,2410
 secator/runners/celery.py,sha256=bqvDTTdoHiGRCt0FRvlgFHQ_nsjKMP5P0PzGbwfCj_0,425
-secator/runners/command.py,sha256=x7ktQLwIy7CWV-AEL6n5xY2sRzAijGCURuB17hJWRpY,25089
+secator/runners/command.py,sha256=PqCOHDKJXvG4weB8mXDTElGxc8i8pK2RoyTKUBpHASU,25480
 secator/runners/scan.py,sha256=tuPuqwL6fIS4UbCoy5WPKthYWm_LL-vCPRD2qK58HZE,1232
 secator/runners/task.py,sha256=f2AduWpIy8JHK-Qitl_2Kh0fia573_YHAyAlV6MsJ50,2068
 secator/runners/workflow.py,sha256=XEhBfL-f3vGH0HgEPnj62d8ITxjH_tPXiNSVkaonuwQ,3862
@@ -91,10 +91,10 @@ secator/tasks/dnsx.py,sha256=nK14_DeyX0QTpAMdIP0LSSEOEu5_tQemyFW0XPjA2f8,2266
 secator/tasks/dnsxbrute.py,sha256=5VnSpd5ken7vWxFX1bcsGcUN8LpaVhcjafnuETzkMGs,1422
 secator/tasks/feroxbuster.py,sha256=3bKolPIdDBhdJ2fu4BP3w1cOlxDyI8WmtM-_2pDQ0AM,2773
 secator/tasks/ffuf.py,sha256=VGrtjFgSX6Q1I8h1wjPO5XwBFCfZXmn0DQsn9gxEUXc,2468
-secator/tasks/fping.py,sha256=m7eSXFU5yIeDD_kWh-h208ufSZAm4SpQzD34Ko0yCu8,1116
+secator/tasks/fping.py,sha256=9nMIahBMizRwsos9py-ltXMEffIiyx1reVytj9sTyKU,1089
 secator/tasks/gau.py,sha256=1Qt0I_FqTh-QyJ0YR8I7i_T80HehWSvem_SS-TQKVm0,1648
 secator/tasks/gf.py,sha256=y8Fc0sRLGqNuwUjTBgLk3HEw3ZOnh09nB_GTufGErNA,962
-secator/tasks/gospider.py,sha256=XKLus6GnwN9MYU_ZFmNED-JeRn6n1Eg0CPgul8g1zLs,2302
+secator/tasks/gospider.py,sha256=mpoBq2VQXUqgwWPLz41fzdW85hJeo8mn9FUUJj_DrUw,2275
 secator/tasks/grype.py,sha256=xoOuldnHCrS0O1Y4IzjbSVvoX5eX-fLSZ74THdRC2so,2447
 secator/tasks/h8mail.py,sha256=wNukV-aB-bXPZNq7WL8n1nFgH5b5tGh6vOF80Yna33I,1934
 secator/tasks/httpx.py,sha256=ONfCdAOV7ARCM9tSnlucIAM3UQeWcMUm8QZX8F7u9Pg,5895
@@ -102,15 +102,15 @@ secator/tasks/katana.py,sha256=J0HKPT4QIrDj4uW2gZe7ByW6iEwPortSszqaHDvziwY,5355
 secator/tasks/maigret.py,sha256=6anhBzB4lEM90Lk23cAD_ku7I_ghTpj0W0i3h6HARD8,2088
 secator/tasks/mapcidr.py,sha256=56ocbaDmB5_C_ns-773CgZXGOKOtkI9q9xJs2Rlfqio,990
 secator/tasks/msfconsole.py,sha256=TXVrvzSWw9Ncv2h9QJtaEinTMbps_z0zX1PFirERVho,6430
-secator/tasks/naabu.py,sha256=aAEkQ10ma3Log8OVj8wHY1zUWmjpVQ5pehAMQLJQEV0,2089
+secator/tasks/naabu.py,sha256=90WORQhwFwy71OGNaFe10pCkIG8IJP1XwWQ24OMgSc4,2091
 secator/tasks/nmap.py,sha256=Zu24sJHnlOf3NXLj3Ohi07-x7m-5Ajr5ULpNsUF-QT0,12546
-secator/tasks/nuclei.py,sha256=o677F5yv3mfIlYEpKY5_W6CT2Dlt315DuFOsCjHLE5c,4270
+secator/tasks/nuclei.py,sha256=bMXCRU5VWyrwI7Cv6BCj84NTpfjuALFumPqUSZ4Y6Ug,4243
 secator/tasks/searchsploit.py,sha256=gvtLZbL2hzAZ07Cf0cSj2Qs0GvWK94XyHvoPFsetXu8,3321
 secator/tasks/subfinder.py,sha256=C6W5NnXT92OUB1aSS9IYseqdI3wDMAz70TOEl8X-o3U,1213
-secator/tasks/wpscan.py,sha256=C8eW3vWfbSFrxm5iPzs3MgcagIfSs7u51QZiecYbT2Q,5577
+secator/tasks/wpscan.py,sha256=036ywiEqZfX_Bt071U7qIm7bi6pNk7vodflmuslJurA,5550
 secator/workflows/__init__.py,sha256=ivpZHiYYlj4JqlXLRmB9cmAPUGdk8QcUrCRL34hIqEA,665
-secator-0.10.0.dist-info/METADATA,sha256=TzYxMIn1BO_mo52EkeS5pyg7Ei13hf3Y787EsSa45GE,14724
-secator-0.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-secator-0.10.0.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
-secator-0.10.0.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
-secator-0.10.0.dist-info/RECORD,,
+secator-0.10.1a0.dist-info/METADATA,sha256=c2JLeTa-Pv7TzcWFDPQlfuR1XLU6YnVegdnb1d5_-gc,14726
+secator-0.10.1a0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+secator-0.10.1a0.dist-info/entry_points.txt,sha256=lPgsqqUXWgiuGSfKy-se5gHdQlAXIwS_A46NYq7Acic,44
+secator-0.10.1a0.dist-info/licenses/LICENSE,sha256=19W5Jsy4WTctNkqmZIqLRV1gTDOp01S3LDj9iSgWaJ0,2867
+secator-0.10.1a0.dist-info/RECORD,,