secator 0.9.4__tar.gz → 0.10.0__tar.gz

{secator-0.9.4 → secator-0.10.0}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [0.10.0](https://github.com/freelabz/secator/compare/v0.9.4...v0.10.0) (2025-03-04)
+
+
+### Features
+
+* **celery:** add single pool job processing options ([#558](https://github.com/freelabz/secator/issues/558)) ([cca9797](https://github.com/freelabz/secator/commit/cca9797339ed95c15ef42604ba9bf897a76da766))
+
+
+### Bug Fixes
+
+* add poll queue and re-route tasks to it when chunked ([#562](https://github.com/freelabz/secator/issues/562)) ([b9e7576](https://github.com/freelabz/secator/commit/b9e7576dcd46db0e95df09dc8beafb762314cc39))
+* **config:** disable key not found error message (too verbose) ([#563](https://github.com/freelabz/secator/issues/563)) ([3b044da](https://github.com/freelabz/secator/commit/3b044da14f3e399e03e10c97566a25e858e7e7d6))
+* **katana:** add -no-sandbox option ([#566](https://github.com/freelabz/secator/issues/566)) ([50cf2fe](https://github.com/freelabz/secator/commit/50cf2fef5eaa1b7cf67c691360e3349b7bf7bfbe))
+* mongodb document bug ([#564](https://github.com/freelabz/secator/issues/564)) ([29c6af4](https://github.com/freelabz/secator/commit/29c6af4f3a2a49ac973f7193ca334f56e7b178d5))
+* reset nuclei input chunk size ([#561](https://github.com/freelabz/secator/issues/561)) ([683b9ef](https://github.com/freelabz/secator/commit/683b9efb28d2425961e2e64c31b9af5cb07c0923))
+
 ## [0.9.4](https://github.com/freelabz/secator/compare/v0.9.3...v0.9.4) (2025-02-25)
 
 

{secator-0.9.4 → secator-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: secator
-Version: 0.9.4
+Version: 0.10.0
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.9.4 → secator-0.10.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'hatchling.build'
 
 [project]
 name = 'secator'
-version = "0.9.4"
+version = "0.10.0"
 authors = [{ name = 'FreeLabz', email = 'sales@freelabz.com' }]
 readme = 'README.md'
 description = "The pentester's swiss knife."

{secator-0.9.4 → secator-0.10.0}/secator/celery.py

@@ -1,17 +1,19 @@
 import gc
 import json
 import logging
+import os
 import sys
 import uuid
 
 from time import time
 
-from celery import Celery, chain, chord, signals
+from celery import Celery, chain, chord
 from celery.app import trace
 
 from rich.logging import RichHandler
 from retry import retry
 
+from secator.celery_signals import setup_handlers
 from secator.config import CONFIG
 from secator.output_types import Info, Error
 from secator.rich import console
@@ -78,7 +80,8 @@ app.conf.update({
 		'secator.celery.run_workflow': {'queue': 'celery'},
 		'secator.celery.run_scan': {'queue': 'celery'},
 		'secator.celery.run_task': {'queue': 'celery'},
-		'secator.hooks.mongodb.tag_duplicates': {'queue': 'mongodb'}
+		'secator.celery.forward_results': {'queue': 'results'},
+		'secator.hooks.mongodb.*': {'queue': 'mongodb'}
 	},
 	'task_store_eager_result': True,
 	'task_send_sent_event': CONFIG.celery.task_send_sent_event,
@@ -93,23 +96,8 @@ app.conf.update({
 	'worker_send_task_events': CONFIG.celery.worker_send_task_events
 })
 app.autodiscover_tasks(['secator.hooks.mongodb'], related_name=None)
-
-
-def maybe_override_logging():
-	def decorator(func):
-		if CONFIG.celery.override_default_logging:
-			return signals.setup_logging.connect(func)
-		else:
-			return func
-	return decorator
-
-
-@maybe_override_logging()
-def void(*args, **kwargs):
-	"""Override celery's logging setup to prevent it from altering our settings.
-	github.com/celery/celery/issues/1867
-	"""
-	pass
+if IN_CELERY_WORKER_PROCESS:
+	setup_handlers()
 
 
 @retry(Exception, tries=3, delay=2)
@@ -188,10 +176,10 @@ def break_task(task, task_opts, targets, results=[], chunk_size=1):
 
 	# Build Celery workflow
 	workflow = chain(
-		forward_results.s(results).set(queue='io'),
+		forward_results.s(results).set(queue='results'),
 		chord(
 			tuple(sigs),
-			forward_results.s().set(queue='io'),
+			forward_results.s().set(queue='results'),
 		)
 	)
 	if task.sync:
@@ -204,6 +192,7 @@ def break_task(task, task_opts, targets, results=[], chunk_size=1):
 
 @app.task(bind=True)
 def run_task(self, args=[], kwargs={}):
+	console.print(Info(message=f'Running task {self.request.id}'))
 	kwargs['context']['celery_id'] = self.request.id
 	task = Task(*args, **kwargs)
 	task.run()
@@ -211,6 +200,7 @@ def run_task(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_workflow(self, args=[], kwargs={}):
+	console.print(Info(message=f'Running workflow {self.request.id}'))
 	kwargs['context']['celery_id'] = self.request.id
 	workflow = Workflow(*args, **kwargs)
 	workflow.run()
@@ -218,6 +208,7 @@ def run_workflow(self, args=[], kwargs={}):
 
 @app.task(bind=True)
 def run_scan(self, args=[], kwargs={}):
+	console.print(Info(message=f'Running scan {self.request.id}'))
 	if 'context' not in kwargs:
 		kwargs['context'] = {}
 	kwargs['context']['celery_id'] = self.request.id
@@ -233,6 +224,7 @@ def run_command(self, results, name, targets, opts={}):
 	# Set Celery request id in context
 	context = opts.get('context', {})
 	context['celery_id'] = self.request.id
+	context['worker_name'] = os.environ.get('WORKER_NAME', 'unknown')
 	opts['context'] = context
 	opts['print_remote_info'] = False
 	opts['results'] = results
@@ -244,6 +236,8 @@ def run_command(self, results, name, targets, opts={}):
 			'print_line': True,
 			'print_cmd': True
 		})
+		routing_key = self.request.delivery_info['routing_key']
+		console.print(Info(message=f'Task "{name}" running with routing key "{routing_key}"'))
 
 	# Flatten + dedupe results
 	results = flatten(results)
@@ -254,7 +248,10 @@ def run_command(self, results, name, targets, opts={}):
 		targets, opts = run_extractors(results, opts, targets)
 		debug('after extractors', obj={'targets': targets, 'opts': opts}, sub='celery.state')
 
+	task = None
+
 	try:
+
 		# Get task class
 		task_cls = Task.get_task_class(name)
 
@@ -268,8 +265,14 @@ def run_command(self, results, name, targets, opts={}):
 			'print_remote_info': False,
 			'has_children': chunk_it,
 		})
+
+		if IN_CELERY_WORKER_PROCESS and chunk_it and routing_key != 'poll':
+			console.print(Info(message=f'Task {name} is chunkable but not running on "poll" queue, re-routing to "poll" queue'))
+			raise self.replace(run_command.si(results, name, targets, opts=opts).set(queue='poll', task_id=self.request.id))
+
 		if chunk_it:
 			task_opts['print_cmd'] = False
+
 		task = task_cls(targets, **task_opts)
 		debug(
 			'',
@@ -295,6 +298,7 @@ def run_command(self, results, name, targets, opts={}):
 				targets,
 				results=results,
 				chunk_size=chunk_size)
+			console.print(Info(message=f'Task "{name}" starts polling for chunked results'))
 
 		# Update state before starting
 		update_state(self, task)
@@ -304,6 +308,8 @@ def run_command(self, results, name, targets, opts={}):
 			update_state(self, task)
 
 	except BaseException as e:
+		if not task:
+			raise e
 		error = Error.from_exception(e)
 		error._source = task.unique_name
 		error._uuid = str(uuid.uuid4())
@@ -311,6 +317,8 @@ def run_command(self, results, name, targets, opts={}):
 		task.stop_celery_tasks()
 
 	finally:
+		if not task:
+			raise
 		update_state(self, task, force=True)
 		gc.collect()
 		debug('', obj={task.unique_name: task.status, 'results': task.results}, sub='celery.results', verbose=True)

secator-0.10.0/secator/celery_signals.py

@@ -0,0 +1,103 @@
+import os
+import signal
+import threading
+
+from celery import signals
+
+from secator.config import CONFIG
+from secator.output_types import Info
+from secator.rich import console
+
+
+IDLE_TIMEOUT = CONFIG.celery.worker_kill_after_idle_seconds
+TASK_IN_PROGRESS = False
+
+
+def kill_worker():
+	""""Kill current worker using it's pid by sending a SIGTERM to Celery master process."""
+	worker_name = os.environ['WORKER_NAME']
+	if not TASK_IN_PROGRESS:
+		pid = os.getpid()
+		console.print(Info(message=f'Sending SIGTERM to worker {worker_name} with pid {pid}'))
+		os.kill(pid, signal.SIGTERM)
+	else:
+		console.print(Info(message=f'Cancelling worker shutdown of {worker_name} since a task is currently in progress'))
+
+
+class IdleTimer:
+	def __init__(self, timeout):
+		self.thread = None
+		self.is_started = False
+		self.thread = threading.Timer(timeout, kill_worker)
+
+	def start(self):
+		if self.is_started:
+			self.cancel()
+		self.thread.start()
+		self.is_started = True
+
+	def cancel(self):
+		self.thread.cancel()
+		self.s_started = False
+
+
+IDLE_TIMER = IdleTimer(IDLE_TIMEOUT)
+
+
+def maybe_override_logging():
+	def decorator(func):
+		if CONFIG.celery.override_default_logging:
+			return signals.setup_logging.connect(func)
+		else:
+			return func
+	return decorator
+
+
+@maybe_override_logging()
+def setup_logging(*args, **kwargs):
+	"""Override celery's logging setup to prevent it from altering our settings.
+	github.com/celery/celery/issues/1867
+	"""
+	pass
+
+
+def capture_worker_name(sender, instance, **kwargs):
+    os.environ["WORKER_NAME"] = '{0}'.format(sender)
+
+
+def worker_init_handler(**kwargs):
+	if IDLE_TIMEOUT != -1:
+		console.print(Info(message=f'Starting inactivity timer for {IDLE_TIMEOUT} seconds ...'))
+		IDLE_TIMER.start()
+
+
+def task_prerun_handler(**kwargs):
+	global TASK_IN_PROGRESS, IDLE_TIMER
+	TASK_IN_PROGRESS = True
+	if IDLE_TIMEOUT != -1:
+		IDLE_TIMER.cancel()
+
+
+def task_postrun_handler(**kwargs):
+	global TASK_IN_PROGRESS, IDLE_TIMER
+	TASK_IN_PROGRESS = False
+	sender_name = kwargs['sender'].name
+
+	if CONFIG.celery.worker_kill_after_task and sender_name.startswith('secator.'):
+		worker_name = os.environ['WORKER_NAME']
+		console.print(Info(message=f'Shutdown worker {worker_name} since config celery.worker_kill_after_task is set.'))
+		IDLE_TIMER.cancel()
+		kill_worker()
+		return
+
+	if IDLE_TIMEOUT != -1:  # restart timer
+		console.print(Info(message=f'Reset inactivity timer to {IDLE_TIMEOUT} seconds'))
+		IDLE_TIMER.start()
+
+
+def setup_handlers():
+	signals.celeryd_after_setup.connect(capture_worker_name)
+	signals.setup_logging.connect(setup_logging)
+	signals.task_prerun.connect(task_prerun_handler)
+	signals.task_postrun.connect(task_postrun_handler)
+	signals.worker_ready.connect(worker_init_handler)

{secator-0.9.4 → secator-0.10.0}/secator/cli.py

@@ -148,7 +148,7 @@ def worker(hostname, concurrency, reload, queue, pool, check, dev, stop, show):
 		return
 
 	if not queue:
-		queue = 'io,cpu,' + ','.join([r['queue'] for r in app.conf.task_routes.values()])
+		queue = 'io,cpu,poll,' + ','.join(set([r['queue'] for r in app.conf.task_routes.values()]))
 
 	app_str = 'secator.celery.app'
 	celery = f'{sys.executable} -m celery'

{secator-0.9.4 → secator-0.10.0}/secator/config.py

@@ -73,6 +73,8 @@ class Celery(StrictModel):
 	worker_max_tasks_per_child: int = 20
 	worker_prefetch_multiplier: int = 1
 	worker_send_task_events: bool = False
+	worker_kill_after_task: bool = False
+	worker_kill_after_idle_seconds: int = -1
 
 
 class Cli(StrictModel):
@@ -499,8 +501,8 @@ class Config(DotMap):
 					self.set(path, value, set_partial=False)
 					if not self.validate(print_errors=False) and print_errors:
 						console.print(f'[bold red]{var} (override failed)[/]')
-				elif print_errors:
-					console.print(f'[bold red]{var} (override failed: key not found)[/]')
+				# elif print_errors:
+				# 	console.print(f'[bold red]{var} (override failed: key not found)[/]')
 
 
 def download_files(data: dict, target_folder: Path, offline_mode: bool, type: str):

{secator-0.9.4 → secator-0.10.0}/secator/decorators.py

@@ -28,6 +28,7 @@ RUNNER_OPTS = {
 RUNNER_GLOBAL_OPTS = {
 	'sync': {'is_flag': True, 'help': 'Run tasks synchronously (automatic if no worker is alive)'},
 	'worker': {'is_flag': True, 'default': False, 'help': 'Run tasks in worker'},
+	'no_poll': {'is_flag': True, 'default': False, 'help': 'Do not live poll for tasks results when running in worker'},
 	'proxy': {'type': str, 'help': 'HTTP proxy'},
 	'driver': {'type': str, 'help': 'Export real-time results. E.g: "mongodb"'}
 	# 'debug': {'type': int, 'default': 0, 'help': 'Debug mode'},

{secator-0.9.4 → secator-0.10.0}/secator/runners/_base.py

@@ -96,6 +96,7 @@ class Runner:
 		self.celery_ids_map = {}
 		self.caller = self.run_opts.get('caller', None)
 		self.threads = []
+		self.no_poll = self.run_opts.get('no_poll', False)
 
 		# Determine exporters
 		exporters_str = self.run_opts.get('output') or self.default_exporters
@@ -139,7 +140,7 @@ class Runner:
 		self.print_progress = self.run_opts.get('print_progress', False) and not self.quiet and not self.print_raw
 		self.print_target = self.run_opts.get('print_target', False) and not self.quiet and not self.print_raw
 		self.print_stat = self.run_opts.get('print_stat', False) and not self.quiet and not self.print_raw
-		self.raise_on_error = self.run_opts.get('raise_on_error', not self.sync)
+		self.raise_on_error = self.run_opts.get('raise_on_error', False)
 		self.print_opts = {k: v for k, v in self.__dict__.items() if k.startswith('print_') if v}
 
 		# Debug
@@ -635,6 +636,8 @@ class Runner:
 
 	def log_results(self):
 		"""Log runner results."""
+		if self.no_poll:
+			return
 		self.done = True
 		self.progress = 100
 		self.end_time = datetime.fromtimestamp(time())

{secator-0.9.4 → secator-0.10.0}/secator/runners/command.py

@@ -110,7 +110,7 @@ class Command(Runner):
 	proxy_http = False
 
 	# Profile
-	profile = 'cpu'
+	profile = 'io'
 
 	def __init__(self, inputs=[], **run_opts):
 

{secator-0.9.4 → secator-0.10.0}/secator/runners/task.py

@@ -26,6 +26,7 @@ class Task(Runner):
 		# Run opts
 		run_opts = self.run_opts.copy()
 		run_opts.pop('output', None)
+		run_opts.pop('no_poll', False)
 
 		# Set task output types
 		self.output_types = task_cls.output_types
@@ -48,6 +49,8 @@ class Task(Runner):
 				message=f'Celery task created: {self.celery_result.id}',
 				task_id=self.celery_result.id
 			)
+			if self.no_poll:
+				return
 			results = CeleryData.iter_results(
 				self.celery_result,
 				ids_map=self.celery_ids_map,

{secator-0.9.4 → secator-0.10.0}/secator/runners/workflow.py

@@ -26,6 +26,7 @@ class Workflow(Runner):
 		# Task opts
 		run_opts = self.run_opts.copy()
 		run_opts['hooks'] = self._hooks.get(Task, {})
+		run_opts.pop('no_poll', False)
 
 		# Build Celery workflow
 		workflow = self.build_celery_workflow(
@@ -46,6 +47,8 @@ class Workflow(Runner):
 				message=f'Celery task created: {self.celery_result.id}',
 				task_id=self.celery_result.id
 			)
+			if self.no_poll:
+				return
 			results = CeleryData.iter_results(
 				self.celery_result,
 				ids_map=self.celery_ids_map,
@@ -70,7 +73,7 @@ class Workflow(Runner):
 			self.inputs,
 			self.config.options,
 			run_opts)
-		sigs = [forward_results.si(results).set(queue='io')] + sigs + [forward_results.s().set(queue='io')]
+		sigs = [forward_results.si(results).set(queue='results')] + sigs + [forward_results.s().set(queue='results')]
 		workflow = chain(*sigs)
 		return workflow
 
@@ -102,7 +105,7 @@ class Workflow(Runner):
 					workflow_opts,
 					run_opts
 				)
-				sig = chord((tasks), forward_results.s().set(queue='io'))
+				sig = chord((tasks), forward_results.s().set(queue='results'))
 			elif task_name == '_chain':
 				tasks = self.get_tasks(
 					task_opts,

{secator-0.9.4 → secator-0.10.0}/secator/tasks/katana.py

@@ -30,7 +30,8 @@ class katana(HttpCrawler):
 		'jsluice': {'is_flag': True, 'short': 'jsl', 'default': True, 'help': 'Enable jsluice parsing in javascript file (memory intensive)'},  # noqa: E501
 		'known_files': {'type': str, 'short': 'kf', 'default': 'all', 'help': 'Enable crawling of known files (all, robotstxt, sitemapxml)'},  # noqa: E501
 		'omit_raw': {'is_flag': True, 'short': 'or', 'default': True, 'help': 'Omit raw requests/responses from jsonl output'},  # noqa: E501
-		'omit_body': {'is_flag': True, 'short': 'ob', 'default': True, 'help': 'Omit response body from jsonl output'}
+		'omit_body': {'is_flag': True, 'short': 'ob', 'default': True, 'help': 'Omit response body from jsonl output'},
+		'no_sandbox': {'is_flag': True, 'short': 'ns', 'default': False, 'help': 'Disable sandboxing'},
 	}
 	opt_key_map = {
 		HEADER: 'headers',

{secator-0.9.4 → secator-0.10.0}/secator/tasks/nuclei.py

@@ -16,7 +16,6 @@ class nuclei(VulnMulti):
 	cmd = 'nuclei'
 	file_flag = '-l'
 	input_flag = '-u'
-	input_chunk_size = 1
 	json_flag = '-jsonl'
 	opts = {
 		'templates': {'type': str, 'short': 't', 'help': 'Templates'},

{secator-0.9.4 → secator-0.10.0}/secator/template.py

@@ -6,8 +6,10 @@ from pathlib import Path
 import yaml
 from dotmap import DotMap
 
-from secator.rich import console
 from secator.config import CONFIG, CONFIGS_FOLDER
+from secator.rich import console
+from secator.utils import convert_functions_to_strings
+
 
 TEMPLATES_DIR_KEYS = ['workflow', 'scan', 'profile']
 
@@ -106,7 +108,7 @@ class TemplateLoader(DotMap):
 				task_opts = task_class.get_supported_opts()
 				for name, conf in task_opts.items():
 					if name not in opts or not opts[name].get('supported', False):
-						opts[name] = conf
+						opts[name] = convert_functions_to_strings(conf)
 		return opts
 
 	def _extract_tasks(self):

{secator-0.9.4 → secator-0.10.0}/secator/utils.py

@@ -2,6 +2,7 @@ import fnmatch
 import inspect
 import importlib
 import itertools
+import json
 import logging
 import operator
 import os
@@ -778,3 +779,22 @@ def process_wordlist(val):
 			offline_mode=CONFIG.offline_mode,
 			type='wordlist'
 		)
+
+
+def convert_functions_to_strings(data):
+	"""Recursively convert functions to strings in a dict.
+
+	Args:
+		data (dict): Dictionary to convert.
+
+	Returns:
+		dict: Converted dictionary.
+	"""
+	if isinstance(data, dict):
+		return {k: convert_functions_to_strings(v) for k, v in data.items()}
+	elif isinstance(data, list):
+		return [convert_functions_to_strings(v) for v in data]
+	elif callable(data):
+		return json.dumps(data.__name__)  # or use inspect.getsource(data) if you want the actual function code
+	else:
+		return data