secator 0.22.0__py3-none-any.whl

secator/runners/python.py

@@ -0,0 +1,126 @@
+"""Python runner for executing custom Python code."""
+import logging
+
+from secator.config import CONFIG
+from secator.runners import Runner
+from secator.template import TemplateLoader
+
+
+logger = logging.getLogger(__name__)
+
+
+class PythonRunner(Runner):
+	"""Base class for Python-based tasks.
+
+	This runner allows creating tasks that execute custom Python code without
+	requiring external command-line tools. Tasks should inherit from this class
+	and override the yielder() method.
+
+	Example:
+		>>> from secator.decorators import task
+		>>> from secator.definitions import HOST
+		>>> from secator.output_types import Tag, Url
+		>>> from secator.runners import PythonRunner
+		>>>
+		>>> @task()
+		>>> class mytask(PythonRunner):
+		...     input_types = [HOST]
+		...     output_types = [Tag, Url]
+		...     opts = {'option1': {'type': str, 'help': 'An option'}}
+		...
+		...     def yielder(self):
+		...         for target in self.inputs:
+		...             yield Url(url=f"http://{target}")
+		...             yield Tag(name="scanned", match=target)
+	"""
+	default_exporters = CONFIG.tasks.exporters
+	tags = []
+	opts = {}
+	profile = 'io'
+
+	def needs_chunking(self, sync):
+		return False
+
+	def __init__(self, inputs=[], **run_opts):
+		"""Initialize PythonRunner.
+
+		Args:
+			inputs (list): List of inputs to pass to the task.
+			**run_opts: Additional runner options.
+		"""
+		# Build runner config on-the-fly
+		config = TemplateLoader(input={
+			'name': self.__class__.__name__,
+			'type': 'task',
+			'input_types': self.input_types,
+			'description': run_opts.get('description', None)
+		})
+
+		# Extract run opts
+		hooks = run_opts.pop('hooks', {})
+		caller = run_opts.get('caller', None)
+		results = run_opts.pop('results', [])
+		context = run_opts.pop('context', {})
+		node_id = context.get('node_id', None)
+		node_name = context.get('node_name', None)
+		if node_id:
+			config.node_id = node_id
+		if node_name:
+			config.node_name = node_name
+		self.skip_if_no_inputs = run_opts.pop('skip_if_no_inputs', False)
+		self.enable_validators = run_opts.pop('enable_validators', True)
+
+		# Prepare validators
+		input_validators = []
+		if not self.skip_if_no_inputs:
+			input_validators.append(self._validate_input_nonempty)
+		if not caller:
+			input_validators.append(self._validate_chunked_input)
+		validators = {'validate_input': input_validators}
+
+		# Call super().__init__
+		super().__init__(
+			config=config,
+			inputs=inputs,
+			results=results,
+			run_opts=run_opts,
+			hooks=hooks,
+			validators=validators,
+			context=context)
+
+	@staticmethod
+	def _validate_input_nonempty(self, inputs):
+		"""Input is empty."""
+		if self.default_inputs is not None:
+			return True
+		if not inputs or len(inputs) == 0:
+			return False
+		return True
+
+	@staticmethod
+	def _validate_chunked_input(self, inputs):
+		"""Command does not support multiple inputs in non-worker mode. Consider running with a remote worker instead."""
+		if len(inputs) > 1:
+			return False
+		return True
+
+	def yielder(self):
+		"""Execute the Python task and yield its results.
+
+		This method should be overridden by subclasses to implement
+		the actual task logic.
+
+		Yields:
+			OutputType: Results from the Python task.
+		"""
+		raise NotImplementedError("Subclasses must implement yielder() method")
+
+	@classmethod
+	def delay(cls, *args, **kwargs):
+		"""Submit task to Celery for async execution."""
+		from secator.celery import run_command
+		kwargs['sync'] = False
+		return run_command.apply_async(
+			kwargs={'args': args, 'kwargs': kwargs},
+			queue=cls.profile if not callable(cls.profile) else cls.profile(kwargs)
+		)

secator/runners/scan.py

@@ -0,0 +1,87 @@
+import logging
+from dotmap import DotMap
+
+from secator.config import CONFIG
+from secator.output_types.info import Info
+from secator.runners._base import Runner
+from secator.runners.workflow import Workflow
+from secator.utils import merge_opts
+
+
+logger = logging.getLogger(__name__)
+
+
+class Scan(Runner):
+
+	default_exporters = CONFIG.scans.exporters
+
+	@classmethod
+	def delay(cls, *args, **kwargs):
+		from secator.celery import run_scan
+		return run_scan.delay(args=args, kwargs=kwargs)
+
+	def build_celery_workflow(self):
+		"""Build Celery workflow for scan execution.
+
+		Returns:
+			celery.Signature: Celery task signature.
+		"""
+		from celery import chain
+		from secator.celery import mark_runner_started, mark_runner_completed
+		from secator.template import TemplateLoader
+
+		scan_opts = self.config.options
+
+		# Set hooks and reports
+		self.enable_hooks = False   # Celery will handle hooks
+		self.enable_reports = True  # Workflow will handle reports
+		self.print_item = not self.sync
+
+		# Build chain of workflows
+		sigs = []
+		sig = None
+		for name, workflow_opts in self.config.workflows.items():
+			run_opts = self.run_opts.copy()
+			run_opts.pop('profiles', None)
+			run_opts['no_poll'] = True
+			run_opts['caller'] = 'Scan'
+			run_opts['has_parent'] = True
+			run_opts['enable_reports'] = False
+			run_opts['print_profiles'] = False
+			opts = merge_opts(scan_opts, workflow_opts, run_opts)
+			name = name.split('/')[0]
+			config = TemplateLoader(name=f'workflow/{name}')
+			if not config:
+				raise ValueError(f'Workflow {name} not found')
+
+			# Skip workflow if condition is not met
+			condition = workflow_opts.pop('if', None) if workflow_opts else None
+			local_ns = {'opts': DotMap(opts)}
+			if condition and not eval(condition, {"__builtins__": {}}, local_ns):
+				self.add_result(Info(message=f'Skipped workflow {name} because condition is not met: {condition}'))
+				continue
+
+			# Build workflow
+			workflow = Workflow(
+				config,
+				self.inputs,
+				results=self.results,
+				run_opts=opts,
+				hooks=self._hooks,
+				context=self.context.copy()
+			)
+			celery_workflow = workflow.build_celery_workflow(chain_previous_results=True)
+			for task_id, task_info in workflow.celery_ids_map.items():
+				self.add_subtask(task_id, task_info['name'], task_info['descr'])
+			sigs.append(celery_workflow)
+
+			for result in workflow.results:
+				self.add_result(result, print=False, hooks=False)
+
+		if sigs:
+			sig = chain(
+				mark_runner_started.si([], self).set(queue='results'),
+				*sigs,
+				mark_runner_completed.s(self).set(queue='results'),
+			)
+		return sig

secator/runners/task.py

@@ -0,0 +1,81 @@
+from secator.config import CONFIG
+from secator.runners import Runner
+from secator.loader import discover_tasks
+from celery import chain
+
+
+class Task(Runner):
+
+	default_exporters = CONFIG.tasks.exporters
+
+	@classmethod
+	def delay(cls, *args, **kwargs):
+		from secator.celery import run_task
+		return run_task.apply_async(kwargs={'args': args, 'kwargs': kwargs}, queue='celery')
+
+	def build_celery_workflow(self):
+		"""Build Celery workflow for task execution.
+
+		Args:
+			run_opts (dict): Run options.
+			results (list): Prior results.
+
+		Returns:
+			celery.Signature: Celery task signature.
+		"""
+		from secator.celery import run_command
+
+		# Get task class
+		task_cls = Task.get_task_class(self.config.name)
+
+		# Run opts
+		opts = self.run_opts.copy()
+		opts.pop('output', None)
+		opts.pop('profiles', None)
+		opts.pop('no_poll', False)
+
+		# Set output types
+		self.output_types = task_cls.output_types
+
+		# Set hooks and reports
+		self.enable_hooks = False   # Celery will handle hooks
+		self.enable_reports = True  # Task will handle reports
+
+		# Get hooks
+		hooks = self._hooks.get(Task, {})
+		opts['hooks'] = hooks
+		opts['context'] = self.context.copy()
+		opts['reports_folder'] = str(self.reports_folder)
+
+		# Task class will handle those
+		opts['enable_reports'] = False
+		opts['enable_profiles'] = False
+		opts['enable_duplicate_check'] = False
+		opts['print_start'] = False
+		opts['print_end'] = False
+		opts['print_target'] = False
+		opts['has_parent'] = False
+		opts['skip_if_no_inputs'] = False
+		opts['caller'] = 'Task'
+
+		# Create task signature
+		profile = task_cls.profile(opts) if callable(task_cls.profile) else task_cls.profile
+		sig = run_command.si(self.results, self.config.name, self.inputs, opts).set(queue=profile)
+		task_id = sig.freeze().task_id
+		self.add_subtask(task_id, self.config.name, self.description)
+		return chain(sig)
+
+	@staticmethod
+	def get_task_class(name):
+		"""Get task class from a name.
+
+		Args:
+			name (str): Task name.
+		"""
+		if '/' in name:
+			name = name.split('/')[0]
+		tasks_classes = discover_tasks()
+		for task_cls in tasks_classes:
+			if task_cls.__name__ == name:
+				return task_cls
+		raise ValueError(f'Task {name} not found. Aborting.')

secator/runners/workflow.py

@@ -0,0 +1,168 @@
+from dotmap import DotMap
+
+from secator.config import CONFIG
+from secator.output_types import Info
+from secator.runners._base import Runner
+from secator.runners.task import Task
+from secator.tree import build_runner_tree, walk_runner_tree
+from secator.utils import merge_opts
+
+
+class Workflow(Runner):
+
+	default_exporters = CONFIG.workflows.exporters
+
+	@classmethod
+	def delay(cls, *args, **kwargs):
+		from secator.celery import run_workflow
+		return run_workflow.delay(args=args, kwargs=kwargs)
+
+	@classmethod
+	def s(cls, *args, **kwargs):
+		from secator.celery import run_workflow
+		return run_workflow.s(args=args, kwargs=kwargs)
+
+	def build_celery_workflow(self, chain_previous_results=False):
+		"""Build Celery workflow for workflow execution.
+
+		Args:
+			chain_previous_results (bool): Chain previous results.
+
+		Returns:
+			celery.Signature: Celery task signature.
+		"""
+		from celery import chain
+		from secator.celery import mark_runner_started, mark_runner_completed, forward_results
+
+		# Prepare run options
+		opts = self.run_opts.copy()
+		opts.pop('output', None)
+		opts.pop('no_poll', False)
+		opts.pop('print_profiles', False)
+
+		# Set hooks and reports
+		self.enable_hooks = False   # Celery will handle hooks
+		self.enable_reports = True  # Workflow will handle reports
+		self.print_item = not self.sync
+
+		# Get hooks
+		hooks = self._hooks.get(Task, {})
+		opts['hooks'] = hooks
+		opts['context'] = self.context.copy()
+		opts['reports_folder'] = str(self.reports_folder)
+		opts['enable_reports'] = False  # Workflow will handle reports
+		opts['enable_duplicate_check'] = False  # Workflow will handle duplicate check
+		opts['has_parent'] = True
+		opts['skip_if_no_inputs'] = True
+		opts['caller'] = 'Workflow'
+
+		# Remove workflow config prefix from opts
+		for k, v in opts.copy().items():
+			if k.startswith(self.config.name + '_'):
+				opts[k.replace(self.config.name + '_', '')] = v
+
+		# Remove dynamic opts from parent runner
+		opts = {k: v for k, v in opts.items() if k not in self.dynamic_opts}
+
+		# Forward workflow opts to first task if needed
+		forwarded_opts = {}
+		if chain_previous_results:
+			forwarded_opts = self.dynamic_opts
+
+		# Build workflow tree
+		tree = build_runner_tree(self.config)
+		current_id = tree.root_nodes[0].id
+		ix = 0
+		sigs = []
+
+		def process_task(node, force=False, parent_ix=None):
+			from celery import chain, group
+			from secator.utils import debug
+			nonlocal ix
+			sig = None
+
+			if node.id is None:
+				return
+
+			if node.type == 'task':
+				if node.parent.type == 'group' and not force:
+					return
+
+				# Skip task if condition is not met
+				condition = node.opts.pop('if', None)
+				local_ns = {'opts': DotMap(opts)}
+				if condition:
+					# debug(f'{node.id} evaluating {condition} with opts {opts}', sub=self.config.name)
+					result = eval(condition, {"__builtins__": {}}, local_ns)
+					if not result:
+						debug(f'{node.id} skipped task because condition is not met: {condition}', sub=self.config.name)
+						self.add_result(Info(message=f'Skipped task [bold gold3]{node.name}[/] because condition is not met: [bold green]{condition}[/]'))  # noqa: E501
+						return
+
+				# Get task class
+				task = Task.get_task_class(node.name)
+
+				# Merge task options (order of priority with overrides)
+				task_opts = merge_opts(self.config.default_options.toDict(), node.opts, opts)
+				if (ix == 0 or parent_ix == 0) and forwarded_opts:
+					task_opts.update(forwarded_opts)
+
+				# Create task signature
+				task_opts['name'] = node.name
+				task_opts['context'] = self.context.copy()
+				task_opts['context']['node_id'] = node.id
+				task_opts['context']['ancestor_id'] = None if (ix == 0 or parent_ix == 0) else current_id
+				task_opts['aliases'] = [node.id, node.name]
+				if task.__name__ != node.name:
+					task_opts['aliases'].append(task.__name__)
+				profile = task.profile(task_opts) if callable(task.profile) else task.profile
+				sig = task.s(self.inputs, **task_opts).set(queue=profile)
+				task_id = sig.freeze().task_id
+				debug(f'{node.id} sig built ix: {ix}, parent_ix: {parent_ix}', sub=self.config.name)
+				# debug(f'{node.id} opts', obj=task_opts, sub=f'workflow.{self.config.name}')
+				debug(f'{node.id} ancestor id: {task_opts.get("context", {}).get("ancestor_id")}', sub=self.config.name)
+				self.add_subtask(task_id, node.name, task_opts.get('description', ''))
+				self.output_types.extend(task.output_types)
+				ix += 1
+
+			elif node.type == 'group' and node.children:
+				parent_ix = ix
+				tasks = [sig for sig in [process_task(child, force=True, parent_ix=parent_ix) for child in node.children] if sig]
+				debug(f'{node.id} group built with {len(tasks)} tasks', sub=self.config.name)
+				if len(tasks) == 1:
+					debug(f'{node.id} downgraded group to task', sub=self.config.name)
+					sig = tasks[0]
+				elif len(tasks) > 1:
+					sig = group(*tasks)
+					last_sig = sigs[-1] if sigs else None
+					if sig and isinstance(last_sig, group):  # cannot chain 2 groups without bridge task
+						debug(f'{node.id} previous is group, adding bridge task forward_results', sub=self.config.name)
+						sigs.append(forward_results.s())
+				else:
+					debug(f'{node.id} group built with 0 tasks', sub=self.config.name)
+				ix += 1
+
+			elif node.type == 'chain' and node.children:
+				tasks = [sig for sig in [process_task(child, force=True, parent_ix=ix) for child in node.children] if sig]
+				sig = chain(*tasks) if tasks else None
+				debug(f'{node.id} chain built with {len(tasks)} tasks', sub=self.config.name)
+				ix += 1
+
+			if sig and node.parent.type != 'group':
+				debug(f'{node.id} added to workflow', sub=self.config.name)
+				sigs.append(sig)
+
+			return sig
+
+		walk_runner_tree(tree, process_task)
+
+		# Build workflow chain with lifecycle management
+		start_sig = mark_runner_started.si([], self, enable_hooks=True).set(queue='results')
+		if chain_previous_results:
+			start_sig = mark_runner_started.s(self, enable_hooks=True).set(queue='results')
+		sig = chain(
+			start_sig,
+			*sigs,
+			mark_runner_completed.s(self, enable_hooks=True).set(queue='results'),
+		)
+		return sig

secator/scans/__init__.py

@@ -0,0 +1,29 @@
+from secator.loader import get_configs_by_type
+from secator.runners import Scan
+
+
+class DynamicScan(Scan):
+	def __init__(self, config):
+		self.config = config
+
+	def __call__(self, targets, **kwargs):
+		hooks = kwargs.pop('hooks', {})
+		results = kwargs.pop('results', [])
+		context = kwargs.pop('context', {})
+		super().__init__(
+			config=self.config,
+			inputs=targets,
+			results=results,
+			hooks=hooks,
+			context=context,
+			run_opts=kwargs)
+		return self
+
+
+DYNAMIC_SCANS = {}
+for scan in get_configs_by_type('scan'):
+	instance = DynamicScan(scan)
+	DYNAMIC_SCANS[scan.name] = instance
+
+globals().update(DYNAMIC_SCANS)
+__all__ = list(DYNAMIC_SCANS)

secator/serializers/__init__.py

@@ -0,0 +1,8 @@
+__all__ = [
+    'JSONSerializer',
+    'RegexSerializer',
+    'DataclassEncoder',
+]
+from secator.serializers.json import JSONSerializer
+from secator.serializers.regex import RegexSerializer
+from secator.serializers.dataclass import DataclassEncoder

secator/serializers/dataclass.py

@@ -0,0 +1,39 @@
+from datetime import date, datetime
+import json
+from pathlib import PosixPath
+from secator.output_types import OUTPUT_TYPES
+
+
+class DataclassEncoder(json.JSONEncoder):
+	def default(self, obj):
+		if hasattr(obj, 'toDict'):
+			return obj.toDict()
+		elif isinstance(obj, PosixPath):
+			return str(obj)
+		elif isinstance(obj, (datetime, date)):
+			return obj.isoformat()
+		else:
+			return json.JSONEncoder.default(self, obj)
+
+
+def get_output_cls(type):
+	try:
+		return [cls for cls in OUTPUT_TYPES if cls.get_name() == type][0]
+	except IndexError:
+		return None
+
+
+def dataclass_decoder(obj):
+	if '_type' in obj:
+		output_cls = get_output_cls(obj['_type'])
+		if output_cls:
+			return output_cls.load(obj)
+	return obj
+
+
+def dumps_dataclass(obj, indent=None):
+	return json.dumps(obj, cls=DataclassEncoder, indent=indent)
+
+
+def loads_dataclass(obj):
+	return json.loads(obj, object_hook=dataclass_decoder)

secator/serializers/json.py

@@ -0,0 +1,45 @@
+import json
+
+
+class JSONSerializer:
+
+	def __init__(self, strict=False, list=False):
+		self.strict = strict
+		self.list = list
+
+	def run(self, line):
+		if self.list:
+			return self._load_list(line)
+		else:
+			return self._load_single(line)
+
+	def _load_single(self, line):
+		start_index = line.find('{')
+		end_index = line.rfind('}')
+		if start_index == -1 or end_index == -1:
+			return
+		if start_index != 0 and self.strict:
+			return
+		try:
+			json_obj = line[start_index:end_index+1]
+			yield json.loads(json_obj)
+		except json.decoder.JSONDecodeError:
+			return
+
+	def _load_list(self, line):
+		start_index = line.find('[{')
+		end_index = line.rfind('}]')
+		if start_index == -1 or end_index == -1:
+			return
+		if start_index != 0 and self.strict:
+			return
+		try:
+			json_obj = line[start_index:end_index+2]
+			obj = json.loads(json_obj)
+			if isinstance(obj, list):
+				for item in obj:
+					yield item
+			else:
+				yield obj
+		except json.decoder.JSONDecodeError:
+			return

secator/serializers/regex.py

@@ -0,0 +1,25 @@
+import re
+
+
+class RegexSerializer:
+
+	def __init__(self, regex, fields=[], findall=False):
+		self.regex = re.compile(regex)
+		self.fields = fields
+		self.findall = findall
+
+	def run(self, line):
+		if self.findall:
+			match = self.regex.findall(line)
+			yield from match
+			return
+		output = {}
+		match = self.regex.match(line)
+		if not match:
+			return
+		if not self.fields:
+			yield match.group(0)
+			return
+		for field in self.fields:
+			output[field] = match.group(field)
+		yield output

secator/tasks/__init__.py

@@ -0,0 +1,8 @@
+from secator.loader import discover_tasks
+TASKS = discover_tasks()
+__all__ = [
+	cls.__name__
+	for cls in TASKS
+]
+for cls in TASKS:
+	exec(f'from .{cls.__name__} import {cls.__name__}')