secator 0.0.1__py3-none-any.whl

tests/integration/test_tasks.py

@@ -0,0 +1,103 @@
+import logging
+import os
+import unittest
+import warnings
+from time import sleep
+
+from secator.definitions import DEBUG
+from secator.rich import console
+from secator.runners import Command
+from secator.utils import setup_logging, merge_opts
+from secator.utils_test import (META_OPTS, TEST_TASKS, CommandOutputTester,
+                              load_fixture)
+from tests.integration.inputs import INPUTS_TASKS
+from tests.integration.outputs import OUTPUTS_TASKS
+
+INTEGRATION_DIR = os.path.dirname(os.path.abspath(__file__))
+level = logging.DEBUG if DEBUG > 0 else logging.INFO
+setup_logging(level)
+
+
+class TestTasks(unittest.TestCase, CommandOutputTester):
+	def setUp(self):
+		warnings.simplefilter('ignore', category=ResourceWarning)
+		warnings.simplefilter('ignore', category=DeprecationWarning)
+		Command.run_command(
+			f'sh {INTEGRATION_DIR}/setup.sh',
+			cwd=INTEGRATION_DIR
+		)
+		sleep(15)
+
+	def tearDown(self):
+		Command.run_command(
+			f'sh {INTEGRATION_DIR}/teardown.sh',
+			cwd=INTEGRATION_DIR
+		)
+
+	def test_tasks(self):
+		opts = META_OPTS.copy()
+		fmt_opts = {
+			'print_cmd': DEBUG > 0,
+			'print_item': DEBUG > 1,
+			'print_item_count': DEBUG > 0,
+			'json': DEBUG > 2
+		}
+		extra_opts = {
+			'dirsearch.filter_size': 1987,
+			'dnsxbrute.wordlist': load_fixture('wordlist_dns', INTEGRATION_DIR, only_path=True),
+			'ffuf.filter_size': 1987,
+			'feroxbuster.filter_size': 1987,
+			'h8mail.local_breach': load_fixture('h8mail_breach', INTEGRATION_DIR, only_path=True),
+			'nmap.port': '3000,8080',
+			'match_codes': '200',
+			'maigret.site': 'github',
+			'wordlist': load_fixture('wordlist', INTEGRATION_DIR, only_path=True),
+		}
+
+		# Merge opts
+		opts = merge_opts(opts, fmt_opts, extra_opts)
+
+		# Remove unit tests options
+		del opts['nmap.output_path']
+		del opts['maigret.output_path']
+		del opts['dirsearch.output_path']
+		del opts['wpscan.output_path']
+		del opts['timeout']
+
+		for cls in TEST_TASKS:
+			if cls.__name__ == 'msfconsole':  # skip msfconsole test as it's stuck
+				continue
+			with self.subTest(name=cls.__name__):
+				console.print(f'Testing {cls.__name__} ...')
+
+				# Get task input
+				input = INPUTS_TASKS.get(cls.__name__) or INPUTS_TASKS.get(cls.input_type)
+				if not input:
+					console.print(f'No input for {cls.__name__} ! Skipping')
+					continue
+
+				# Get task output
+				outputs = OUTPUTS_TASKS.get(cls.__name__, [])
+
+				# Init task
+				task = cls(input, **opts)
+
+				# Run task
+				results = task.run()
+				if DEBUG > 2:
+					console.print([list(r.toDict() for r in results)])
+
+				# Check return code
+				if not task.ignore_return_code:
+					self.assertEqual(task.return_code, 0)
+
+				if not results:
+					console.print(f'No results from {cls.__name__} ! Skipping item check.')
+					continue
+
+				# Test result types
+				self._test_task_output(
+					results,
+					expected_output_types=cls.output_types,
+					expected_results=outputs,
+					empty_results_allowed=True)

tests/integration/test_workflows.py

@@ -0,0 +1,163 @@
+import logging
+import os
+import unittest
+import warnings
+from time import sleep
+
+from secator.config import ConfigLoader
+from secator.runners import Task
+from secator.output_types import Target, Port, Url
+from secator.definitions import DEBUG
+from secator.rich import console
+from secator.runners import Command, Workflow
+from secator.utils import setup_logging, merge_opts
+from secator.utils_test import TEST_WORKFLOWS, CommandOutputTester, load_fixture
+from tests.integration.inputs import INPUTS_WORKFLOWS
+from tests.integration.outputs import OUTPUTS_WORKFLOWS
+
+INTEGRATION_DIR = os.path.dirname(os.path.abspath(__file__))
+level = logging.DEBUG if DEBUG > 0 else logging.INFO
+setup_logging(level)
+
+
+def hook_workflow_init(self):
+	self.context['workflow_id'] = 1
+
+
+def hook_task_init(self):
+	self.context['task_id'] = 1
+
+
+def hook_item(self, item):
+	print(item.toDict())
+	return item
+
+
+class TestWorkflows(unittest.TestCase, CommandOutputTester):
+
+	def setUp(self):
+		warnings.simplefilter('ignore', category=ResourceWarning)
+		warnings.simplefilter('ignore', category=DeprecationWarning)
+		Command.run_command(
+			f'sh {INTEGRATION_DIR}/setup.sh',
+			cwd=INTEGRATION_DIR
+		)
+		sleep(15)
+
+	def tearDown(self):
+		Command.run_command(
+			f'sh {INTEGRATION_DIR}/teardown.sh',
+			cwd=INTEGRATION_DIR
+		)
+
+	def test_default_workflows(self):
+		fmt_opts = {
+			'print_cmd': DEBUG > 0,
+			'print_item': DEBUG > 1,
+			'print_line': DEBUG > 2,
+			'table': DEBUG > 1,
+			'output': 'table' if DEBUG > 1 else ''
+		}
+		opts = {
+			'ffuf.filter_size': 1987,
+			'feroxbuster.filter_size': 1987,
+			'dirsearch.filter_size': 1987,
+			'follow_redirect': True,
+			'match_codes': '200',
+			'httpx.match_codes': False,
+			'httpx.filter_size': False,
+			'nuclei.retries': 5,
+			'nuclei.timeout': 15,
+			'rate_limit': 1000,
+			'wordlist': load_fixture('wordlist', INTEGRATION_DIR, only_path=True),
+			'timeout': 7,
+			'depth': 2
+		}
+		opts = merge_opts(opts, fmt_opts)
+
+		for conf in TEST_WORKFLOWS:
+			with self.subTest(name=conf.name):
+				console.print(f'Testing workflow {conf.name} ...')
+				inputs = INPUTS_WORKFLOWS.get(conf.name, [])
+				outputs = OUTPUTS_WORKFLOWS.get(conf.name, [])
+				if not inputs:
+					console.print(
+						f'No inputs for workflow {conf.name} ! Skipping.', style='dim red'
+					)
+					continue
+				workflow = Workflow(conf, targets=inputs, run_opts=opts)
+				results = workflow.run()
+				if DEBUG > 0:
+					for result in results:
+						print(repr(result))
+				if not outputs:
+					console.print(
+						f'No outputs for workflow {conf.name} ! Skipping.', style='dim red'
+					)
+					continue
+				self._test_task_output(
+					results,
+					expected_results=outputs)
+
+	def test_adhoc_workflow(self):
+		# Ignore if TEST_WORKFLOWS are defined
+		if TEST_WORKFLOWS:
+			return
+
+		# Expected results / context
+		expected_results = [
+			Port(port=9999, host='localhost', service_name='fake', _source='unknown'),
+			Port(port=3000, host='localhost', ip='127.0.0.1', _source='naabu'),
+			Port(port=8080, host='localhost', ip='127.0.0.1', _source='naabu'),
+			Url(url='http://localhost:3000', host='127.0.0.1', status_code=200, title='OWASP Juice Shop', content_type='text/html', _source='httpx'),
+			Url(url='http://localhost:8080', host='127.0.0.1', status_code=400, title='', content_type='application/json', _source='httpx'),
+		]
+		expected_context = {
+			'task_id': 1,
+			'workflow_id': 1
+		}
+
+		# Create ad-hoc workflow
+		conf = {
+			'name': 'my_workflow',
+			'description': 'Test workflow',
+			'tasks': {
+				'naabu': {},
+				'httpx': {
+					'targets_': {'type': 'port', 'field': '{host}:{port}'}
+				}
+			}
+		}
+		config = ConfigLoader(conf)
+		workflow = Workflow(
+			config,
+			targets=['localhost'],
+			results=[
+				Port(port=9999, host='localhost', service_name='fake', _source='unknown', _context=expected_context)
+			],
+			hooks = {
+				Workflow: {
+					'on_init': [hook_workflow_init],
+				},
+				Task: {
+					'on_init': [hook_task_init],
+					'on_item': [hook_item],
+				}
+			}
+		)
+		uuids = []
+		results = []
+
+		# Verify no duplicates and context added from hook is present in output
+		for result in workflow:
+			self.assertEqual(result._context, expected_context)
+			self.assertNotIn(result._uuid, uuids)
+			uuids.append(result._uuid)
+			results.append(result)
+
+		# Verify results yielded from workflow and workflow.results are equal
+		self.assertEqual(results, workflow.results)
+
+		# Verify expected results are there
+		for res in expected_results:
+			self.assertIn(res, workflow.results)

tests/performance/loadtester.py

@@ -0,0 +1,56 @@
+import eventlet
+eventlet.monkey_patch()
+from secator.runners import Workflow
+from secator.config import ConfigLoader
+from secator.rich import console
+from secator.celery import *
+import _thread
+import os
+import sys
+import json
+from time import sleep, time
+
+pool = eventlet.GreenPool(100)
+
+from kombu.serialization import register
+
+nworkflows = int(os.environ.get('SECATOR_WORKFLOWS_COUNT', '1'))
+targets = [c.strip() for c in os.environ.get('SECATOR_TARGETS', '').split(',') if c]
+
+if not targets:
+	console.log('Please specify test targets with SECATOR_TARGETS keys (hosts are comma-separated)', style='bold red')
+	sys.exit(1)
+
+if not is_celery_worker_alive():
+	console.log('A Celery worker must be running. Start one using "secator worker"', style='bold red')
+	sys.exit(1)
+
+console.log(f'Load tester initialized with {len(targets)} targets.')
+console.log(f'Targets: {targets}')
+
+def create_runner(index):
+	register('json', json.dumps, json.loads, content_type='application/json', content_encoding='utf-8')
+	config = ConfigLoader(name='workflows/subdomain_recon')
+	run_opts = {
+		'sync': False,
+		# 'print_start': True,
+		'print_item': True,
+		# 'print_remote_status': True,
+		# 'print_run_summary': True,
+		'json': False
+	}
+	result = Workflow.delay(config, targets, run_opts=run_opts)
+	while not result.ready():
+		print(f'Running workflow {index + 1}/{nworkflows} ..')
+		sleep(2)
+	result.get()
+
+
+if __name__ == '__main__':
+	print(f'Starting {nworkflows} workflows ...')
+	start_time = time()
+	for i in range(nworkflows):
+		pool.spawn(create_runner, i)
+	pool.waitall()
+	elapsed_time = time() - start_time
+	print(f'All workflows completed in {elapsed_time:.2f}s.')

tests/unit/test_celery.py

@@ -0,0 +1,39 @@
+from secator.celery import *
+import unittest
+
+
+# class TestCeleryCommand(unittest.TestCase):
+
+#     def test_parent_run_command(self):
+#         result = run_command.apply(
+#             args=[
+#                 [],
+#                 'ffuf',
+#                 ['https://mydomain.com', 'https://media.mydomain.com']
+#             ]
+#         )
+#         results = result.get()
+#         self.assertTrue(isinstance(results, list))
+
+    # def test_parent_run_command_delay(self):
+    #     result = run_command.delay(
+    #         [],
+    #         'ffuf',
+    #         ['https://mydomain.com', 'https://media.mydomain.com'],
+    #         opts={
+    #             'sync': False
+    #         }
+    #     )
+    #     results = result.get()
+    #     self.assertTrue(isinstance(results, dict))
+    #     self.assertTrue('results' in results)
+
+    # def test_parent_command_delay(self):
+    #     result = ffuf.delay(
+    #         ['https://mydomain.com', 'https://media.mydomain.com'],
+    #         print_cmd=True,
+    #         print_item=True,
+    #     )
+    #     ffuf.poll(result)
+    #     results = result.get()
+    #     print(results)

tests/unit/test_serializers.py

@@ -0,0 +1,51 @@
+from secator.celery import *
+import unittest
+from secator.serializers.dataclass import dumps_dataclass, loads_dataclass
+from secator.output_types import Port, Vulnerability
+
+
+class TestSerializers(unittest.TestCase):
+
+	def test_dumps_loads(self):
+		results = [
+			Port(port=53, ip='127.0.0.1', host='localhost'),
+			Vulnerability(matched_at='localhost', name='CVE-123123123', provider='nmap')
+		]
+		results = dumps_dataclass(results)
+		results = loads_dataclass(results)
+		self.assertTrue(isinstance(results, list))
+		self.assertEqual(len(results), 2)
+		self.assertTrue(all(type(result) in [Port, Vulnerability] for result in results))
+		self.assertTrue(isinstance(results, list))
+
+	def test_dumps_loads_nested(self):
+		results = {
+			'info': {'name': 'test'},
+			'results': {
+				'ports': [
+					{'port': 53, 'ip': '127.0.0.1', 'host': 'localhost', '_type': 'port'},
+				],
+				'vulnerabilities': [
+					{'matched_at': 'localhost', 'name': 'CVE-123123123', 'provider': 'nmap', '_type': 'vulnerability'}
+				]
+			}
+		}
+		results = loads_dataclass(dumps_dataclass(results))
+		self.assertTrue(isinstance(results['results']['ports'][0], Port))
+		self.assertTrue(isinstance(results['results']['vulnerabilities'][0], Vulnerability))
+
+	def test_dumps_loads_nested_obj(self):
+		results = {
+			'info': {'name': 'test'},
+			'results': {
+				'ports': [
+					Port(port=53, ip='127.0.0.1', host='localhost'),
+				],
+				'vulnerabilities': [
+					Vulnerability(matched_at='localhost', name='CVE-123123123', provider='nmap')
+				]
+			}
+		}
+		results = loads_dataclass(dumps_dataclass(results))
+		self.assertTrue(isinstance(results['results']['ports'][0], Port))
+		self.assertTrue(isinstance(results['results']['vulnerabilities'][0], Vulnerability))