secator 0.0.1__py3-none-any.whl → 0.3.6__py3-none-any.whl

tests/unit/test_tasks.py

@@ -1,348 +0,0 @@
-import copy
-import json
-import logging
-import unittest
-import unittest.mock
-import warnings
-
-from secator.definitions import (DEBUG, DELAY, FOLLOW_REDIRECT, HEADER, HOST,
-							   MATCH_CODES, OPT_NOT_SUPPORTED, RATE_LIMIT,
-							   THREADS, TIMEOUT, DEFAULT_HTTPX_FLAGS)
-from secator.rich import console
-from secator.runners import Command
-from secator.tasks import httpx
-from secator.utils import setup_logging
-from secator.utils_test import (FIXTURES_TASKS, FIXTURES_DIR, INPUTS_TASKS, META_OPTS,
-							  TEST_TASKS, CommandOutputTester, load_fixture,
-							  mock_command, mock_subprocess_popen)
-
-level = logging.DEBUG if DEBUG > 0 else logging.ERROR
-setup_logging(level)
-
-
-class FakeCmd(Command):
-	opts = {
-		'opt1': {'type': int, 'default': 10},
-		'opt2': {'type': str, 'default': '1,2,3'},
-		'opt3': {'is_flag': True, 'default': False}, # optional
-		'opt_with_underscore': {'type': str}, # optional
-		'opt-with-hyphen': {'type': str}
-	}
-	opt_prefix = '-' # all options have '-' as option char
-	opt_key_map = {
-		'opt3': '--opt3', # but opt3 has '--' prefix
-		'opt4': OPT_NOT_SUPPORTED
-	}
-	opt_value_map = {
-		'opt1': lambda x: float(x) # actually opt1 value should be cast to a float
-	}
-
-
-class TestCommandProcessOpts(unittest.TestCase):
-	
-	def setUp(self):
-		self.maxDiff = None
-
-	def test_process_opts_defaults(self):
-		run_opts = {}
-		opts_str = FakeCmd._process_opts(
-			run_opts,
-			FakeCmd.opts,
-			FakeCmd.opt_key_map,
-			FakeCmd.opt_value_map)
-		self.assertEqual(opts_str, '-opt1 10.0 -opt2 1,2,3')
-
-	def test_process_opts(self):
-		run_opts = {
-			'opt1': 41,
-			'opt2': False, # intentionally omit arg, overriding default value
-			'opt3': True
-		}
-		opts_str = FakeCmd._process_opts(
-			run_opts,
-			FakeCmd.opts,
-			FakeCmd.opt_key_map,
-			FakeCmd.opt_value_map)
-		self.assertEqual(opts_str, '-opt1 41.0 --opt3')
-
-	def test_process_opts_with_prefix(self):
-		run_opts = {
-			'fakecmd_opt1': 41, # should override opt1 below
-			'opt1': 45,
-			'opt2': False, # intentionally omit arg, overriding default value
-			'opt3': True
-		}
-		opts_str = FakeCmd._process_opts(
-			run_opts,
-			FakeCmd.opts,
-			FakeCmd.opt_key_map,
-			FakeCmd.opt_value_map,
-			command_name='fakecmd')
-		self.assertEqual(opts_str, '-opt1 41.0 --opt3')
-
-	def test_process_opts_with_unsupported(self):
-		run_opts = {
-			'fakecmd_opt1': 41, # should override opt1 below
-			'opt1': 45,
-			'opt2': False, # intentionally omit arg, overriding default value
-			'opt3': True,
-			'opt4': 'test_unsupported'
-		}
-		opts_str = FakeCmd._process_opts(
-			run_opts,
-			FakeCmd.opts,
-			FakeCmd.opt_key_map,
-			FakeCmd.opt_value_map,
-			command_name='fakecmd')
-		self.assertEqual(opts_str, '-opt1 41.0 --opt3')
-
-	def test_process_opts_with_convert_underscore(self):
-		run_opts = {
-			'fakecmd_opt1': 41, # should override opt1 below
-			'opt1': 45,
-			'opt2': False, # intentionally omit arg, overriding default value
-			'opt3': True,
-			'opt4': 'test_unsupported',
-			'opt_with_underscore': 'test'
-		}
-		opts_str = FakeCmd._process_opts(
-			run_opts,
-			FakeCmd.opts,
-			FakeCmd.opt_key_map,
-			FakeCmd.opt_value_map,
-			command_name='fakecmd')
-		self.assertEqual(opts_str, '-opt1 41.0 --opt3 -opt-with-underscore test')
-
-	def test_get_opt_value(self):
-		run_opts = {
-			'fakecmd_opt1': 41,
-			'opt1': 45
-		}
-		opt_value = FakeCmd._get_opt_value(
-			run_opts,
-			opt_name='opt1',
-			opt_prefix='fakecmd',
-			default=10)
-		self.assertEqual(opt_value, 41)
-
-	def test_get_opt_value_false(self):
-		run_opts = {
-			'fakecmd_opt1': False,
-			'opt1': 45
-		}
-		opt_value = FakeCmd._get_opt_value(
-			run_opts,
-			opt_name='opt1',
-			opt_prefix='fakecmd',
-			default=10)
-		self.assertEqual(opt_value, False)
-
-	def test_get_opt_value_not_supported(self):
-		run_opts = {
-			'fakecmd_opt1': False,
-			'opt1': 45,
-			'opt4': OPT_NOT_SUPPORTED
-		}
-		opt_value = FakeCmd._get_opt_value(
-			run_opts,
-			opt_name='opt4',
-			opt_prefix='fakecmd',
-			default=10)
-		self.assertEqual(opt_value, None)
-
-	def test_httpx_build_cmd_defaults(self):
-		if not httpx in TEST_TASKS:
-			return
-		run_opts = {}
-		host = 'test.synology.me'
-		cls = httpx(host, **run_opts)
-		default_threads = cls.meta_opts[THREADS]['default']
-		expected_cmd = f'httpx {DEFAULT_HTTPX_FLAGS} -u {host} -json -threads {default_threads}'
-		self.assertEqual(cls.cmd, expected_cmd)
-		self.assertEqual(cls.print_line, False)
-		self.assertEqual(cls.print_item, False)
-		self.assertEqual(cls.print_item_count, False)
-		self.assertEqual(cls.print_cmd, False)
-		self.assertEqual(cls.print_cmd_prefix, False)
-		self.assertEqual(cls.output_json, True)
-
-	def test_httpx_build_cmd_with_opts(self):
-		if not httpx in TEST_TASKS:
-			return
-		run_opts = {
-			FOLLOW_REDIRECT: False,
-			DELAY: 1,
-			RATE_LIMIT: 120,
-			THREADS: 10,
-			TIMEOUT: 1,
-			HEADER: 'Content-Type: application/xml',
-			MATCH_CODES: False, # intentionally omit arg, overriding default value
-			'filter_codes': '500',
-			'filter_size': '23,33'
-		}
-		host = 'test.synology.me'
-		cls = httpx(host, **run_opts)
-		expected_cmd = f"httpx {DEFAULT_HTTPX_FLAGS} -u {host} -json -header 'Content-Type: application/xml' -delay 1s -rate-limit 120 -threads 10 -timeout 1 -filter-code 500 -filter-length 23,33"
-		self.assertEqual(cls.cmd, expected_cmd)
-		self.assertEqual(cls.print_line, False)
-		self.assertEqual(cls.print_item, False)
-		self.assertEqual(cls.print_item_count, False)
-		self.assertEqual(cls.print_cmd, False)
-		self.assertEqual(cls.print_cmd_prefix, False)
-		self.assertEqual(cls.output_json, True)
-
-	def test_httpx_build_cmd_with_opts_with_prefix(self):
-		if not httpx in TEST_TASKS:
-			return
-		run_opts = {
-			FOLLOW_REDIRECT: False,
-			DELAY: 1,
-			RATE_LIMIT: 120,
-			THREADS: 10,
-			TIMEOUT: 1,
-			HEADER: 'Content-Type: application/xml',
-			MATCH_CODES: False, # intentionally omit arg, overriding default value
-			'filter_code': '200',
-			'filter_length': 50,
-			'httpx.filter_codes': '500',    # prefixed option keys should override
-			'httpx_filter_size': '23,33' # prefixed option keys should override
-		}
-		host = 'test.synology.me'
-		cls = httpx(host, **run_opts)
-		expected_cmd = f"httpx {DEFAULT_HTTPX_FLAGS} -u {host} -json -header 'Content-Type: application/xml' -delay 1s -rate-limit 120 -threads 10 -timeout 1 -filter-code 500 -filter-length 23,33"
-		self.assertEqual(cls.cmd, expected_cmd)
-		self.assertEqual(cls.print_line, False)
-		self.assertEqual(cls.print_item, False)
-		self.assertEqual(cls.print_item_count, False)
-		self.assertEqual(cls.print_cmd, False)
-		self.assertEqual(cls.print_cmd_prefix, False)
-		self.assertEqual(cls.output_json, True)
-
-
-class TestCommandRun(unittest.TestCase, CommandOutputTester):
-
-	def setUp(self):
-		warnings.simplefilter('ignore', category=ResourceWarning)
-		warnings.simplefilter('ignore', category=DeprecationWarning)
-
-	def _valid_fixture(self, cls, fixture):
-		if not fixture:
-			if len(FIXTURES_TASKS.keys()) == 1: # make test fail.
-				raise AssertionError(f'No fixture for {cls.__name__}! Add one to the tests/fixtures directory (must not be an empty file / empty json / empty list).')
-			console.print(f'[dim gold3] skipped (no fixture)[/]')
-			return False
-		return True
-
-	def test_cmd_converted_schema(self):
-		console.print('')
-
-		for cls, fixture in FIXTURES_TASKS.items():
-			console.print(f'\t[bold grey35]{cls.__name__} ...[/] ', end='')
-			with self.subTest(name=cls.__name__):
-
-				# Validate fixture
-				if not self._valid_fixture(cls, fixture):
-					continue
-
-				# Run command
-				targets = INPUTS_TASKS[cls.input_type]
-
-				with mock_command(cls, targets, META_OPTS, fixture, 'run') as results:
-					self._test_task_output(
-						results,
-						expected_output_types=cls.output_types)
-
-	def test_cmd_original_schema(self):
-		console.print('')
-		for cls, fixture in FIXTURES_TASKS.items():
-
-			with self.subTest(name=cls.__name__):
-				console.print(f'\t[bold grey35]{cls.__name__} ...[/]', end='')
-
-				# Validate fixture
-				if not self._valid_fixture(cls, fixture):
-					continue
-
-				# Get expected output keys from fixture
-				expected_output_keys = None
-				if isinstance(fixture, dict):
-					if 'results' in fixture: # fix for JSON files having a 'results' key
-						expected_output_keys = fixture['results'][0].keys()
-					else:
-						expected_output_keys = fixture.keys()
-
-				# Run command
-				targets = INPUTS_TASKS[cls.input_type]
-				opts = copy.deepcopy(META_OPTS)
-				opts.update({
-					'orig': True,
-					'raw': isinstance(fixture, str)
-				})
-				with mock_command(cls, targets, opts, fixture, 'run') as results:
-					if not len(cls.output_types) == 1:
-						console.print(f'[dim gold3] skipped (multi-output task with single schema)[/]')
-						return
-					self._test_task_output(
-						results,
-						expected_output_keys=expected_output_keys)
-
-
-class TestCommandHooks(unittest.TestCase):
-
-	def test_cmd_hooks(self):
-		if not httpx in TEST_TASKS:
-			return
-
-		def on_item_pre_convert(self, item):
-			item['url'] = 'test_changed_url'
-			return item
-
-		def on_item(self, item):
-			item.status_code = 500
-			return item
-
-		def on_end(self):
-			self.results = [{'url': 'test_changed_result'}]
-
-		def on_init(self):
-			self.cmd = 'test_changed_cmd_init'
-			self.run_opts = {}
-
-		def on_start(self):
-			self.cmd = 'test_changed_cmd_start'
-
-		hooks = {
-			'on_init': [on_init],
-			'on_start': [on_start],
-			'on_end': [on_end],
-			'on_item_pre_convert': [on_item_pre_convert],
-			'on_item': [on_item],
-			'on_end': [on_end],
-		}
-		fixture = load_fixture('httpx_output', FIXTURES_DIR)
-		with mock_subprocess_popen([json.dumps(fixture)]):
-			input = INPUTS_TASKS[HOST]
-			cls = httpx(input, hooks=hooks)
-			self.assertEqual(cls.cmd.split(' ')[0], 'test_changed_cmd_init')
-			items = cls.run()
-			item = items[0]
-			self.assertEqual(item.status_code, 500)
-			self.assertEqual(item.url, 'test_changed_url')
-			self.assertEqual(cls.cmd.split(' ')[0], 'test_changed_cmd_start')
-			self.assertEqual(cls.results, [{'url': 'test_changed_result'}])
-
-	def test_cmd_failed_hook(self):
-		if not httpx in TEST_TASKS:
-			return
-
-		def on_init(self):
-			raise Exception('Test passed')
-
-		hooks = {
-			'on_init': [on_init]
-		}
-		fixture = FIXTURES_TASKS[httpx]
-		with mock_subprocess_popen([json.dumps(fixture)]):
-			with self.assertRaises(Exception, msg='Test passed'):
-				input = INPUTS_TASKS[HOST]
-				httpx(input, hooks=hooks)

tests/unit/test_workflows.py

@@ -1,96 +0,0 @@
-from celery import chain
-from secator.celery import app
-from secator.tasks import httpx
-import unittest
-import json
-from secator.definitions import DEBUG
-from secator.utils_test import mock_command, FIXTURES_TASKS, TEST_TASKS
-from secator.celery import forward_results
-from secator.rich import console
-
-TARGETS = ['bing.com', 'google.com', 'wikipedia.org', 'ibm.com', 'cnn.com', 'karate.com']
-
-
-class TestAdHocWorkflow(unittest.TestCase):
-
-	def test_chain(self):
-		if not httpx in TEST_TASKS:
-			return
-
-		with mock_command(httpx, fixture=[FIXTURES_TASKS[httpx]] * len(TARGETS)):
-			sigs = [forward_results.si([])] + [httpx.s(target) for target in TARGETS]
-			workflow = chain(*sigs)
-			result = workflow.apply()
-			results = result.get()
-			if DEBUG > 1:
-				console.print_json(json.dumps(results))
-			urls = [r.url for r in results]
-			self.assertEqual(len(urls), len(TARGETS))
-
-	# def test_chain_with_results(self):
-	# 	existing_results = [{
-	# 		"url": "https://example.synology.me",
-	# 		"method": "GET",
-	# 		"status_code": 200,
-	# 		"words": 438,
-	# 		"lines": 136,
-	# 		"content_type":
-	# 		"text/html",
-	# 		"content_length": 11577,
-	# 		"host": "82.66.157.114",
-	# 		"time": 0.16246860100000002,
-	# 		"_source": "httpx",
-	# 		"_type": "url"
-	# 	}]
-	# 	with mock_command(httpx, fixture=[FIXTURES_TASKS[httpx]] * len(TARGETS)):
-	# 		sigs = [httpx.s(target) for target in TARGETS]
-	# 		sigs = [forward_results.si(existing_results)] + sigs
-	# 		workflow = chain(*sigs)
-	# 		result = workflow.apply()
-	# 		results = result.get()
-	# 		if DEBUG:
-	# 			console.print_json(json.dumps(results))
-	# 		urls = [r['url'] for r in results]
-	# 		self.assertEqual(len(urls), len(TARGETS))
-	# 		self.assertIn(existing_results[0], results)
-
-	# def test_complex_workflow():
-	# 	targets = ['bing.com', 'google.com', 'wikipedia.org', 'ibm.com', 'cnn.com', 'karate.com']
-	# 	task = chain(
-	# 		forward_results.s([]),
-	# 		httpx().s(targets[0]),
-	# 		chord((
-	# 			httpx().s(targets[1]),
-	# 			httpx().s(targets[2]),
-	# 		), forward_results.s()),
-	# 		httpx().s(targets[3]),
-	# 		chord((
-	# 			httpx().s(targets[4]),
-	# 			httpx().s(targets[5]),
-	# 		), forward_results.s())
-	# 	)
-	# 	workflow = task.delay()
-	# 	results = workflow.get()
-	# 	urls = [r['url'] for r in results]
-	# 	print(urls)
-	# 	return workflow
-
-
-	# def test_nested_collect():
-	# 	console.log(task)
-	# 	workflow = task.delay()
-	# 	results = workflow.get()
-	# 	console.print_item(json.dumps(results))
-	# 	# results = get_results(workflow)
-	# 	# console.log(results)
-	# 	# console.log([r['url'] for r in results])
-	# 	# urls = [r['url'] for r in results]
-	# 	# for target in targets:
-	# 	#     assert any(target in url for url in urls)
-	# 	return workflow
-
-	# 	# Polling approach
-	# 	# for task_id, name, result in poll_task(find_root_task(workflow), seen):
-	# 	#     print(task_id, name, result)
-	# 	#     results.append(result)
-	# 	# print([r for r in results if r._type == 'url'])