secator 0.4.1__tar.gz → 0.5.0__tar.gz

{secator-0.4.1 → secator-0.5.0}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [0.5.0](https://github.com/freelabz/secator/compare/v0.4.1...v0.5.0) (2024-05-03)
+
+
+### Features
+
+* add searchsploit output fields ([#278](https://github.com/freelabz/secator/issues/278)) ([00872c4](https://github.com/freelabz/secator/commit/00872c4a7f9b1ec76ee1bfd7a00919d53cbdb30a))
+* **cli:** add report list / export commands ([#367](https://github.com/freelabz/secator/issues/367)) ([ab396a3](https://github.com/freelabz/secator/commit/ab396a3098c6d4c46cf9c9b29bd5c54579421646))
+* **config:** load external tasks from template dir ([#373](https://github.com/freelabz/secator/issues/373)) ([0c63c02](https://github.com/freelabz/secator/commit/0c63c02c8eca477a6752f4af466c4303801019de))
+
+
+### Bug Fixes
+
+* **cli:** catch JSON parse errors ([#378](https://github.com/freelabz/secator/issues/378)) ([5e3d7f2](https://github.com/freelabz/secator/commit/5e3d7f2d2938a857e7599a429a6cfabf3b12347b))
+* **nmap:** resolve -sS tcp syn stealth issue ([#376](https://github.com/freelabz/secator/issues/376)) ([a3efc65](https://github.com/freelabz/secator/commit/a3efc651dfa4d8fa34d611b9aea2e156352fdc45))
+
 ## [0.4.1](https://github.com/freelabz/secator/compare/v0.4.0...v0.4.1) (2024-04-30)
 
 

{secator-0.4.1 → secator-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: secator
-Version: 0.4.1
+Version: 0.5.0
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.4.1 → secator-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "secator"
-version = "0.4.1"
+version = "0.5.0"
 authors = [{ name = "FreeLabz", email = "sales@freelabz.com" }]
 readme = "README.md"
 description = "The pentester's swiss knife."

{secator-0.4.1 → secator-0.5.0}/secator/cli.py

@@ -20,7 +20,8 @@ from secator.decorators import OrderedGroup, register_runner
 from secator.definitions import ADDONS_ENABLED, ASCII, DEV_PACKAGE, OPT_NOT_SUPPORTED, VERSION
 from secator.installer import ToolInstaller, fmt_health_table_row, get_health_table, get_version_info
 from secator.rich import console
-from secator.runners import Command
+from secator.runners import Command, Runner
+from secator.report import Report
 from secator.serializers.dataclass import loads_dataclass
 from secator.utils import debug, detect_host, discover_tasks, flatten, print_results_table, print_version
 
@@ -536,17 +537,72 @@ def report():
 
 @report.command('show')
 @click.argument('json_path')
+@click.option('-o', '--output', type=str, default='console', help='Format')
 @click.option('-e', '--exclude-fields', type=str, default='', help='List of fields to exclude (comma-separated)')
-def report_show(json_path, exclude_fields):
-	"""Show a JSON report as a nicely-formatted table."""
+def report_show(json_path, output, exclude_fields):
+	"""Show a JSON report."""
 	with open(json_path, 'r') as f:
 		report = loads_dataclass(f.read())
 		results = flatten(list(report['results'].values()))
-	exclude_fields = exclude_fields.split(',')
-	print_results_table(
-		results,
-		title=report['info']['title'],
-		exclude_fields=exclude_fields)
+	if output == 'console':
+		for result in results:
+			console.print(result)
+	elif output == 'table':
+		exclude_fields = exclude_fields.split(',')
+		print_results_table(
+			results,
+			title=report['info']['title'],
+			exclude_fields=exclude_fields)
+
+
+@report.command('list')
+@click.option('-ws', '--workspace', type=str)
+def report_list(workspace):
+	reports_dir = CONFIG.dirs.reports
+	json_reports = reports_dir.glob("**/**/report.json")
+	ws_reports = {}
+	for path in json_reports:
+		ws, runner, number = str(path).split('/')[-4:-1]
+		if ws not in ws_reports:
+			ws_reports[ws] = []
+		with open(path, 'r') as f:
+			try:
+				content = json.loads(f.read())
+				data = {'path': path, 'name': content['info']['name'], 'runner': runner}
+				ws_reports[ws].append(data)
+			except json.JSONDecodeError as e:
+				console.print(f'[bold red]Could not load {path}: {str(e)}')
+
+	for ws in ws_reports:
+		if workspace and not ws == workspace:
+			continue
+		console.print(f'[bold gold3]{ws}:')
+		for data in sorted(ws_reports[ws], key=lambda x: x['path']):
+			console.print(f'   • {data["path"]} ([bold blue]{data["name"]}[/] [dim]{data["runner"][:-1]}[/])')
+
+
+@report.command('export')
+@click.argument('json_path', type=str)
+@click.option('--output-folder', '-of', type=str)
+@click.option('-output', '-o', type=str)
+def report_export(json_path, output_folder, output):
+	with open(json_path, 'r') as f:
+		data = loads_dataclass(f.read())
+		flatten(list(data['results'].values()))
+
+	runner_instance = DotMap({
+		"config": {
+			"name": data['info']['name']
+		},
+		"workspace_name": json_path.split('/')[-4],
+		"reports_folder": output_folder or Path.cwd(),
+		"data": data,
+		"results": flatten(list(data['results'].values()))
+	})
+	exporters = Runner.resolve_exporters(output)
+	report = Report(runner_instance, title=data['info']['title'], exporters=exporters)
+	report.data = data
+	report.send()
 
 
 #--------#
@@ -1052,5 +1108,5 @@ def integration(tasks, workflows, scans, test, debug):
 @test.command()
 def coverage():
 	"""Run coverage report."""
-	cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*'
+	cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*,*/templates/*'
 	run_test(cmd, 'coverage')

{secator-0.4.1 → secator-0.5.0}/secator/output_types/exploit.py

@@ -34,6 +34,9 @@ class Exploit(OutputType):
 	]
 	_sort_by = ('matched_at', 'name')
 
+	def __str__(self):
+		return self.name
+
 	def __repr__(self):
 		s = f'[bold red]⍼[/]  \[[bold red]{self.name}'
 		if self.reference:

{secator-0.4.1 → secator-0.5.0}/secator/output_types/vulnerability.py

@@ -89,11 +89,5 @@ class Vulnerability(OutputType):
 			s = f'[dim]{s}[/]'
 		return rich_to_ansi(s)
 
-	# def __gt__(self, other):
-	# 	# favor httpx over other url info tools
-	# 	if self._source == 'httpx' and other._source != 'httpx':
-	# 		return True
-	# 	return super().__gt__(other)
-
 	def __str__(self):
 		return self.matched_at + ' -> ' + self.name

{secator-0.4.1 → secator-0.5.0}/secator/report.py

@@ -1,7 +1,7 @@
 import operator
 
 from secator.output_types import OUTPUT_TYPES, OutputType
-from secator.utils import merge_opts, get_file_timestamp, print_results_table
+from secator.utils import merge_opts, get_file_timestamp
 from secator.rich import console
 
 
@@ -22,9 +22,6 @@ class Report:
 		self.workspace_name = runner.workspace_name
 		self.output_folder = runner.reports_folder
 
-	def as_table(self):
-		print_results_table(self.results, self.title)
-
 	def send(self):
 		for report_cls in self.exporters:
 			try:

{secator-0.4.1 → secator-0.5.0}/secator/runners/_base.py

@@ -92,7 +92,6 @@ class Runner:
 		self.workspace_name = context.get('workspace_name', 'default')
 		self.run_opts = run_opts.copy()
 		self.sync = run_opts.get('sync', True)
-		self.exporters = self.resolve_exporters()
 		self.done = False
 		self.start_time = datetime.fromtimestamp(time())
 		self.last_updated = None
@@ -109,6 +108,10 @@ class Runner:
 		self.uuids = []
 		self.celery_result = None
 
+		# Determine exporters
+		exporters_str = self.run_opts.get('output') or self.default_exporters
+		self.exporters = Runner.resolve_exporters(exporters_str)
+
 		# Determine report folder
 		default_reports_folder_base = f'{CONFIG.dirs.reports}/{self.workspace_name}/{self.config.type}s'
 		_id = get_task_folder_id(default_reports_folder_base)
@@ -294,7 +297,7 @@ class Runner:
 		debug('running duplicate check', id=self.config.name, sub='runner.mark_duplicates')
 		dupe_count = 0
 		for item in self.results:
-			debug('running duplicate check', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5)
+			# debug('running duplicate check', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5)
 			others = [f for f in self.results if f == item and f._uuid != item._uuid]
 			if others:
 				main = max(item, *others)
@@ -303,6 +306,7 @@ class Runner:
 				main._related.extend([dupe._uuid for dupe in dupes])
 				main._related = list(dict.fromkeys(main._related))
 				if main._uuid != item._uuid:
+					debug(f'found {len(others)} duplicates for', obj=item.toDict(), obj_breaklines=True, sub='runner.mark_duplicates', level=5)  # noqa: E501
 					item._duplicate = True
 					item = self.run_hooks('on_item', item)
 					if item._uuid not in main._related:
@@ -390,19 +394,19 @@ class Runner:
 				return False
 		return True
 
-	def resolve_exporters(self):
+	@staticmethod
+	def resolve_exporters(exporters):
 		"""Resolve exporters from output options."""
-		output = self.run_opts.get('output') or self.default_exporters
-		if not output or output in ['false', 'False']:
+		if not exporters or exporters in ['false', 'False']:
 			return []
-		if isinstance(output, str):
-			output = output.split(',')
-		exporters = [
+		if isinstance(exporters, str):
+			exporters = exporters.split(',')
+		classes = [
 			import_dynamic(f'secator.exporters.{o.capitalize()}Exporter', 'Exporter')
-			for o in output
+			for o in exporters
 			if o
 		]
-		return [e for e in exporters if e]
+		return [cls for cls in classes if cls]
 
 	def log_start(self):
 		"""Log runner start."""

secator-0.5.0/secator/tasks/__init__.py

@@ -0,0 +1,8 @@
+from secator.utils import discover_tasks
+TASKS = discover_tasks()
+__all__ = [
+    cls.__name__
+    for cls in TASKS
+]
+for cls in TASKS:
+    exec(f'from .{cls.__name__} import {cls.__name__}')

{secator-0.4.1 → secator-0.5.0}/secator/tasks/nmap.py

@@ -51,7 +51,8 @@ class nmap(VulnMulti):
 
 		# Nmap opts
 		PORTS: '-p',
-		'output_path': '-oX'
+		'output_path': '-oX',
+		'tcp_syn_stealth': '-sS'
 	}
 	opt_value_map = {
 		PORTS: lambda x: ','.join([str(p) for p in x]) if isinstance(x, list) else x
@@ -73,6 +74,10 @@ class nmap(VulnMulti):
 			output_path = f'{self.reports_folder}/.outputs/{self.unique_name}.xml'
 		self.output_path = output_path
 		self.cmd += f' -oX {self.output_path}'
+		tcp_syn_stealth = self.get_opt_value('tcp_syn_stealth')
+		if tcp_syn_stealth:
+			self.cmd = f'sudo {self.cmd}'
+			self.cmd = self.cmd.replace('-sT', '')
 
 	def yielder(self):
 		yield from super().yielder()

{secator-0.4.1 → secator-0.5.0}/secator/tasks/searchsploit.py

@@ -1,3 +1,5 @@
+import re
+
 from secator.decorators import task
 from secator.definitions import (CVES, EXTRA_DATA, ID, MATCHED_AT, NAME,
 								 PROVIDER, REFERENCE, TAGS, OPT_NOT_SUPPORTED)
@@ -5,6 +7,9 @@ from secator.output_types import Exploit
 from secator.runners import Command
 
 
+SEARCHSPLOIT_TITLE_REGEX = re.compile(r'^((?:[a-zA-Z\-_!\.()]+\d?\s?)+)\.?\s*(.*)$')
+
+
 @task()
 class searchsploit(Command):
 	"""Exploit-DB command line search tool."""
@@ -19,12 +24,15 @@ class searchsploit(Command):
 	output_types = [Exploit]
 	output_map = {
 		Exploit: {
-			NAME: lambda x: '-'.join(x['Title'].split('-')[1:]).strip(),
-			PROVIDER: lambda x: 'EDB',
+			NAME: 'Title',
 			ID: 'EDB-ID',
+			PROVIDER: lambda x: 'EDB',
 			CVES: lambda x: [c for c in x['Codes'].split(';') if c.startswith('CVE-')],
 			REFERENCE: lambda x: f'https://exploit-db.com/exploits/{x["EDB-ID"]}',
-			EXTRA_DATA: lambda x: {'verified': x['Verified']}
+			TAGS: lambda x: searchsploit.tags_extractor(x),
+			EXTRA_DATA: lambda x: {
+				k.lower().replace('date_', ''): v for k, v in x.items() if k not in ['Title', 'EDB-ID', 'Codes', 'Tags', 'Source'] and v != ''  # noqa: E501
+			}
 		}
 	}
 	install_cmd = 'sudo git clone https://gitlab.com/exploit-database/exploitdb.git /opt/exploitdb || true && sudo ln -sf /opt/exploitdb/searchsploit /usr/local/bin/searchsploit'  # noqa: E501
@@ -34,6 +42,18 @@ class searchsploit(Command):
 	input_chunk_size = 1
 	profile = 'io'
 
+	@staticmethod
+	def tags_extractor(item):
+		tags = []
+		for tag in item['Tags'].split(','):
+			_tag = '_'.join(
+				tag.lower().replace('-', '_',).replace('(', '').replace(')', '').split(' ')
+			)
+			if not _tag:
+				continue
+			tags.append(tag)
+		return tags
+
 	@staticmethod
 	def before_init(self):
 		_in = self.input
@@ -49,5 +69,23 @@ class searchsploit(Command):
 	def on_item_pre_convert(self, item):
 		if self.matched_at:
 			item[MATCHED_AT] = self.matched_at
-		item[TAGS] = [self.input.replace('\'', '')]
+		return item
+
+	@staticmethod
+	def on_item(self, item):
+		match = SEARCHSPLOIT_TITLE_REGEX.match(item.name)
+		# if not match:
+		# 	self._print(f'[bold red]{item.name} ({item.reference}) did not match SEARCHSPLOIT_TITLE_REGEX. Please report this issue.[/]')  # noqa: E501
+		if match:
+			group = match.groups()
+			product = '-'.join(group[0].strip().split(' '))
+			if len(group[1]) > 1:
+				versions, title = tuple(group[1].split(' - '))
+				item.name = title
+				product_info = [f'{product.lower()} {v.strip()}' for v in versions.split('/')]
+				item.tags = product_info + item.tags
+			# else:
+			# 	self._print(f'[bold red]{item.name} ({item.reference}) did not quite match SEARCHSPLOIT_TITLE_REGEX. Please report this issue.[/]')  # noqa: E501
+		input_tag = '-'.join(self.input.replace('\'', '').split(' '))
+		item.tags = [input_tag] + item.tags
 		return item

{secator-0.4.1 → secator-0.5.0}/secator/utils.py

@@ -1,4 +1,5 @@
 import inspect
+import importlib
 import itertools
 import logging
 import operator
@@ -8,7 +9,7 @@ import select
 import sys
 import warnings
 from datetime import datetime
-from importlib import import_module
+
 from inspect import isclass
 from pathlib import Path
 from pkgutil import iter_modules
@@ -138,7 +139,7 @@ def discover_internal_tasks():
 		if module_name.startswith('_'):
 			continue
 		try:
-			module = import_module(f'secator.tasks.{module_name}')
+			module = importlib.import_module(f'secator.tasks.{module_name}')
 		except ImportError as e:
 			console.print(f'[bold red]Could not import secator.tasks.{module_name}:[/]')
 			console.print(f'\t[bold red]{type(e).__name__}[/]: {str(e)}')
@@ -160,17 +161,32 @@ def discover_internal_tasks():
 
 def discover_external_tasks():
 	"""Find external secator tasks."""
-	if not os.path.exists('config.secator'):
-		return []
-	with open('config.secator', 'r') as f:
-		classes = f.read().splitlines()
 	output = []
-	for cls_path in classes:
-		cls = import_dynamic(cls_path, cls_root='Command')
-		if not cls:
-			continue
-		# logger.warning(f'Added external tool {cls_path}')
-		output.append(cls)
+	sys.dont_write_bytecode = True
+	for path in CONFIG.dirs.templates.glob('**/*.py'):
+		try:
+			task_name = path.stem
+			module_name = f'secator.tasks.{task_name}'
+
+			# console.print(f'Importing module {module_name} from {path}')
+			spec = importlib.util.spec_from_file_location(module_name, path)
+			module = importlib.util.module_from_spec(spec)
+			# console.print(f'Adding module "{module_name}" to sys path')
+			sys.modules[module_name] = module
+
+			# console.print(f'Executing module "{module}"')
+			spec.loader.exec_module(module)
+
+			# console.print(f'Checking that {module} contains task {task_name}')
+			if not hasattr(module, task_name):
+				console.print(f'[bold orange1]Could not load external task "{task_name}" from module {path.name}[/] ({path})')
+				continue
+			cls = getattr(module, task_name)
+			console.print(f'[bold green]Successfully loaded external task "{task_name}"[/] ({path})')
+			output.append(cls)
+		except Exception as e:
+			console.print(f'[bold red]Could not load external module {path.name}. Reason: {str(e)}.[/] ({path})')
+	sys.dont_write_bytecode = False
 	return output
 
 
@@ -194,7 +210,7 @@ def import_dynamic(cls_path, cls_root='Command'):
 	"""
 	try:
 		package, name = cls_path.rsplit(".", maxsplit=1)
-		cls = getattr(import_module(package), name)
+		cls = getattr(importlib.import_module(package), name)
 		root_cls = inspect.getmro(cls)[-2]
 		if root_cls.__name__ == cls_root:
 			return cls

{secator-0.4.1 → secator-0.5.0}/secator/utils_test.py

@@ -1,6 +1,7 @@
 import contextlib
 import json
 import os
+import sys
 import unittest.mock
 
 from fp.fp import FreeProxy
@@ -182,3 +183,15 @@ class CommandOutputTester:  # Mixin for unittest.TestCase
 			raise
 
 		console.print('[bold green] ok[/]')
+
+
+def clear_modules():
+	"""Clear all secator modules imports.
+	See https://stackoverflow.com/questions/7460363/re-import-module-under-test-to-lose-context for context.
+	"""
+	keys_to_delete = []
+	for k, _ in sys.modules.items():
+		if k.startswith('secator'):
+			keys_to_delete.append(k)
+	for k in keys_to_delete:
+		del sys.modules[k]

secator-0.5.0/tests/fixtures/ls.py

@@ -0,0 +1,36 @@
+from secator.runners import Command
+from secator.decorators import task
+from secator.output_types import Vulnerability
+
+
+@task()
+class ls(Command):
+    cmd = 'ls -al'
+    output_types = [Vulnerability]
+    output_map = {
+        Vulnerability: {}
+	}
+
+    @staticmethod
+    def item_loader(self, line):
+        fields = ['permissions', 'link_count', 'owner', 'group', 'size', 'month', 'day', 'hour', 'path']
+        result = [c for c in line.split(' ') if c]
+        if len(result) != len(fields):
+            return None
+        data = {}
+        for ix, value in enumerate(result):
+            data[fields[ix]] = value
+
+        # Output vulnerabilities
+        permissions = data['permissions']
+        path = data['path']
+        full_path = f'{self.input}/{path}'
+        if permissions[-2] == 'w':  # found a vulnerability !
+            yield Vulnerability(
+                name='World-writeable path',
+                severity='high',
+                confidence='high',
+                provider='ls',
+                matched_at=full_path,
+                extra_data={k: v for k, v in data.items() if k != 'path'}
+            )

{secator-0.4.1 → secator-0.5.0}/tests/unit/test_offline.py

@@ -1,17 +1,12 @@
 import os
-import sys
 import unittest
 
+from secator.utils_test import clear_modules
+
 
 class TestOffline(unittest.TestCase):
 	def setUp(self):
-		try:
-			# This allows to drop the secator module loaded from other tests in order to reload the config with modified
-   			# environment variables.
-			# See https://stackoverflow.com/questions/7460363/re-import-module-under-test-to-lose-context for context.
-			del sys.modules['secator']
-		except KeyError:
-			pass
+		clear_modules()
 		os.environ['SECATOR_OFFLINE_MODE'] = '1'
 
 	def test_offline_cve_lookup(self):

secator-0.5.0/tests/unit/test_template.py

@@ -0,0 +1,53 @@
+import unittest
+from secator.config import CONFIG
+from secator.output_types import Vulnerability
+from secator.utils_test import FIXTURES_DIR, clear_modules
+import os
+
+import shutil
+
+
+class TestTemplate(unittest.TestCase):
+	def setUp(self):
+		self.template_dir = CONFIG.dirs.templates
+		self.custom_task_path = self.template_dir / 'ls.py'
+		self.writeable_file = self.template_dir / 'test.txt'
+		self.custom_workflow_path = self.template_dir / 'ls.yml'
+		shutil.copy(f'{FIXTURES_DIR}/ls.py', self.custom_task_path)
+		shutil.copy(f'{FIXTURES_DIR}/ls.yml', self.custom_workflow_path)
+		self.writeable_file.touch()
+		os.chmod(self.writeable_file, 0o007)
+		self.expected_vuln = Vulnerability(
+			name='World-writeable path',
+			severity='high',
+			confidence='high',
+			provider='ls',
+			matched_at=f'{str(self.writeable_file)}',
+			_source='ls',
+		)
+		clear_modules()
+		self.maxDiff = None
+
+	def tearDown(self):
+		self.custom_task_path.unlink()
+		self.custom_workflow_path.unlink()
+		self.writeable_file.unlink()
+
+	def test_external_task(self):
+		from secator.tasks import ls
+		results = ls(str(self.template_dir)).run()
+		self.assertEqual(len(results), 1)
+		self.assertTrue(self.expected_vuln == Vulnerability.load(results[0].toDict()))
+
+	def test_external_workflow(self):
+		from secator.cli import ALL_WORKFLOWS
+		from secator.runners import Workflow
+		ls_workflow = None
+		for w in ALL_WORKFLOWS:
+			if w.name == 'ls':
+				ls_workflow = w
+		self.assertIsNotNone(ls_workflow)
+		results = Workflow(ls_workflow, targets=[str(self.template_dir)]).run()
+		self.assertEqual(len(results), 2)
+		self.assertTrue(self.expected_vuln == Vulnerability.load(results[1].toDict()))
+

secator-0.4.1/secator/tasks/__init__.py

@@ -1,10 +0,0 @@
-from secator.utils import discover_internal_tasks, discover_external_tasks
-INTERNAL_TASKS = discover_internal_tasks()
-EXTERNAL_TASKS = discover_external_tasks()
-ALL_TASKS = INTERNAL_TASKS + EXTERNAL_TASKS
-__all__ = [
-    cls.__name__
-    for cls in ALL_TASKS
-]
-for cls in INTERNAL_TASKS:
-    exec(f'from .{cls.__name__} import {cls.__name__}')