secator 0.0.1__py3-none-any.whl → 0.3.5__py3-none-any.whl

secator/decorators.py

@@ -1,11 +1,11 @@
+import sys
 from collections import OrderedDict
 
 import rich_click as click
 from rich_click.rich_click import _get_rich_console
 from rich_click.rich_group import RichGroup
 
-from secator.celery import is_celery_worker_alive
-from secator.definitions import OPT_NOT_SUPPORTED
+from secator.definitions import ADDONS_ENABLED, OPT_NOT_SUPPORTED
 from secator.runners import Scan, Task, Workflow
 from secator.utils import (deduplicate, expand_input, get_command_category,
 						   get_command_cls)
@@ -38,18 +38,47 @@ class OrderedGroup(RichGroup):
 		super(OrderedGroup, self).__init__(name, commands, **attrs)
 		self.commands = commands or OrderedDict()
 
+	def command(self, *args, **kwargs):
+		"""Behaves the same as `click.Group.command()` but supports aliases.
+		"""
+		def decorator(f):
+			aliases = kwargs.pop("aliases", None)
+			if aliases:
+				max_width = _get_rich_console().width
+				aliases_str = ', '.join(f'[bold cyan]{alias}[/]' for alias in aliases)
+				padding = max_width // 4
+
+				name = kwargs.pop("name", None)
+				if not name:
+					raise click.UsageError("`name` command argument is required when using aliases.")
+
+				f.__doc__ = f.__doc__ or '\0'.ljust(padding+1)
+				f.__doc__ = f'{f.__doc__:<{padding}}[dim](aliases)[/] {aliases_str}'
+				base_command = super(OrderedGroup, self).command(
+					name, *args, **kwargs
+				)(f)
+				for alias in aliases:
+					cmd = super(OrderedGroup, self).command(alias, *args, hidden=True, **kwargs)(f)
+					cmd.help = f"Alias for '{name}'.\n\n{cmd.help}"
+					cmd.params = base_command.params
+
+			else:
+				cmd = super(OrderedGroup, self).command(*args, **kwargs)(f)
+
+			return cmd
+		return decorator
+
 	def group(self, *args, **kwargs):
-		"""Behaves the same as `click.Group.group()` except if passed
-		a list of names, all after the first will be aliases for the first.
+		"""Behaves the same as `click.Group.group()` but supports aliases.
 		"""
 		def decorator(f):
 			aliases = kwargs.pop('aliases', [])
 			aliased_group = []
 			if aliases:
 				max_width = _get_rich_console().width
-				# we have a list so create group aliases
 				aliases_str = ', '.join(f'[bold cyan]{alias}[/]' for alias in aliases)
 				padding = max_width // 4
+				f.__doc__ = f.__doc__ or '\0'.ljust(padding+1)
 				f.__doc__ = f'{f.__doc__:<{padding}}[dim](aliases)[/] {aliases_str}'
 				for alias in aliases:
 					grp = super(OrderedGroup, self).group(
@@ -143,6 +172,7 @@ def decorate_command_options(opts):
 		for opt_name, opt_conf in reversed_opts.items():
 			conf = opt_conf.copy()
 			short = conf.pop('short', None)
+			conf.pop('internal', False)
 			conf.pop('prefix', None)
 			long = f'--{opt_name}'
 			short = f'-{short}' if short else f'-{opt_name}'
@@ -185,6 +215,7 @@ def register_runner(cli_endpoint, config):
 			short_help += f' [dim]alias: {config.alias}'
 		fmt_opts['print_start'] = True
 		fmt_opts['print_run_summary'] = True
+		fmt_opts['print_progress'] = False
 		runner_cls = Scan
 
 	elif cli_endpoint.name == 'workflow':
@@ -199,6 +230,7 @@ def register_runner(cli_endpoint, config):
 			short_help = f'{short_help:<55} [dim](alias)[/][bold cyan] {config.alias}'
 		fmt_opts['print_start'] = True
 		fmt_opts['print_run_summary'] = True
+		fmt_opts['print_progress'] = False
 		runner_cls = Workflow
 
 	elif cli_endpoint.name == 'task':
@@ -243,11 +275,12 @@ def register_runner(cli_endpoint, config):
 		# opts.update(unknown_opts)
 		targets = opts.pop(input_type)
 		targets = expand_input(targets)
-		if sync or show:
+		if sync or show or not ADDONS_ENABLED['worker']:
 			sync = True
 		elif worker:
 			sync = False
 		else:  # automatically run in worker if it's alive
+			from secator.celery import is_celery_worker_alive
 			sync = not is_celery_worker_alive()
 		opts['sync'] = sync
 		opts.update({
@@ -260,6 +293,9 @@ def register_runner(cli_endpoint, config):
 		# Build hooks from driver name
 		hooks = {}
 		if driver == 'mongodb':
+			if not ADDONS_ENABLED['mongo']:
+				_get_rich_console().print('[bold red]Missing MongoDB dependencies: please run `secator install addons mongodb`[/].')
+				sys.exit(1)
 			from secator.hooks.mongodb import MONGODB_HOOKS
 			hooks = MONGODB_HOOKS
 

secator/definitions.py

@@ -1,49 +1,45 @@
 #!/usr/bin/python
 
 import os
+import requests
+
+from dotenv import find_dotenv, load_dotenv
+from importlib.metadata import version
+
+from secator.rich import console
 
-from pkg_resources import get_distribution
-from dotenv import load_dotenv, find_dotenv
 load_dotenv(find_dotenv(usecwd=True), override=False)
 
 # Globals
-VERSION = get_distribution('secator').version
+VERSION = version('secator')
 ASCII = f"""
-                         __            
+			 __            
    ________  _________ _/ /_____  _____
   / ___/ _ \/ ___/ __ `/ __/ __ \/ ___/
  (__  /  __/ /__/ /_/ / /_/ /_/ / /    
 /____/\___/\___/\__,_/\__/\____/_/     v{VERSION}
 
-                    freelabz.com
+			freelabz.com
 """  # noqa: W605,W291
 
 # Secator folders
 ROOT_FOLDER = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-CONFIGS_FOLDER = ROOT_FOLDER + '/secator/configs'
+LIB_FOLDER = ROOT_FOLDER + '/secator'
+CONFIGS_FOLDER = LIB_FOLDER + '/configs'
 EXTRA_CONFIGS_FOLDER = os.environ.get('SECATOR_EXTRA_CONFIGS_FOLDER')
+BIN_FOLDER = os.environ.get('SECATOR_BIN_FOLDER', f'{os.path.expanduser("~")}/.local/bin')
 DATA_FOLDER = os.environ.get('SECATOR_DATA_FOLDER', f'{os.path.expanduser("~")}/.secator')
-TASKS_FOLDER = os.environ.get('SECATOR_TASKS_FOLDER', f'{DATA_FOLDER}/tasks')
 REPORTS_FOLDER = os.environ.get('SECATOR_REPORTS_FOLDER', f'{DATA_FOLDER}/reports')
-WORDLISTS_FOLDER = os.environ.get('SECATOR_WORDLISTS_FOLDER', '/usr/share/seclists')
+WORDLISTS_FOLDER = os.environ.get('SECATOR_WORDLISTS_FOLDER', f'{DATA_FOLDER}/wordlists')
 SCRIPTS_FOLDER = f'{ROOT_FOLDER}/scripts'
 CVES_FOLDER = f'{DATA_FOLDER}/cves'
 PAYLOADS_FOLDER = f'{DATA_FOLDER}/payloads'
 REVSHELLS_FOLDER = f'{DATA_FOLDER}/revshells'
-os.makedirs(DATA_FOLDER, exist_ok=True)
-os.makedirs(TASKS_FOLDER, exist_ok=True)
-os.makedirs(REPORTS_FOLDER, exist_ok=True)
-os.makedirs(WORDLISTS_FOLDER, exist_ok=True)
-os.makedirs(SCRIPTS_FOLDER, exist_ok=True)
-os.makedirs(CVES_FOLDER, exist_ok=True)
-os.makedirs(PAYLOADS_FOLDER, exist_ok=True)
-os.makedirs(REVSHELLS_FOLDER, exist_ok=True)
+TESTS_FOLDER = f'{ROOT_FOLDER}/tests'
 
 # Celery local fs folders
 CELERY_DATA_FOLDER = f'{DATA_FOLDER}/celery/data'
 CELERY_RESULTS_FOLDER = f'{DATA_FOLDER}/celery/results'
-os.makedirs(CELERY_DATA_FOLDER, exist_ok=True)
-os.makedirs(CELERY_RESULTS_FOLDER, exist_ok=True)
 
 # Environment variables
 DEBUG = int(os.environ.get('DEBUG', '0'))
@@ -57,11 +53,12 @@ CELERY_BROKER_VISIBILITY_TIMEOUT = int(os.environ.get('CELERY_BROKER_VISIBILITY_
 CELERY_OVERRIDE_DEFAULT_LOGGING = bool(int(os.environ.get('CELERY_OVERRIDE_DEFAULT_LOGGING', 1)))
 GOOGLE_DRIVE_PARENT_FOLDER_ID = os.environ.get('GOOGLE_DRIVE_PARENT_FOLDER_ID')
 GOOGLE_CREDENTIALS_PATH = os.environ.get('GOOGLE_CREDENTIALS_PATH')
+GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
 
 # Defaults HTTP and Proxy settings
 DEFAULT_SOCKS5_PROXY = os.environ.get('SOCKS5_PROXY', "socks5://127.0.0.1:9050")
 DEFAULT_HTTP_PROXY = os.environ.get('HTTP_PROXY', "https://127.0.0.1:9080")
-DEFAULT_STORE_HTTP_RESPONSES = bool(int(os.environ.get('STORE_HTTP_RESPONSES', 1)))
+DEFAULT_STORE_HTTP_RESPONSES = bool(int(os.environ.get('DEFAULT_STORE_HTTP_RESPONSES', 1)))
 DEFAULT_PROXYCHAINS_COMMAND = "proxychains"
 DEFAULT_FREEPROXY_TIMEOUT = 1  # seconds
 
@@ -74,11 +71,14 @@ DEFAULT_HTTPX_FLAGS = os.environ.get('DEFAULT_HTTPX_FLAGS', '-td')
 DEFAULT_KATANA_FLAGS = os.environ.get('DEFAULT_KATANA_FLAGS', '-jc -js-crawl -known-files all -or -ob')
 DEFAULT_NUCLEI_FLAGS = os.environ.get('DEFAULT_NUCLEI_FLAGS', '-stats -sj -si 20 -hm -or')
 DEFAULT_FEROXBUSTER_FLAGS = os.environ.get('DEFAULT_FEROXBUSTER_FLAGS', '--auto-bail --no-state')
-DEFAULT_PROGRESS_UPDATE_FREQUENCY = 10
+DEFAULT_PROGRESS_UPDATE_FREQUENCY = int(os.environ.get('DEFAULT_PROGRESS_UPDATE_FREQUENCY', 60))
+DEFAULT_SKIP_CVE_SEARCH = bool(int(os.environ.get('DEFAULT_SKIP_CVE_SEARCH', 0)))
 
 # Default wordlists
-DEFAULT_HTTP_WORDLIST = os.environ.get('DEFAULT_HTTP_WORDLIST', f'{WORDLISTS_FOLDER}/Fuzzing/fuzz-Bo0oM.txt')
-DEFAULT_DNS_WORDLIST = os.environ.get('DEFAULT_DNS_WORDLIST', f'{WORDLISTS_FOLDER}/Discovery/DNS/combined_subdomains.txt')  # noqa:E501
+DEFAULT_HTTP_WORDLIST = os.environ.get('DEFAULT_HTTP_WORDLIST', f'{WORDLISTS_FOLDER}/fuzz-Bo0oM.txt')
+DEFAULT_HTTP_WORDLIST_URL = 'https://raw.githubusercontent.com/Bo0oM/fuzz.txt/master/fuzz.txt'
+DEFAULT_DNS_WORDLIST = os.environ.get('DEFAULT_DNS_WORDLIST', f'{WORDLISTS_FOLDER}/combined_subdomains.txt')
+DEFAULT_DNS_WORDLIST_URL = 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/combined_subdomains.txt'  # noqa: E501
 
 # Constants
 OPT_NOT_SUPPORTED = -1
@@ -87,7 +87,6 @@ OPT_PIPE_INPUT = -1
 # Vocab
 ALIVE = 'alive'
 AUTO_CALIBRATION = 'auto_calibration'
-COOKIES = 'cookies'
 CONTENT_TYPE = 'content_type'
 CONTENT_LENGTH = 'content_length'
 CIDR_RANGE = 'cidr_range'
@@ -98,7 +97,6 @@ DOMAIN = 'domain'
 DEPTH = 'depth'
 EXTRA_DATA = 'extra_data'
 EMAIL = 'email'
-FAILED_HTTP_STATUS = -1
 FILTER_CODES = 'filter_codes'
 FILTER_WORDS = 'filter_words'
 FOLLOW_REDIRECT = 'follow_redirect'
@@ -106,9 +104,7 @@ FILTER_REGEX = 'filter_regex'
 FILTER_SIZE = 'filter_size'
 HEADER = 'header'
 HOST = 'host'
-INPUT = 'input'
 IP = 'ip'
-JSON = 'json'
 LINES = 'lines'
 METHOD = 'method'
 MATCH_CODES = 'match_codes'
@@ -117,13 +113,10 @@ MATCH_SIZE = 'match_size'
 MATCH_WORDS = 'match_words'
 OUTPUT_PATH = 'output_path'
 PATH = 'path'
-PAYLOAD = 'payload'
 PERCENT = 'percent'
-PROBE = 'probe'
 PORTS = 'ports'
 PORT = 'port'
 PROXY = 'proxy'
-QUIET = 'quiet'
 RATE_LIMIT = 'rate_limit'
 RETRIES = 'retries'
 TAGS = 'tags'
@@ -135,19 +128,16 @@ TYPE = 'type'
 URL = 'url'
 USER_AGENT = 'user_agent'
 USERNAME = 'username'
-SCREENSHOT_PATH = 'screenshot_path'
 STORED_RESPONSE_PATH = 'stored_response_path'
 SCRIPT = 'script'
 SERVICE_NAME = 'service_name'
 SOURCES = 'sources'
 STATE = 'state'
 STATUS_CODE = 'status_code'
-SUBDOMAIN = 'subdomain'
 TECH = 'tech'
 TITLE = 'title'
 SITE_NAME = 'site_name'
 SERVICE_NAME = 'service_name'
-VULN = 'vulnerability'
 CONFIDENCE = 'confidence'
 CVSS_SCORE = 'cvss_score'
 DESCRIPTION = 'description'
@@ -159,7 +149,88 @@ REFERENCE = 'reference'
 REFERENCES = 'references'
 SEVERITY = 'severity'
 TAGS = 'tags'
-VULN_TYPE = 'type'
 WEBSERVER = 'webserver'
 WORDLIST = 'wordlist'
 WORDS = 'words'
+
+
+# Create all folders
+for folder in [BIN_FOLDER, DATA_FOLDER, REPORTS_FOLDER, WORDLISTS_FOLDER, CVES_FOLDER, PAYLOADS_FOLDER,
+			   REVSHELLS_FOLDER, CELERY_DATA_FOLDER, CELERY_RESULTS_FOLDER]:
+	if not os.path.exists(folder):
+		console.print(f'[bold turquoise4]Creating folder {folder} ...[/] ', end='')
+		os.makedirs(folder)
+		console.print('[bold green]ok.[/]')
+
+
+# Download default wordlists
+for wordlist in ['HTTP', 'DNS']:
+	wordlist_path = globals()[f'DEFAULT_{wordlist}_WORDLIST']
+	wordlist_url = globals()[f'DEFAULT_{wordlist}_WORDLIST_URL']
+	if not os.path.exists(wordlist_path):
+		try:
+			console.print(f'[bold turquoise4]Downloading default {wordlist} wordlist {wordlist_path} ...[/] ', end='')
+			resp = requests.get(wordlist_url)
+			with open(wordlist_path, 'w') as f:
+				f.write(resp.text)
+			console.print('[bold green]ok.[/]')
+		except requests.exceptions.RequestException as e:
+			console.print(f'[bold green]failed ({type(e).__name__}).[/]')
+			pass
+
+ADDONS_ENABLED = {}
+
+# Check worker addon
+try:
+	import eventlet  # noqa: F401
+	ADDONS_ENABLED['worker'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['worker'] = False
+
+# Check google addon
+try:
+	import gspread  # noqa: F401
+	ADDONS_ENABLED['google'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['google'] = False
+
+# Check mongodb addon
+try:
+	import pymongo  # noqa: F401
+	ADDONS_ENABLED['mongodb'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['mongodb'] = False
+
+# Check redis addon
+try:
+	import redis  # noqa: F401
+	ADDONS_ENABLED['redis'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['redis'] = False
+
+# Check dev addon
+try:
+	import flake8  # noqa: F401
+	ADDONS_ENABLED['dev'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['dev'] = False
+
+# Check build addon
+try:
+	import hatch  # noqa: F401
+	ADDONS_ENABLED['build'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['build'] = False
+
+# Check trace addon
+try:
+	import memray  # noqa: F401
+	ADDONS_ENABLED['trace'] = True
+except ModuleNotFoundError:
+	ADDONS_ENABLED['trace'] = False
+
+# Check dev package
+if os.path.exists(f'{ROOT_FOLDER}/pyproject.toml'):
+	DEV_PACKAGE = True
+else:
+	DEV_PACKAGE = False

secator/exporters/csv.py

@@ -6,7 +6,6 @@ from secator.rich import console
 
 class CsvExporter(Exporter):
     def send(self):
-        title = self.report.data['info']['title']
         results = self.report.data['results']
         csv_paths = []
 
@@ -15,7 +14,7 @@ class CsvExporter(Exporter):
             if not items:
                 continue
             keys = list(items[0].keys())
-            csv_path = f'{self.report.output_folder}/{title}_{output_type}_{self.report.timestamp}.csv'
+            csv_path = f'{self.report.output_folder}/report_{output_type}.csv'
             csv_paths.append(csv_path)
             with open(csv_path, 'w', newline='') as output_file:
                 dict_writer = _csv.DictWriter(output_file, keys)

secator/exporters/gdrive.py

@@ -55,7 +55,7 @@ class GdriveExporter(Exporter):
 				k.replace('_', ' ').upper()
 				for k in list(items[0].keys())
 			]
-			csv_path = f'{self.report.output_folder}/{title}_{output_type}_{self.report.timestamp}.csv'
+			csv_path = f'{self.report.output_folder}/report_{output_type}.csv'
 			if not os.path.exists(csv_path):
 				console.print(
 					f'Unable to find CSV at {csv_path}. For Google sheets reports, please enable CSV reports as well.')

secator/exporters/json.py

@@ -5,8 +5,7 @@ from secator.serializers.dataclass import dumps_dataclass
 
 class JsonExporter(Exporter):
 	def send(self):
-		title = self.report.data['info']['title']
-		json_path = f'{self.report.output_folder}/{title}_{self.report.timestamp}.json'
+		json_path = f'{self.report.output_folder}/report.json'
 
 		# Save JSON report to file
 		with open(json_path, 'w') as f:

secator/exporters/txt.py

@@ -4,7 +4,6 @@ from secator.rich import console
 
 class TxtExporter(Exporter):
     def send(self):
-        title = self.report.data['info']['title']
         results = self.report.data['results']
         txt_paths = []
 
@@ -12,7 +11,7 @@ class TxtExporter(Exporter):
             items = [str(i) for i in items]
             if not items:
                 continue
-            txt_path = f'{self.report.output_folder}/{title}_{output_type}_{self.report.timestamp}.txt'
+            txt_path = f'{self.report.output_folder}/report_{output_type}.txt'
             with open(txt_path, 'w') as f:
                 f.write('\n'.join(items))
             txt_paths.append(txt_path)

secator/hooks/mongodb.py

@@ -1,40 +1,43 @@
-from bson.objectid import ObjectId
-import os
 import logging
+import os
 import time
 
+import pymongo
+from bson.objectid import ObjectId
 from celery import shared_task
 
 from secator.definitions import DEFAULT_PROGRESS_UPDATE_FREQUENCY
-from secator.runners import Task, Workflow, Scan
 from secator.output_types import OUTPUT_TYPES
-from secator.utils import debug
+from secator.runners import Scan, Task, Workflow
+from secator.utils import debug, escape_mongodb_url
 
-import pymongo
 # import gevent.monkey
 # gevent.monkey.patch_all()
 
 MONGODB_URL = os.environ.get('MONGODB_URL', 'mongodb://localhost')
 MONGODB_UPDATE_FREQUENCY = int(os.environ.get('MONGODB_UPDATE_FREQUENCY', DEFAULT_PROGRESS_UPDATE_FREQUENCY))
 MAX_POOL_SIZE = 100
-client = pymongo.MongoClient(MONGODB_URL, maxPoolSize=MAX_POOL_SIZE)
 
 logger = logging.getLogger(__name__)
 
+client = pymongo.MongoClient(escape_mongodb_url(MONGODB_URL), maxPoolSize=MAX_POOL_SIZE)
+
 
 def update_runner(self):
 	db = client.main
 	type = self.config.type
 	collection = f'{type}s'
 	update = self.toDict()
+	debug_obj = {'type': 'runner', 'name': self.name, 'status': self.status}
 	chunk = update.get('chunk')
 	_id = self.context.get(f'{type}_chunk_id') if chunk else self.context.get(f'{type}_id')
+	debug('update', sub='hooks.mongodb', id=_id, obj=update, obj_after=True, level=4)
 	start_time = time.time()
 	if _id:
 		delta = start_time - self.last_updated if self.last_updated else MONGODB_UPDATE_FREQUENCY
 		if self.last_updated and delta < MONGODB_UPDATE_FREQUENCY and self.status == 'RUNNING':
 			debug(f'skipped ({delta:>.2f}s < {MONGODB_UPDATE_FREQUENCY}s)',
-		 		  sub='hooks.mongodb', id=_id, obj={self.name: self.status}, obj_after=False, level=3)
+				  sub='hooks.mongodb', id=_id, obj=debug_obj, obj_after=False, level=3)
 			return
 		db = client.main
 		start_time = time.time()
@@ -42,8 +45,7 @@ def update_runner(self):
 		end_time = time.time()
 		elapsed = end_time - start_time
 		debug(
-			f'[dim gold4]updated in {elapsed:.4f}s[/]',
-			sub='hooks.mongodb', id=_id, obj={self.name: self.status}, obj_after=False, level=2)
+			f'[dim gold4]updated in {elapsed:.4f}s[/]', sub='hooks.mongodb', id=_id, obj=debug_obj, obj_after=False, level=2)
 		self.last_updated = start_time
 	else:  # sync update and save result to runner object
 		runner = db[collection].insert_one(update)
@@ -54,9 +56,7 @@ def update_runner(self):
 			self.context[f'{type}_id'] = _id
 		end_time = time.time()
 		elapsed = end_time - start_time
-		debug(
-			f'created in {elapsed:.4f}s',
-			sub='hooks.mongodb', id=_id, obj={self.name: self.status}, obj_after=False, level=2)
+		debug(f'created in {elapsed:.4f}s', sub='hooks.mongodb', id=_id, obj=debug_obj, obj_after=False, level=2)
 
 
 def update_finding(self, item):