secator 0.22.0__py3-none-any.whl

secator/decorators.py

@@ -0,0 +1,7 @@
+
+
+def task():
+	def decorator(cls):
+		cls.__task__ = True
+		return cls
+	return decorator

secator/definitions.py

@@ -0,0 +1,168 @@
+#!/usr/bin/python
+
+import os
+
+from importlib.metadata import version
+
+from secator.config import CONFIG, ROOT_FOLDER
+
+
+# Globals
+VERSION = version('secator')
+ASCII = rf"""
+			 __            
+   ________  _________ _/ /_____  _____
+  / ___/ _ \/ ___/ __ `/ __/ __ \/ ___/
+ (__  /  __/ /__/ /_/ / /_/ /_/ / /    
+/____/\___/\___/\__,_/\__/\____/_/     v{VERSION}
+
+			freelabz.com
+"""  # noqa: W605,W291
+
+# Debug
+DEBUG = CONFIG.debug.split(',')
+FORCE_TTY = CONFIG.runners.force_tty
+
+# Constants
+OPT_NOT_SUPPORTED = -1
+OPT_PIPE_INPUT = -2
+OPT_SPACE_SEPARATED = -3
+STATE_COLORS = {
+	'PENDING': 'dim yellow3',
+	'RUNNING': 'bold yellow3',
+	'SUCCESS': 'bold green',
+	'FAILURE': 'bold red',
+	'REVOKED': 'bold magenta'
+}
+
+# Vocab
+ALIVE = 'alive'
+AUTO_CALIBRATION = 'auto_calibration'
+CONTENT_TYPE = 'content_type'
+CONTENT_LENGTH = 'content_length'
+CERTIFICATE_STATUS_UNKNOWN = 'Unknown'
+CERTIFICATE_STATUS_REVOKED = 'Revoked'
+CERTIFICATE_STATUS_TRUSTED = 'Trusted'
+CIDR_RANGE = 'cidr_range'
+CONFIDENCE = 'confidence'
+CPES = 'cpes'
+CVES = 'cves'
+CVSS_SCORE = 'cvss_score'
+DATA = 'data'
+DELAY = 'delay'
+DESCRIPTION = 'description'
+DOCKER_IMAGE = 'docker_image'
+DOMAIN = 'domain'
+DEPTH = 'depth'
+EXTRA_DATA = 'extra_data'
+EMAIL = 'email'
+FILENAME = 'filename'
+FILTER_CODES = 'filter_codes'
+FILTER_WORDS = 'filter_words'
+FOLLOW_REDIRECT = 'follow_redirect'
+FILTER_REGEX = 'filter_regex'
+FILTER_SIZE = 'filter_size'
+GCS_URL = 'gcs_url'
+GIT_REPOSITORY = 'git_repository'
+HEADER = 'header'
+HOST = 'host'
+HOST_PORT = 'host:port'
+IBAN = 'iban'
+ID = 'id'
+IP = 'ip'
+PROTOCOL = 'protocol'
+LINES = 'lines'
+METHOD = 'method'
+MAC_ADDRESS = 'mac'
+MATCHED_AT = 'matched_at'
+MATCH_CODES = 'match_codes'
+MATCH_REGEX = 'match_regex'
+MATCH_SIZE = 'match_size'
+MATCH_WORDS = 'match_words'
+NAME = 'name'
+ORG_NAME = 'org_name'
+OUTPUT_PATH = 'output_path'
+PATH = 'path'
+PERCENT = 'percent'
+PORTS = 'ports'
+PORT = 'port'
+PROVIDER = 'provider'
+PROXY = 'proxy'
+RATE_LIMIT = 'rate_limit'
+REFERENCE = 'reference'
+REFERENCES = 'references'
+RETRIES = 'retries'
+SCRIPT = 'script'
+SERVICE_NAME = 'service_name'
+SEVERITY = 'severity'
+SITE_NAME = 'site_name'
+SLUG = 'slug'
+SOURCES = 'sources'
+STORED_RESPONSE_PATH = 'stored_response_path'
+STATE = 'state'
+STATUS_CODE = 'status_code'
+STRING = 'str'
+TAGS = 'tags'
+TECH = 'tech'
+TECHNOLOGY = 'technology'
+THREADS = 'threads'
+TIME = 'time'
+TIMEOUT = 'timeout'
+TITLE = 'title'
+TOP_PORTS = 'top_ports'
+TYPE = 'type'
+URL = 'url'
+USER_AGENT = 'user_agent'
+USERNAME = 'username'
+UUID = 'uuid'
+WEBSERVER = 'webserver'
+WORDLIST = 'wordlist'
+WORDS = 'words'
+
+# Allowed input types
+INPUT_TYPES = [
+	URL,
+	IP,
+	CIDR_RANGE,
+	HOST,
+	MAC_ADDRESS,
+	EMAIL,
+	IBAN,
+	UUID,
+	PATH,
+	SLUG,
+	STRING,
+]
+
+
+def is_importable(module_to_import):
+	import importlib
+	try:
+		importlib.import_module(module_to_import)
+		return True
+	except ModuleNotFoundError:
+		return False
+	except Exception as e:
+		print(f'Failed trying to import {module_to_import}: {str(e)}')
+		return False
+
+
+ADDONS_ENABLED = {}
+
+for addon, module in [
+	('worker', 'eventlet'),
+	('gdrive', 'gspread'),
+	('gcs', 'google.cloud.storage'),
+	('mongodb', 'pymongo'),
+	('redis', 'redis'),
+	('dev', 'flake8'),
+	('trace', 'memray'),
+	('build', 'hatch')
+]:
+	ADDONS_ENABLED[addon] = is_importable(module)
+
+# Check dev package
+if os.path.exists(f'{ROOT_FOLDER}/pyproject.toml'):
+	DEV_PACKAGE = True
+else:
+	DEV_PACKAGE = False

secator/exporters/__init__.py

@@ -0,0 +1,14 @@
+__all__ = [
+	'ConsoleExporter',
+	'CsvExporter',
+	'GdriveExporter',
+	'JsonExporter',
+	'TableExporter',
+	'TxtExporter'
+]
+from secator.exporters.console import ConsoleExporter
+from secator.exporters.csv import CsvExporter
+from secator.exporters.gdrive import GdriveExporter
+from secator.exporters.json import JsonExporter
+from secator.exporters.table import TableExporter
+from secator.exporters.txt import TxtExporter

secator/exporters/_base.py

@@ -0,0 +1,3 @@
+class Exporter:
+	def __init__(self, report):
+		self.report = report

secator/exporters/console.py

@@ -0,0 +1,10 @@
+from secator.exporters._base import Exporter
+from secator.rich import console_stdout
+
+
+class ConsoleExporter(Exporter):
+	def send(self):
+		results = self.report.data['results']
+		for items in results.values():
+			for item in items:
+				console_stdout.print(item)

secator/exporters/csv.py

@@ -0,0 +1,37 @@
+import csv as _csv
+
+from dataclasses import fields
+
+from secator.exporters._base import Exporter
+from secator.rich import console
+from secator.output_types import FINDING_TYPES
+from secator.output_types import Info
+
+
+class CsvExporter(Exporter):
+	def send(self):
+		results = self.report.data['results']
+		if not results:
+			return
+		csv_paths = []
+
+		for output_type, items in results.items():
+			output_cls = [o for o in FINDING_TYPES if o._type == output_type][0]
+			keys = [o.name for o in fields(output_cls)]
+			items = [i.toDict() for i in items]
+			if not items:
+				continue
+			csv_path = f'{self.report.output_folder}/report_{output_type}.csv'
+			csv_paths.append(csv_path)
+			with open(csv_path, 'w', newline='') as output_file:
+				dict_writer = _csv.DictWriter(output_file, keys)
+				dict_writer.writeheader()
+				dict_writer.writerows(items)
+
+		if len(csv_paths) == 1:
+			csv_paths_str = csv_paths[0]
+		else:
+			csv_paths_str = '\n   • ' + '\n   • '.join(csv_paths)
+
+		info = Info(message=f'Saved CSV reports to {csv_paths_str}')
+		console.print(info)

secator/exporters/gdrive.py

@@ -0,0 +1,123 @@
+import os
+import csv
+import yaml
+
+from secator.config import CONFIG
+from secator.exporters._base import Exporter
+from secator.output_types import Info, Error
+from secator.rich import console
+from secator.utils import pluralize
+
+
+class GdriveExporter(Exporter):
+	def send(self):
+		import gspread
+		ws = self.report.workspace_name
+		info = self.report.data['info']
+		title = self.report.data['info']['title']
+		sheet_title = f'{self.report.data["info"]["title"]}_{self.report.timestamp}'
+		results = self.report.data['results']
+		if not CONFIG.addons.gdrive.credentials_path:
+			error = Error('Missing CONFIG.addons.gdrive.credentials_path to save to Google Sheets')
+			console.print(error)
+			return
+		if not CONFIG.addons.gdrive.drive_parent_folder_id:
+			error = Error('Missing CONFIG.addons.gdrive.drive_parent_folder_id to save to Google Sheets.')
+			console.print(error)
+			return
+		client = gspread.service_account(CONFIG.addons.gdrive.credentials_path)
+
+		# Create workspace folder if it doesn't exist
+		folder_id = self.get_folder_by_name(ws, parent_id=CONFIG.addons.gdrive.drive_parent_folder_id)
+		if ws and not folder_id:
+			folder_id = self.create_folder(
+				folder_name=ws,
+				parent_id=CONFIG.addons.gdrive.drive_parent_folder_id)
+
+		# Create worksheet
+		sheet = client.create(title, folder_id=folder_id)
+
+		# Add options worksheet for input data
+		info = self.report.data['info']
+		info['targets'] = '\n'.join(info['targets'])
+		info['opts'] = yaml.dump(info['opts'])
+		keys = [k.replace('_', ' ').upper() for k in list(info.keys())]
+		ws = sheet.add_worksheet('OPTIONS', rows=2, cols=len(keys))
+		sheet.values_update(
+			ws.title,
+			params={'valueInputOption': 'USER_ENTERED'},
+			body={'values': [keys, list(info.values())]}
+		)
+
+		# Add one worksheet per output type
+		for output_type, items in results.items():
+			items = [i.toDict() for i in items]
+			if not items:
+				continue
+			keys = [
+				k.replace('_', ' ').upper()
+				for k in list(items[0].keys())
+			]
+			csv_path = f'{self.report.output_folder}/report_{output_type}.csv'
+			if not os.path.exists(csv_path):
+				error = Error(
+					f'Unable to find CSV at {csv_path}. For Google sheets reports, please enable CSV reports as well.')
+				console.print(error)
+				return
+			sheet_title = pluralize(output_type).upper()
+			ws = sheet.add_worksheet(sheet_title, rows=len(items), cols=len(keys))
+			with open(csv_path, 'r') as f:
+				data = csv.reader(f)
+				data = list(data)
+				data[0] = [
+					k.replace('_', ' ').upper()
+					for k in data[0]
+				]
+				sheet.values_update(
+					ws.title,
+					params={'valueInputOption': 'USER_ENTERED'},
+					body={'values': data}
+				)
+
+		# Delete 'default' worksheet
+		ws = sheet.get_worksheet(0)
+		sheet.del_worksheet(ws)
+
+		info = Info(message=f'Saved Google Sheets reports to [u magenta]{sheet.url}')
+		console.print(info)
+
+	def create_folder(self, folder_name, parent_id=None):
+		from googleapiclient.discovery import build
+		from google.oauth2 import service_account
+		creds = service_account.Credentials.from_service_account_file(CONFIG.addons.gdrive.credentials_path)
+		service = build('drive', 'v3', credentials=creds)
+		body = {
+			'name': folder_name,
+			'mimeType': "application/vnd.google-apps.folder"
+		}
+		if parent_id:
+			body['parents'] = [parent_id]
+		folder = service.files().create(body=body, fields='id').execute()
+		return folder['id']
+
+	def list_folders(self, parent_id):
+		from googleapiclient.discovery import build
+		from google.oauth2 import service_account
+		creds = service_account.Credentials.from_service_account_file(CONFIG.addons.gdrive.credentials_path)
+		service = build('drive', 'v3', credentials=creds)
+		driveid = service.files().get(fileId='root').execute()['id']
+		response = service.files().list(
+			q=f"'{parent_id}' in parents and mimeType='application/vnd.google-apps.folder'",
+			driveId=driveid,
+			corpora='drive',
+			includeItemsFromAllDrives=True,
+			supportsAllDrives=True
+		).execute()
+		return response
+
+	def get_folder_by_name(self, name, parent_id=None):
+		response = self.list_folders(parent_id=parent_id)
+		existing = [i for i in response['files'] if i['name'] == name]
+		if existing:
+			return existing[0]['id']
+		return None

secator/exporters/json.py

@@ -0,0 +1,16 @@
+from secator.exporters._base import Exporter
+from secator.output_types import Info
+from secator.rich import console
+from secator.serializers.dataclass import dumps_dataclass
+
+
+class JsonExporter(Exporter):
+	def send(self):
+		json_path = f'{self.report.output_folder}/report.json'
+
+		# Save JSON report to file
+		with open(json_path, 'w') as f:
+			f.write(dumps_dataclass(self.report.data, indent=2))
+
+		info = Info(f'Saved JSON report to {json_path}')
+		console.print(info)

secator/exporters/table.py

@@ -0,0 +1,36 @@
+from rich.markdown import Markdown
+
+from secator.exporters._base import Exporter
+from secator.output_types import OutputType
+from secator.rich import build_table, console
+from secator.utils import pluralize
+
+
+class TableExporter(Exporter):
+	def send(self):
+		results = self.report.data['results']
+		if not results:
+			return
+		title = self.report.title
+		_print = console.print
+		_print()
+		if title:
+			title = ' '.join(title.capitalize().split('_')) + ' results'
+			h1 = Markdown(f'# {title}')
+			_print(h1, style='bold magenta', width=50)
+			_print()
+		for output_type, items in results.items():
+			if output_type == 'progress':
+				continue
+			if items:
+				is_output_type = isinstance(items[0], OutputType)
+				output_fields = items[0]._table_fields if is_output_type else None
+				sort_by = items[0]._sort_by if is_output_type else []
+				_table = build_table(
+					items,
+					output_fields=output_fields,
+					sort_by=sort_by)
+				title = pluralize(items[0]._type).upper() if is_output_type else 'Results'
+				_print(f':wrench: {title}', style='bold gold3', justify='left')
+				_print(_table)
+				_print()

secator/exporters/txt.py

@@ -0,0 +1,28 @@
+from secator.exporters._base import Exporter
+from secator.output_types import Info
+from secator.rich import console
+
+
+class TxtExporter(Exporter):
+	def send(self):
+		results = self.report.data['results']
+		if not results:
+			return
+		txt_paths = []
+
+		for output_type, items in results.items():
+			items = list(set(str(i) for i in items))
+			if not items:
+				continue
+			txt_path = f'{self.report.output_folder}/report_{output_type}.txt'
+			with open(txt_path, 'w') as f:
+				f.write('\n'.join(items))
+			txt_paths.append(txt_path)
+
+		if len(txt_paths) == 1:
+			txt_paths_str = txt_paths[0]
+		else:
+			txt_paths_str = '\n   • ' + '\n   • '.join(txt_paths)
+
+		info = Info(f'Saved TXT reports to {txt_paths_str}')
+		console.print(info)

secator/hooks/gcs.py

@@ -0,0 +1,80 @@
+import warnings
+
+from pathlib import Path
+from time import time
+
+from google.cloud import storage
+
+from secator.config import CONFIG
+from secator.runners import Task
+from secator.thread import Thread
+from secator.utils import debug
+
+warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")
+
+GCS_BUCKET_NAME = CONFIG.addons.gcs.bucket_name
+ITEMS_TO_SEND = {
+	'url': ['screenshot_path', 'stored_response_path']
+}
+
+_gcs_client = None
+
+
+def get_gcs_client():
+	"""Get or create GCS client"""
+	global _gcs_client
+	if _gcs_client is None:
+		_gcs_client = storage.Client()
+	return _gcs_client
+
+
+def process_item(self, item):
+	if item._type not in ITEMS_TO_SEND.keys():
+		return item
+	if not GCS_BUCKET_NAME:
+		debug('skipped since addons.gcs.bucket_name is empty.', sub='hooks.gcs')
+		return item
+	to_send = ITEMS_TO_SEND[item._type]
+	for k, v in item.toDict().items():
+		if k in to_send and v:
+			path = Path(v)
+			if not path.exists():
+				continue
+			ext = path.suffix
+			blob_name = f'{item._uuid}_{k}{ext}'
+			t = Thread(target=upload_blob, args=(GCS_BUCKET_NAME, v, blob_name))
+			t.start()
+			self.threads.append(t)
+			setattr(item, k, f'gs://{GCS_BUCKET_NAME}/{blob_name}')
+	return item
+
+
+def upload_blob(bucket_name, source_file_name, destination_blob_name):
+	"""Uploads a file to the bucket."""
+	start_time = time()
+	storage_client = get_gcs_client()
+	bucket = storage_client.bucket(bucket_name)
+	blob = bucket.blob(destination_blob_name)
+	with open(source_file_name, 'rb') as f:
+		f.seek(0)
+		blob.upload_from_file(f)
+	end_time = time()
+	elapsed = end_time - start_time
+	debug(f'in {elapsed:.4f}s', obj={'blob': 'UPLOADED', 'blob_name': destination_blob_name, 'bucket': bucket_name}, obj_after=False, sub='hooks.gcs', verbose=True)  # noqa: E501
+
+
+def download_blob(bucket_name, source_blob_name, destination_file_name):
+	"""Downloads a file from the bucket."""
+	start_time = time()
+	storage_client = get_gcs_client()
+	bucket = storage_client.bucket(bucket_name)
+	blob = bucket.blob(source_blob_name)
+	blob.download_to_filename(destination_file_name)
+	end_time = time()
+	elapsed = end_time - start_time
+	debug(f'in {elapsed:.4f}s', obj={'blob': 'DOWNLOADED', 'blob_name': source_blob_name, 'bucket': bucket_name}, obj_after=False, sub='hooks.gcs', verbose=True)  # noqa: E501
+
+
+HOOKS = {
+	Task: {'on_item': [process_item]}
+}