secator 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

secator/config.py

@@ -20,6 +20,7 @@ StrExpandHome = Annotated[str, AfterValidator(lambda v: v.replace('~', str(Path.
 ROOT_FOLDER = Path(__file__).parent.parent
 LIB_FOLDER = ROOT_FOLDER / 'secator'
 CONFIGS_FOLDER = LIB_FOLDER / 'configs'
+DATA_FOLDER = os.environ.get('SECATOR_DIRS_DATA') or str(Path.home() / '.secator')
 
 
 class StrictModel(BaseModel, extra='forbid'):
@@ -28,12 +29,14 @@ class StrictModel(BaseModel, extra='forbid'):
 
 class Directories(StrictModel):
 	bin: Directory = Path.home() / '.local' / 'bin'
-	data: Directory = Path.home() / '.secator'
+	share: Directory = Path.home() / '.local' / 'share'
+	data: Directory = Path(DATA_FOLDER)
 	templates: Directory = ''
 	reports: Directory = ''
 	wordlists: Directory = ''
 	cves: Directory = ''
 	payloads: Directory = ''
+	performance: Directory = ''
 	revshells: Directory = ''
 	celery: Directory = ''
 	celery_data: Directory = ''
@@ -42,7 +45,7 @@ class Directories(StrictModel):
 	@model_validator(mode='after')
 	def set_default_folders(self) -> Self:
 		"""Set folders to be relative to the data folders if they are unspecified in config."""
-		for folder in ['templates', 'reports', 'wordlists', 'cves', 'payloads', 'revshells', 'celery', 'celery_data', 'celery_results']:  # noqa: E501
+		for folder in ['templates', 'reports', 'wordlists', 'cves', 'payloads', 'performance', 'revshells', 'celery', 'celery_data', 'celery_results']:  # noqa: E501
 			rel_target = '/'.join(folder.split('_'))
 			val = getattr(self, folder) or self.data / rel_target
 			setattr(self, folder, val)
@@ -61,40 +64,52 @@ class Celery(StrictModel):
 	broker_visibility_timeout: int = 3600
 	override_default_logging: bool = True
 	result_backend: StrExpandHome = ''
+	result_expires: int = 86400  # 1 day
 
 
 class Cli(StrictModel):
-	github_token: str = ''
+	github_token: str = os.environ.get('GITHUB_TOKEN', '')
 	record: bool = False
 	stdin_timeout: int = 1000
 
 
 class Runners(StrictModel):
-	input_chunk_size: int = 1000
-	progress_update_frequency: int = 60
+	input_chunk_size: int = 100
+	progress_update_frequency: int = 20
+	stat_update_frequency: int = 20
+	backend_update_frequency: int = 5
+	poll_frequency: int = 5
 	skip_cve_search: bool = False
-	skip_cve_low_confidence: bool = True
+	skip_exploit_search: bool = False
+	skip_cve_low_confidence: bool = False
 	remove_duplicates: bool = False
 
 
+class Security(StrictModel):
+	allow_local_file_access: bool = True
+	auto_install_commands: bool = True
+	force_source_install: bool = False
+
+
 class HTTP(StrictModel):
 	socks5_proxy: str = 'socks5://127.0.0.1:9050'
 	http_proxy: str = 'https://127.0.0.1:9080'
 	store_responses: bool = False
+	response_max_size_bytes: int = 100000  # 100MB
 	proxychains_command: str = 'proxychains'
 	freeproxy_timeout: int = 1
 
 
 class Tasks(StrictModel):
-	exporters: List[str] = ['json', 'csv']
+	exporters: List[str] = ['json', 'csv', 'txt']
 
 
 class Workflows(StrictModel):
-	exporters: List[str] = ['json', 'csv']
+	exporters: List[str] = ['json', 'csv', 'txt']
 
 
 class Scans(StrictModel):
-	exporters: List[str] = ['json', 'csv']
+	exporters: List[str] = ['json', 'csv', 'txt']
 
 
 class Payloads(StrictModel):
@@ -109,17 +124,24 @@ class Wordlists(StrictModel):
 	defaults: Dict[str, str] = {'http': 'bo0m_fuzz', 'dns': 'combined_subdomains'}
 	templates: Dict[str, str] = {
 		'bo0m_fuzz': 'https://raw.githubusercontent.com/Bo0oM/fuzz.txt/master/fuzz.txt',
-		'combined_subdomains': 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/combined_subdomains.txt'  # noqa: E501
+		'combined_subdomains': 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/combined_subdomains.txt',  # noqa: E501
+		'directory_list_small': 'https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/Web-Content/directory-list-2.3-small.txt',  # noqa: E501
 	}
 	lists: Dict[str, List[str]] = {}
 
 
-class GoogleAddon(StrictModel):
+class GoogleDriveAddon(StrictModel):
 	enabled: bool = False
 	drive_parent_folder_id: str = ''
 	credentials_path: str = ''
 
 
+class GoogleCloudStorageAddon(StrictModel):
+	enabled: bool = False
+	bucket_name: str = ''
+	credentials_path: str = ''
+
+
 class WorkerAddon(StrictModel):
 	enabled: bool = False
 
@@ -128,10 +150,13 @@ class MongodbAddon(StrictModel):
 	enabled: bool = False
 	url: str = 'mongodb://localhost'
 	update_frequency: int = 60
+	max_pool_size: int = 10
+	server_selection_timeout_ms: int = 5000
 
 
 class Addons(StrictModel):
-	google: GoogleAddon = GoogleAddon()
+	gdrive: GoogleDriveAddon = GoogleDriveAddon()
+	gcs: GoogleCloudStorageAddon = GoogleCloudStorageAddon()
 	worker: WorkerAddon = WorkerAddon()
 	mongodb: MongodbAddon = MongodbAddon()
 
@@ -149,6 +174,7 @@ class SecatorConfig(StrictModel):
 	payloads: Payloads = Payloads()
 	wordlists: Wordlists = Wordlists()
 	addons: Addons = Addons()
+	security: Security = Security()
 	offline_mode: bool = False
 
 
@@ -161,7 +187,7 @@ class Config(DotMap):
 	>>> config = Config.parse(path='/path/to/config.yml')  # get custom config (from YAML file).
 	>>> config.print() 									   # print config without defaults.
 	>>> config.print(partial=False)  					   # print full config.
-	>>> config.set('addons.google.enabled', False)         # set value in config.
+	>>> config.set('addons.gdrive.enabled', False)         # set value in config.
 	>>> config.save()									   # save config back to disk.
 	"""
 
@@ -479,56 +505,81 @@ def download_files(data: dict, target_folder: Path, offline_mode: bool, type: st
 		offline_mode (bool): Offline mode.
 	"""
 	for name, url_or_path in data.items():
-		if url_or_path.startswith('git+'):
-			# Clone Git repository
-			git_url = url_or_path[4:]  # remove 'git+' prefix
-			repo_name = git_url.split('/')[-1]
-			if repo_name.endswith('.git'):
-				repo_name = repo_name[:-4]
-			target_path = target_folder / repo_name
-			if not target_path.exists():
-				console.print(f'[bold turquoise4]Cloning git {type} [bold magenta]{repo_name}[/] ...[/] ', end='')
+		target_path = download_file(url_or_path, target_folder, offline_mode, type, name=name)
+		if target_path:
+			data[name] = target_path
+
+
+def download_file(url_or_path, target_folder: Path, offline_mode: bool, type: str, name: str = None):
+	"""Download remote file to target folder, clone git repos, or symlink local files.
+
+	Args:
+		data (dict): Dict of name to url or local path prefixed with 'git+' for Git repos.
+		target_folder (Path): Target folder for storing files or repos.
+		offline_mode (bool): Offline mode.
+		type (str): Type of files to handle.
+		name (str, Optional): Name of object.
+
+	Returns:
+		path (Path): Path to downloaded file / folder.
+	"""
+	if url_or_path.startswith('git+'):
+		# Clone Git repository
+		git_url = url_or_path[4:]  # remove 'git+' prefix
+		repo_name = git_url.split('/')[-1]
+		if repo_name.endswith('.git'):
+			repo_name = repo_name[:-4]
+		target_path = target_folder / repo_name
+		if not target_path.exists():
+			console.print(f'[bold turquoise4]Cloning git {type} [bold magenta]{repo_name}[/] ...[/] ', end='')
+			if offline_mode:
+				console.print('[bold orange1]skipped [dim][offline[/].[/]')
+				return
+			try:
+				call(['git', 'clone', git_url, str(target_path)], stderr=DEVNULL, stdout=DEVNULL)
+				console.print('[bold green]ok.[/]')
+			except Exception as e:
+				console.print(f'[bold red]failed ({str(e)}).[/]')
+		return target_path.resolve()
+	elif Path(url_or_path).exists():
+		# Create a symbolic link for a local file
+		local_path = Path(url_or_path)
+		target_path = target_folder / local_path.name
+		if not name:
+			name = url_or_path.split('/')[-1]
+		if not CONFIG.security.allow_local_file_access:
+			console.print(f'[bold red]Cannot reference local file {url_or_path}(disabled for security reasons)[/]')
+			return
+		if not target_path.exists():
+			console.print(f'[bold turquoise4]Symlinking {type} [bold magenta]{name}[/] ...[/] ', end='')
+			try:
+				target_path.symlink_to(local_path)
+				console.print('[bold green]ok.[/]')
+			except Exception as e:
+				console.print(f'[bold red]failed ({str(e)}).[/]')
+		return target_path.resolve()
+	else:
+		# Download file from URL
+		ext = url_or_path.split('.')[-1]
+		if not name:
+			name = url_or_path.split('/')[-1]
+		filename = f'{name}.{ext}' if not name.endswith(ext) else name
+		target_path = target_folder / filename
+		if not target_path.exists():
+			try:
+				console.print(f'[bold turquoise4]Downloading {type} [bold magenta]{filename}[/] ...[/] ', end='')
 				if offline_mode:
-					console.print('[bold orange1]skipped [dim][offline[/].[/]')
-					continue
-				try:
-					call(['git', 'clone', git_url, str(target_path)], stderr=DEVNULL, stdout=DEVNULL)
-					console.print('[bold green]ok.[/]')
-				except Exception as e:
-					console.print(f'[bold red]failed ({str(e)}).[/]')
-			data[name] = target_path.resolve()
-		elif Path(url_or_path).exists():
-			# Create a symbolic link for a local file
-			local_path = Path(url_or_path)
-			target_path = target_folder / local_path.name
-			if not target_path.exists():
-				console.print(f'[bold turquoise4]Symlinking {type} [bold magenta]{name}[/] ...[/] ', end='')
-				try:
-					target_path.symlink_to(local_path)
-					console.print('[bold green]ok.[/]')
-				except Exception as e:
-					console.print(f'[bold red]failed ({str(e)}).[/]')
-			data[name] = target_path.resolve()
-		else:
-			# Download file from URL
-			ext = url_or_path.split('.')[-1]
-			filename = f'{name}.{ext}'
-			target_path = target_folder / filename
-			if not target_path.exists():
-				try:
-					console.print(f'[bold turquoise4]Downloading {type} [bold magenta]{filename}[/] ...[/] ', end='')
-					if offline_mode:
-						console.print('[bold orange1]skipped [dim](offline)[/].[/]')
-						continue
-					resp = requests.get(url_or_path, timeout=3)
-					resp.raise_for_status()
-					with open(target_path, 'wb') as f:
-						f.write(resp.content)
-					console.print('[bold green]ok.[/]')
-				except requests.RequestException as e:
-					console.print(f'[bold red]failed ({str(e)}).[/]')
-					continue
-			data[name] = target_path.resolve()
+					console.print('[bold orange1]skipped [dim](offline)[/].[/]')
+					return
+				resp = requests.get(url_or_path, timeout=3)
+				resp.raise_for_status()
+				with open(target_path, 'wb') as f:
+					f.write(resp.content)
+				console.print('[bold green]ok.[/]')
+			except requests.RequestException as e:
+				console.print(f'[bold red]failed ({str(e)}).[/]')
+				return
+		return target_path.resolve()
 
 
 # Load default_config
@@ -560,13 +611,8 @@ for name, dir in CONFIG.dirs.items():
 		dir.mkdir(parents=False)
 		console.print('[bold green]ok.[/]')
 
-# Download wordlists and set defaults
+# Download wordlists and payloads
 download_files(CONFIG.wordlists.templates, CONFIG.dirs.wordlists, CONFIG.offline_mode, 'wordlist')
-for category, name in CONFIG.wordlists.defaults.items():
-	if name in CONFIG.wordlists.templates.keys():
-		CONFIG.wordlists.defaults[category] = str(CONFIG.wordlists.templates[name])
-
-# Download payloads
 download_files(CONFIG.payloads.templates, CONFIG.dirs.payloads, CONFIG.offline_mode, 'payload')
 
 # Print config

secator/configs/workflows/host_recon.yaml

@@ -11,6 +11,8 @@ tasks:
     description: Find open ports
   nmap:
     description: Search for vulnerabilities on open ports
+    skip_host_discovery: True
+    version_detection: True
     targets_: port.host
     ports_: port.port
   httpx:
@@ -18,7 +20,7 @@ tasks:
     targets_:
       - type: port
         field: '{host}:{port}'
-        condition: item._source == 'nmap'
+        condition: item._source.startswith('nmap')
   _group:
     nuclei/network:
       description: Scan network and SSL vulnerabilities
@@ -32,10 +34,10 @@ tasks:
           condition: item.status_code != 0
 results:
   - type: port
-    condition: item._source == 'nmap'
+    condition: item._source.startswith('nmap')
 
   - type: vulnerability
   #   condition: item.confidence == 'high'
 
   - type: url
-    condition: item.status_code != 0
+    condition: item.status_code != 0

secator/configs/workflows/port_scan.yaml

@@ -5,11 +5,15 @@ description: Port scan
 tags: [recon, network, http, vuln]
 input_types:
   - host
+  - cidr_range
 tasks:
   naabu:
     description: Find open ports
+    ports: "-"  # scan all ports
   nmap:
     description: Search for vulnerabilities on open ports
+    skip_host_discovery: True
+    version_detection: True
     targets_: port.host
     ports_: port.port
   _group:
@@ -18,17 +22,17 @@ tasks:
       targets_:
       - type: port
         field: '{host}~{service_name}'
-        condition: item._source == 'nmap' and len(item.service_name.split('/')) > 1
+        condition: item._source.startswith('nmap') and len(item.service_name.split('/')) > 1
     httpx:
       description: Probe HTTP services on open ports
       targets_:
         - type: port
           field: '{host}:{port}'
-          condition: item._source == 'nmap'
+          condition: item._source.startswith('nmap')
 results:
   - type: port
 
   - type: url
     condition: item.status_code != 0
 
-  - type: vulnerability
+  - type: vulnerability

secator/configs/workflows/subdomain_recon.yaml

@@ -13,12 +13,12 @@ tasks:
   #   input: vhost
   #   domain_:
   #     - target.name
-  #   wordlist: /usr/share/seclists/Discovery/DNS/combined_subdomains.txt
+  #   wordlist: combined_subdomains
   # gobuster:
   #   input: dns
   #   domain_:
   #     - target.name
-  #   wordlist: /usr/share/seclists/Discovery/DNS/combined_subdomains.txt
+  #   wordlist: combined_subdomains
   _group:
     nuclei:
       description: Check for subdomain takeovers

secator/configs/workflows/url_bypass.yaml

@@ -0,0 +1,10 @@
+type: workflow
+name: url_bypass
+alias: urlbypass
+description: Try bypass techniques for 4xx URLs
+tags: [http, crawl]
+input_types:
+  - url
+tasks:
+  bup:
+    description: Bypass 4xx

secator/configs/workflows/url_dirsearch.yaml

@@ -8,7 +8,7 @@ input_types:
 tasks:
   ffuf:
     description: Search for HTTP directories
-    wordlist: /usr/share/seclists/Discovery/Web-Content/directory-list-2.3-small.txt
+    wordlist: directory_list_small
     targets_:
       - type: target
         field: '{name}/FUZZ'

secator/configs/workflows/url_vuln.yaml

@@ -34,7 +34,7 @@ tasks:
     targets_:
       - type: tag
         field: match
-        condition: item._source == "gf"
+        condition: item._source.startswith("gf")
 
   # TODO: Add support for SQLMap
   # sqlmap: