secator 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

secator/output_types/ip.py

@@ -1,16 +1,23 @@
 import time
 from dataclasses import dataclass, field
+from enum import Enum
 
 from secator.definitions import ALIVE, IP
 from secator.output_types import OutputType
 from secator.utils import rich_to_ansi
 
 
+class IpProtocol(str, Enum):
+	IPv6 = 'IPv6'
+	IPv4 = 'IPv4'
+
+
 @dataclass
 class Ip(OutputType):
 	ip: str
 	host: str = ''
 	alive: bool = False
+	protocol: str = field(default=IpProtocol.IPv4)
 	_source: str = field(default='', repr=True)
 	_type: str = field(default='ip', repr=True)
 	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
@@ -29,5 +36,5 @@ class Ip(OutputType):
 	def __repr__(self) -> str:
 		s = f'💻 [bold white]{self.ip}[/]'
 		if self.host:
-			s += f' \[[bold magenta]{self.host}[/]]'
+			s += rf' \[[bold magenta]{self.host}[/]]'
 		return rich_to_ansi(s)

secator/output_types/port.py

@@ -14,7 +14,9 @@ class Port(OutputType):
 	service_name: str = field(default='', compare=False)
 	cpes: list = field(default_factory=list, compare=False)
 	host: str = field(default='', repr=True, compare=False)
+	protocol: str = field(default='tcp', repr=True, compare=False)
 	extra_data: dict = field(default_factory=dict, compare=False)
+	confidence: str = field(default='low', repr=False, compare=False)
 	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
 	_source: str = field(default='', repr=True, compare=False)
 	_type: str = field(default='port', repr=True)
@@ -38,8 +40,13 @@ class Port(OutputType):
 
 	def __repr__(self) -> str:
 		s = f'🔓 {self.ip}:[bold red]{self.port:<4}[/] [bold yellow]{self.state.upper()}[/]'
+		if self.protocol != 'TCP':
+			s += rf' \[[yellow3]{self.protocol}[/]]'
 		if self.service_name:
-			s += f' \[[bold purple]{self.service_name}[/]]'
+			conf = ''
+			if self.confidence == 'low':
+				conf = '?'
+			s += rf' \[[bold purple]{self.service_name}{conf}[/]]'
 		if self.host:
-			s += f' \[[cyan]{self.host}[/]]'
+			s += rf' \[[cyan]{self.host}[/]]'
 		return rich_to_ansi(s)

secator/output_types/progress.py

@@ -23,6 +23,11 @@ class Progress(OutputType):
 	_table_fields = ['percent', 'duration']
 	_sort_by = ('percent',)
 
+	def __post_init__(self):
+		super().__post_init__()
+		if not 0 <= self.percent <= 100:
+			self.percent = 0
+
 	def __str__(self) -> str:
 		return f'{self.percent}%'
 

secator/output_types/record.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
 
 from secator.definitions import HOST, NAME, TYPE
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, rich_escape as _s
 
 
 @dataclass
@@ -28,7 +28,9 @@ class Record(OutputType):
 		return self.name
 
 	def __repr__(self) -> str:
-		s = f'🎤 [bold white]{self.name}[/] \[[green]{self.type}[/]] \[[magenta]{self.host}[/]]'
+		s = rf'🎤 [bold white]{self.name}[/] \[[green]{self.type}[/]]'
+		if self.host:
+			s += rf' \[[magenta]{self.host}[/]]'
 		if self.extra_data:
-			s += ' \[[bold yellow]' + ','.join(f'{k}={v}' for k, v in self.extra_data.items()) + '[/]]'
+			s += r' \[[bold yellow]' + ','.join(f'{_s(k)}={_s(v)}' for k, v in self.extra_data.items()) + '[/]]'
 		return rich_to_ansi(s)

secator/output_types/stat.py

@@ -0,0 +1,33 @@
+import time
+from dataclasses import dataclass, field
+
+from secator.output_types import OutputType
+from secator.utils import rich_to_ansi
+
+
+@dataclass
+class Stat(OutputType):
+	name: str
+	pid: int
+	cpu: int
+	memory: int
+	net_conns: int = field(default=None, repr=True)
+	extra_data: dict = field(default_factory=dict)
+	_source: str = field(default='', repr=True)
+	_type: str = field(default='stat', repr=True)
+	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
+	_uuid: str = field(default='', repr=True, compare=False)
+	_context: dict = field(default_factory=dict, repr=True, compare=False)
+	_tagged: bool = field(default=False, repr=True, compare=False)
+	_duplicate: bool = field(default=False, repr=True, compare=False)
+	_related: list = field(default_factory=list, compare=False)
+
+	_table_fields = ['name', 'pid', 'cpu', 'memory']
+	_sort_by = ('name', 'pid')
+
+	def __repr__(self) -> str:
+		s = rf'[dim yellow3]📊 {self.name} \[pid={self.pid}] \[cpu={self.cpu:.2f}%] \[memory={self.memory:.2f}%]'
+		if self.net_conns:
+			s += rf' \[connections={self.net_conns}]'
+		s += ' [/]'
+		return rich_to_ansi(s)

secator/output_types/subdomain.py

@@ -38,5 +38,5 @@ class Subdomain(OutputType):
 		if sources_str:
 			s += f' [{sources_str}]'
 		if self.extra_data:
-			s += ' \[[bold yellow]' + ', '.join(f'{k}:{v}' for k, v in self.extra_data.items()) + '[/]]'
+			s += r' \[[bold yellow]' + ', '.join(f'{k}:{v}' for k, v in self.extra_data.items()) + '[/]]'
 		return rich_to_ansi(s)

secator/output_types/tag.py

@@ -2,7 +2,7 @@ import time
 from dataclasses import dataclass, field
 
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, trim_string, rich_escape as _s
 
 
 @dataclass
@@ -30,17 +30,19 @@ class Tag(OutputType):
 
 	def __repr__(self) -> str:
 		s = f'🏷️  [bold magenta]{self.name}[/]'
-		s += f' found @ [bold]{self.match}[/]'
+		s += f' found @ [bold]{_s(self.match)}[/]'
 		ed = ''
 		if self.extra_data:
 			for k, v in self.extra_data.items():
 				sep = ' '
 				if not v:
 					continue
-				if len(v) >= 80:
-					v = v.replace('\n', '\n' + '  ').replace('...TRUNCATED', '\n[italic bold red]...truncated to 1000 chars[/]')
-					sep = '\n    '
-				ed += f'\n    [dim red]{k}[/]:{sep}[dim yellow]{v}[/]'
+				if isinstance(v, str):
+					v = trim_string(v, max_length=1000)
+					if len(v) > 1000:
+						v = v.replace('\n', '\n' + sep)
+						sep = '\n    '
+				ed += f'\n    [dim red]{_s(k)}[/]:{sep}[dim yellow]{_s(v)}[/]'
 		if ed:
 			s += ed
 		return rich_to_ansi(s)

secator/output_types/target.py

@@ -2,7 +2,7 @@ import time
 from dataclasses import dataclass, field
 
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, rich_escape as _s
 
 
 @dataclass
@@ -26,5 +26,5 @@ class Target(OutputType):
 		return self.name
 
 	def __repr__(self):
-		s = f'🎯 {self.name}'
+		s = f'🎯 {_s(self.name)}'
 		return rich_to_ansi(s)

secator/output_types/url.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from secator.definitions import (CONTENT_LENGTH, CONTENT_TYPE, STATUS_CODE,
 								 TECH, TIME, TITLE, URL, WEBSERVER)
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, trim_string, rich_escape as _s
+from secator.config import CONFIG
 
 
 @dataclass
@@ -55,25 +56,27 @@ class Url(OutputType):
 		return self.url
 
 	def __repr__(self):
-		s = f'🔗 [white]{self.url}'
+		s = f'🔗 [white]{_s(self.url)}'
 		if self.method and self.method != 'GET':
-			s += f' \[[turquoise4]{self.method}[/]]'
+			s += rf' \[[turquoise4]{self.method}[/]]'
 		if self.status_code and self.status_code != 0:
 			if self.status_code < 400:
-				s += f' \[[green]{self.status_code}[/]]'
+				s += rf' \[[green]{self.status_code}[/]]'
 			else:
-				s += f' \[[red]{self.status_code}[/]]'
+				s += rf' \[[red]{self.status_code}[/]]'
 		if self.title:
-			s += f' \[[green]{self.title}[/]]'
+			s += rf' \[[green]{trim_string(self.title)}[/]]'
 		if self.webserver:
-			s += f' \[[magenta]{self.webserver}[/]]'
+			s += rf' \[[magenta]{_s(self.webserver)}[/]]'
 		if self.tech:
-			techs_str = ', '.join([f'[magenta]{tech}[/]' for tech in self.tech])
+			techs_str = ', '.join([f'[magenta]{_s(tech)}[/]' for tech in self.tech])
 			s += f' [{techs_str}]'
 		if self.content_type:
-			s += f' \[[magenta]{self.content_type}[/]]'
+			s += rf' \[[magenta]{_s(self.content_type)}[/]]'
 		if self.content_length:
-			s += f' \[[magenta]{self.content_length}[/]]'
+			cl = str(self.content_length)
+			cl += '[bold red]+[/]' if self.content_length == CONFIG.http.response_max_size_bytes else ''
+			s += rf' \[[magenta]{cl}[/]]'
 		if self.screenshot_path:
-			s += f' \[[magenta]{self.screenshot_path}[/]]'
+			s += rf' \[[magenta]{_s(self.screenshot_path)}[/]]'
 		return rich_to_ansi(s)

secator/output_types/user_account.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
 
 from secator.definitions import SITE_NAME, URL, USERNAME
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, rich_escape as _s
 
 
 @dataclass
@@ -29,13 +29,13 @@ class UserAccount(OutputType):
 		return self.url
 
 	def __repr__(self) -> str:
-		s = f'👤 [green]{self.username}[/]'
+		s = f'👤 [green]{_s(self.username)}[/]'
 		if self.email:
-			s += f' \[[bold yellow]{self.email}[/]]'
+			s += rf' \[[bold yellow]{_s(self.email)}[/]]'
 		if self.site_name:
-			s += f' \[[bold blue]{self.site_name}[/]]'
+			s += rf' \[[bold blue]{self.site_name}[/]]'
 		if self.url:
-			s += f' \[[white]{self.url}[/]]'
+			s += rf' \[[white]{_s(self.url)}[/]]'
 		if self.extra_data:
-			s += ' \[[bold yellow]' + ', '.join(f'{k}:{v}' for k, v in self.extra_data.items()) + '[/]]'
+			s += r' \[[bold yellow]' + _s(', '.join(f'{k}:{v}' for k, v in self.extra_data.items()) + '[/]]')
 		return rich_to_ansi(s)

secator/output_types/vulnerability.py

@@ -5,7 +5,7 @@ from typing import List
 from secator.definitions import (CONFIDENCE, CVSS_SCORE, EXTRA_DATA, ID,
 								 MATCHED_AT, NAME, REFERENCE, SEVERITY, TAGS)
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, rich_escape as _s
 
 
 @dataclass
@@ -58,7 +58,8 @@ class Vulnerability(OutputType):
 			'unknown': 5,
 			None: 6
 		}
-		self.severity_nb = severity_map[self.severity]
+		self.severity = self.severity.lower()  # normalize severity
+		self.severity_nb = severity_map.get(self.severity, 6)
 		self.confidence_nb = severity_map[self.confidence]
 		if len(self.references) > 0:
 			self.reference = self.references[0]
@@ -69,6 +70,7 @@ class Vulnerability(OutputType):
 			data = ','.join(data['data'])
 		elif isinstance(data, dict):
 			data = ', '.join([f'{k}:{v}' for k, v in data.items()])
+		data = _s(data)
 		tags = self.tags
 		colors = {
 			'critical': 'bold red',
@@ -78,13 +80,13 @@ class Vulnerability(OutputType):
 			'info': 'magenta',
 			'unknown': 'dim magenta'
 		}
-		c = colors[self.severity]
-		s = f'🚨 \[[green]{self.name} [link={self.reference}]🡕[/link][/]] \[[{c}]{self.severity}[/]] {self.matched_at}'
+		c = colors.get(self.severity, 'dim magenta')
+		s = rf'🚨 \[[green]{_s(self.name)} [link={_s(self.reference)}]🡕[/link][/]] \[[{c}]{self.severity}[/]] {_s(self.matched_at)}'  # noqa: E501
 		if tags:
 			tags_str = ','.join(tags)
-			s += f' \[[cyan]{tags_str}[/]]'
+			s += rf' \[[cyan]{_s(tags_str)}[/]]'
 		if data:
-			s += f' \[[yellow]{str(data)}[/]]'
+			s += rf' \[[yellow]{str(data)}[/]]'
 		if self.confidence == 'low':
 			s = f'[dim]{s}[/]'
 		return rich_to_ansi(s)

secator/output_types/warning.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass, field
+import time
+from secator.output_types import OutputType
+from secator.utils import rich_to_ansi, rich_escape as _s
+
+
+@dataclass
+class Warning(OutputType):
+	message: str
+	task_id: str = field(default='', compare=False)
+	_source: str = field(default='', repr=True)
+	_type: str = field(default='warning', repr=True)
+	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
+	_uuid: str = field(default='', repr=True, compare=False)
+	_context: dict = field(default_factory=dict, repr=True, compare=False)
+	_duplicate: bool = field(default=False, repr=True, compare=False)
+	_related: list = field(default_factory=list, compare=False)
+
+	_table_fields = ['task_name', 'message']
+	_sort_by = ('_timestamp',)
+
+	def __repr__(self):
+		s = rf"\[[yellow]WRN[/]] {_s(self.message)}"
+		return rich_to_ansi(s)

secator/report.py

@@ -1,9 +1,31 @@
 import operator
 
 from secator.config import CONFIG
-from secator.output_types import OUTPUT_TYPES, OutputType
-from secator.utils import merge_opts, get_file_timestamp
+from secator.output_types import FINDING_TYPES, OutputType
+from secator.utils import merge_opts, get_file_timestamp, traceback_as_string
 from secator.rich import console
+from secator.runners._helpers import extract_from_results
+
+import concurrent.futures
+from threading import Lock
+
+
+def remove_duplicates(objects):
+	unique_objects = []
+	lock = Lock()
+
+	def add_if_unique(obj):
+		nonlocal unique_objects
+		with lock:
+			# Perform linear search to check for duplicates
+			if all(obj != existing_obj for existing_obj in unique_objects):
+				unique_objects.append(obj)
+
+	with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+		# Execute the function concurrently for each object
+		executor.map(add_if_unique, objects)
+
+	return unique_objects
 
 
 # TODO: initialize from data, not from runner
@@ -29,54 +51,65 @@ class Report:
 				report_cls(self).send()
 			except Exception as e:
 				console.print(
-					f'Could not create exporter {report_cls.__name__} for {self.__class__.__name__}: {str(e)}',
-					style='bold red')
+					f'[bold red]Could not create exporter {report_cls.__name__} for {self.__class__.__name__}: '
+					f'{str(e)}[/]\n[dim]{traceback_as_string(e)}[/]',
+				)
 
-	def build(self):
+	def build(self, extractors=[], dedupe=False):
 		# Trim options
 		from secator.decorators import DEFAULT_CLI_OPTIONS
 		opts = merge_opts(self.runner.config.options, self.runner.run_opts)
 		opts = {
 			k: v for k, v in opts.items()
-			if k not in DEFAULT_CLI_OPTIONS
-			and not k.startswith('print_')
+			if k not in DEFAULT_CLI_OPTIONS and k not in self.runner.print_opts
 			and v is not None
 		}
+		runner_fields = {
+			'name',
+			'status',
+			'targets',
+			'start_time',
+			'end_time',
+			'elapsed',
+			'elapsed_human',
+			'run_opts',
+			'results_count'
+		}
 
 		# Prepare report structure
 		data = {
-			'info': {
-				'title': self.title,
-				'runner': self.runner.__class__.__name__,
-				'name': self.runner.config.name,
-				'targets': self.runner.targets,
-				'total_time': str(self.runner.elapsed),
-				'total_human': self.runner.elapsed_human,
-				'opts': opts,
-			},
-			'results': {},
+			'info': {k: v for k, v in self.runner.toDict().items() if k in runner_fields},
+			'results': {}
 		}
+		if 'results' in data['info']:
+			del data['info']['results']
+		data['info']['title'] = self.title
+		data['info']['errors'] = self.runner.errors
 
 		# Fill report
-		for output_type in OUTPUT_TYPES:
-			if output_type.__name__ == 'Progress':
-				continue
+		for output_type in FINDING_TYPES:
 			output_name = output_type.get_name()
 			sort_by, _ = get_table_fields(output_type)
 			items = [
 				item for item in self.runner.results
 				if isinstance(item, OutputType) and item._type == output_name
 			]
-			if CONFIG.runners.remove_duplicates:
-				items = [item for item in items if not item._duplicate]
 			if items:
 				if sort_by and all(sort_by):
 					items = sorted(items, key=operator.attrgetter(*sort_by))
+				if dedupe and CONFIG.runners.remove_duplicates:
+					items = remove_duplicates(items)
+					# items = [item for item in items if not item._duplicate and item not in dedupe_from]
+				for extractor in extractors:
+					items = extract_from_results(items, extractors=[extractor])
 				data['results'][output_name] = items
 
 		# Save data
 		self.data = data
 
+	def is_empty(self):
+		return all(not items for items in self.data['results'].values())
+
 
 def get_table_fields(output_type):
 	"""Get output fields and sort fields based on output type.
@@ -89,7 +122,7 @@ def get_table_fields(output_type):
 	"""
 	sort_by = ()
 	output_fields = []
-	if output_type in OUTPUT_TYPES:
+	if output_type in FINDING_TYPES:
 		sort_by = output_type._sort_by
 		output_fields = output_type._table_fields
 	return sort_by, output_fields

secator/rich.py

@@ -1,7 +1,6 @@
 import operator
 
 import yaml
-from rich import box
 from rich.console import Console
 from rich.table import Table
 
@@ -67,51 +66,57 @@ def build_table(items, output_fields=[], exclude_fields=[], sort_by=None):
 		items = sorted(items, key=operator.attrgetter(*sort_by))
 
 	# Create rich table
-	box_style = box.ROUNDED
-	table = Table(show_lines=True, box=box_style)
+	table = Table(show_lines=True)
 
 	# Get table schema if any, default to first item keys
-	keys = output_fields
-
-	# List of fields to exclude
-	keys = [k for k in keys if k not in exclude_fields]
-
-	# Remove meta fields not needed in output
-	if '_cls' in keys:
-		keys.remove('_cls')
-	if '_type' in keys:
-		keys.remove('_type')
-	if '_uuid' in keys:
-		keys.remove('_uuid')
-
-	# Add _source field
-	if '_source' not in keys:
-		keys.append('_source')
-
-	# Create table columns
-	for key in keys:
-		key_str = key
-		if not key.startswith('_'):
-			key_str = ' '.join(key.split('_')).upper()
-		no_wrap = key in ['url', 'reference', 'references', 'matched_at']
-		overflow = None if no_wrap else 'fold'
+	keys = []
+	if output_fields:
+		keys = [k for k in output_fields if k not in exclude_fields]
+		# Remove meta fields not needed in output
+		if '_cls' in keys:
+			keys.remove('_cls')
+		if '_type' in keys:
+			keys.remove('_type')
+		if '_uuid' in keys:
+			keys.remove('_uuid')
+
+		# Add _source field
+		if '_source' not in keys:
+			keys.append('_source')
+
+		# Create table columns
+		for key in keys:
+			key_str = key
+			if not key.startswith('_'):
+				key_str = ' '.join(key.split('_')).title()
+			no_wrap = key in ['url', 'reference', 'references', 'matched_at']
+			overflow = None if no_wrap else 'fold'
+			table.add_column(
+				key_str,
+				overflow=overflow,
+				min_width=10,
+				no_wrap=no_wrap)
+
+	if not keys:
 		table.add_column(
-			key_str,
-			overflow=overflow,
+			'Extracted values',
+			overflow=False,
 			min_width=10,
-			no_wrap=no_wrap,
-			header_style='bold blue')
+			no_wrap=False)
 
 	# Create table rows
 	for item in items:
 		values = []
-		for key in keys:
-			value = getattr(item, key)
-			value = FORMATTERS.get(key, lambda x: x)(value)
-			if isinstance(value, dict) or isinstance(value, list):
-				value = yaml.dump(value)
-			elif isinstance(value, int) or isinstance(value, float):
-				value = str(value)
-			values.append(value)
+		if keys:
+			for key in keys:
+				value = getattr(item, key) if keys else item
+				value = FORMATTERS.get(key, lambda x: x)(value) if keys else item
+				if isinstance(value, dict) or isinstance(value, list):
+					value = yaml.dump(value)
+				elif isinstance(value, int) or isinstance(value, float):
+					value = str(value)
+				values.append(value)
+		else:
+			values = [item]
 		table.add_row(*values)
 	return table