secator 0.5.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

secator/output_types/progress.py

@@ -8,7 +8,7 @@ from secator.utils import rich_to_ansi
 @dataclass
 class Progress(OutputType):
 	duration: str
-	percent: int
+	percent: int = 0
 	errors: list = field(default_factory=list)
 	extra_data: dict = field(default_factory=dict)
 	_source: str = field(default='', repr=True)
@@ -23,6 +23,11 @@ class Progress(OutputType):
 	_table_fields = ['percent', 'duration']
 	_sort_by = ('percent',)
 
+	def __post_init__(self):
+		super().__post_init__()
+		if not 0 <= self.percent <= 100:
+			self.percent = 0
+
 	def __str__(self) -> str:
 		return f'{self.percent}%'
 

secator/output_types/record.py

@@ -28,7 +28,9 @@ class Record(OutputType):
 		return self.name
 
 	def __repr__(self) -> str:
-		s = f'🎤 [bold white]{self.name}[/] \[[green]{self.type}[/]] \[[magenta]{self.host}[/]]'
+		s = f'🎤 [bold white]{self.name}[/] \[[green]{self.type}[/]]'
+		if self.host:
+			s += f' \[[magenta]{self.host}[/]]'
 		if self.extra_data:
 			s += ' \[[bold yellow]' + ','.join(f'{k}={v}' for k, v in self.extra_data.items()) + '[/]]'
 		return rich_to_ansi(s)

secator/output_types/stat.py

@@ -0,0 +1,33 @@
+import time
+from dataclasses import dataclass, field
+
+from secator.output_types import OutputType
+from secator.utils import rich_to_ansi
+
+
+@dataclass
+class Stat(OutputType):
+	name: str
+	pid: int
+	cpu: int
+	memory: int
+	net_conns: int = field(default=None, repr=True)
+	extra_data: dict = field(default_factory=dict)
+	_source: str = field(default='', repr=True)
+	_type: str = field(default='stat', repr=True)
+	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
+	_uuid: str = field(default='', repr=True, compare=False)
+	_context: dict = field(default_factory=dict, repr=True, compare=False)
+	_tagged: bool = field(default=False, repr=True, compare=False)
+	_duplicate: bool = field(default=False, repr=True, compare=False)
+	_related: list = field(default_factory=list, compare=False)
+
+	_table_fields = ['name', 'pid', 'cpu', 'memory']
+	_sort_by = ('name', 'pid')
+
+	def __repr__(self) -> str:
+		s = f'[dim yellow3]📊 {self.name} \[pid={self.pid}] \[cpu={self.cpu:.2f}%] \[memory={self.memory:.2f}%]'
+		if self.net_conns:
+			s += f' \[connections={self.net_conns}]'
+		s += ' [/]'
+		return rich_to_ansi(s)

secator/output_types/tag.py

@@ -2,7 +2,7 @@ import time
 from dataclasses import dataclass, field
 
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, trim_string
 
 
 @dataclass
@@ -37,9 +37,11 @@ class Tag(OutputType):
 				sep = ' '
 				if not v:
 					continue
-				if len(v) >= 80:
-					v = v.replace('\n', '\n' + '  ').replace('...TRUNCATED', '\n[italic bold red]...truncated to 1000 chars[/]')
-					sep = '\n    '
+				if isinstance(v, str):
+					v = trim_string(v, max_length=1000)
+					if len(v) > 1000:
+						v = v.replace('\n', '\n' + sep)
+						sep = '\n    '
 				ed += f'\n    [dim red]{k}[/]:{sep}[dim yellow]{v}[/]'
 		if ed:
 			s += ed

secator/output_types/url.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from secator.definitions import (CONTENT_LENGTH, CONTENT_TYPE, STATUS_CODE,
 								 TECH, TIME, TITLE, URL, WEBSERVER)
 from secator.output_types import OutputType
-from secator.utils import rich_to_ansi
+from secator.utils import rich_to_ansi, trim_string
+from secator.config import CONFIG
 
 
 @dataclass
@@ -64,7 +65,7 @@ class Url(OutputType):
 			else:
 				s += f' \[[red]{self.status_code}[/]]'
 		if self.title:
-			s += f' \[[green]{self.title}[/]]'
+			s += f' \[[green]{trim_string(self.title)}[/]]'
 		if self.webserver:
 			s += f' \[[magenta]{self.webserver}[/]]'
 		if self.tech:
@@ -73,7 +74,9 @@ class Url(OutputType):
 		if self.content_type:
 			s += f' \[[magenta]{self.content_type}[/]]'
 		if self.content_length:
-			s += f' \[[magenta]{self.content_length}[/]]'
+			cl = str(self.content_length)
+			cl += '[bold red]+[/]' if self.content_length == CONFIG.http.response_max_size_bytes else ''
+			s += f' \[[magenta]{cl}[/]]'
 		if self.screenshot_path:
 			s += f' \[[magenta]{self.screenshot_path}[/]]'
 		return rich_to_ansi(s)

secator/output_types/vulnerability.py

@@ -58,7 +58,8 @@ class Vulnerability(OutputType):
 			'unknown': 5,
 			None: 6
 		}
-		self.severity_nb = severity_map[self.severity]
+		self.severity = self.severity.lower()  # normalize severity
+		self.severity_nb = severity_map.get(self.severity, 6)
 		self.confidence_nb = severity_map[self.confidence]
 		if len(self.references) > 0:
 			self.reference = self.references[0]
@@ -78,7 +79,7 @@ class Vulnerability(OutputType):
 			'info': 'magenta',
 			'unknown': 'dim magenta'
 		}
-		c = colors[self.severity]
+		c = colors.get(self.severity, 'dim magenta')
 		s = f'🚨 \[[green]{self.name} [link={self.reference}]🡕[/link][/]] \[[{c}]{self.severity}[/]] {self.matched_at}'
 		if tags:
 			tags_str = ','.join(tags)

secator/output_types/warning.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass, field
+import time
+from secator.output_types import OutputType
+from secator.utils import rich_to_ansi
+
+
+@dataclass
+class Warning(OutputType):
+	message: str
+	task_id: str = field(default='', compare=False)
+	_source: str = field(default='', repr=True)
+	_type: str = field(default='warning', repr=True)
+	_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
+	_uuid: str = field(default='', repr=True, compare=False)
+	_context: dict = field(default_factory=dict, repr=True, compare=False)
+	_duplicate: bool = field(default=False, repr=True, compare=False)
+	_related: list = field(default_factory=list, compare=False)
+
+	_table_fields = ['task_name', 'message']
+	_sort_by = ('_timestamp',)
+
+	def __repr__(self):
+		s = f"[orange4]⚠ {self.message}[/]"
+		return rich_to_ansi(s)

secator/report.py

@@ -1,9 +1,31 @@
 import operator
 
 from secator.config import CONFIG
-from secator.output_types import OUTPUT_TYPES, OutputType
-from secator.utils import merge_opts, get_file_timestamp
+from secator.output_types import FINDING_TYPES, OutputType
+from secator.utils import merge_opts, get_file_timestamp, traceback_as_string
 from secator.rich import console
+from secator.runners._helpers import extract_from_results
+
+import concurrent.futures
+from threading import Lock
+
+
+def remove_duplicates(objects):
+	unique_objects = []
+	lock = Lock()
+
+	def add_if_unique(obj):
+		nonlocal unique_objects
+		with lock:
+			# Perform linear search to check for duplicates
+			if all(obj != existing_obj for existing_obj in unique_objects):
+				unique_objects.append(obj)
+
+	with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+		# Execute the function concurrently for each object
+		executor.map(add_if_unique, objects)
+
+	return unique_objects
 
 
 # TODO: initialize from data, not from runner
@@ -29,54 +51,64 @@ class Report:
 				report_cls(self).send()
 			except Exception as e:
 				console.print(
-					f'Could not create exporter {report_cls.__name__} for {self.__class__.__name__}: {str(e)}',
-					style='bold red')
+					f'[bold red]Could not create exporter {report_cls.__name__} for {self.__class__.__name__}: '
+					f'{str(e)}[/]\n[dim]{traceback_as_string(e)}[/]',
+				)
 
-	def build(self):
+	def build(self, extractors=[], dedupe=False):
 		# Trim options
 		from secator.decorators import DEFAULT_CLI_OPTIONS
 		opts = merge_opts(self.runner.config.options, self.runner.run_opts)
 		opts = {
 			k: v for k, v in opts.items()
-			if k not in DEFAULT_CLI_OPTIONS
-			and not k.startswith('print_')
+			if k not in DEFAULT_CLI_OPTIONS and k not in self.runner.print_opts
 			and v is not None
 		}
+		runner_fields = {
+			'name',
+			'status',
+			'targets',
+			'start_time',
+			'end_time',
+			'elapsed',
+			'elapsed_human',
+			'run_opts',
+			'results_count'
+		}
 
 		# Prepare report structure
 		data = {
-			'info': {
-				'title': self.title,
-				'runner': self.runner.__class__.__name__,
-				'name': self.runner.config.name,
-				'targets': self.runner.targets,
-				'total_time': str(self.runner.elapsed),
-				'total_human': self.runner.elapsed_human,
-				'opts': opts,
-			},
-			'results': {},
+			'info': {k: v for k, v in self.runner.toDict().items() if k in runner_fields},
+			'results': {}
 		}
+		if 'results' in data['info']:
+			del data['info']['results']
+		data['info']['title'] = self.title
 
 		# Fill report
-		for output_type in OUTPUT_TYPES:
-			if output_type.__name__ == 'Progress':
-				continue
+		for output_type in FINDING_TYPES:
 			output_name = output_type.get_name()
 			sort_by, _ = get_table_fields(output_type)
 			items = [
 				item for item in self.runner.results
 				if isinstance(item, OutputType) and item._type == output_name
 			]
-			if CONFIG.runners.remove_duplicates:
-				items = [item for item in items if not item._duplicate]
 			if items:
 				if sort_by and all(sort_by):
 					items = sorted(items, key=operator.attrgetter(*sort_by))
+				if dedupe and CONFIG.runners.remove_duplicates:
+					items = remove_duplicates(items)
+					# items = [item for item in items if not item._duplicate and item not in dedupe_from]
+				for extractor in extractors:
+					items = extract_from_results(items, extractors=[extractor])
 				data['results'][output_name] = items
 
 		# Save data
 		self.data = data
 
+	def is_empty(self):
+		return all(not items for items in self.data['results'].values())
+
 
 def get_table_fields(output_type):
 	"""Get output fields and sort fields based on output type.
@@ -89,7 +121,7 @@ def get_table_fields(output_type):
 	"""
 	sort_by = ()
 	output_fields = []
-	if output_type in OUTPUT_TYPES:
+	if output_type in FINDING_TYPES:
 		sort_by = output_type._sort_by
 		output_fields = output_type._table_fields
 	return sort_by, output_fields

secator/rich.py

@@ -1,7 +1,6 @@
 import operator
 
 import yaml
-from rich import box
 from rich.console import Console
 from rich.table import Table
 
@@ -67,51 +66,57 @@ def build_table(items, output_fields=[], exclude_fields=[], sort_by=None):
 		items = sorted(items, key=operator.attrgetter(*sort_by))
 
 	# Create rich table
-	box_style = box.ROUNDED
-	table = Table(show_lines=True, box=box_style)
+	table = Table(show_lines=True)
 
 	# Get table schema if any, default to first item keys
-	keys = output_fields
-
-	# List of fields to exclude
-	keys = [k for k in keys if k not in exclude_fields]
-
-	# Remove meta fields not needed in output
-	if '_cls' in keys:
-		keys.remove('_cls')
-	if '_type' in keys:
-		keys.remove('_type')
-	if '_uuid' in keys:
-		keys.remove('_uuid')
-
-	# Add _source field
-	if '_source' not in keys:
-		keys.append('_source')
-
-	# Create table columns
-	for key in keys:
-		key_str = key
-		if not key.startswith('_'):
-			key_str = ' '.join(key.split('_')).upper()
-		no_wrap = key in ['url', 'reference', 'references', 'matched_at']
-		overflow = None if no_wrap else 'fold'
+	keys = []
+	if output_fields:
+		keys = [k for k in output_fields if k not in exclude_fields]
+		# Remove meta fields not needed in output
+		if '_cls' in keys:
+			keys.remove('_cls')
+		if '_type' in keys:
+			keys.remove('_type')
+		if '_uuid' in keys:
+			keys.remove('_uuid')
+
+		# Add _source field
+		if '_source' not in keys:
+			keys.append('_source')
+
+		# Create table columns
+		for key in keys:
+			key_str = key
+			if not key.startswith('_'):
+				key_str = ' '.join(key.split('_')).title()
+			no_wrap = key in ['url', 'reference', 'references', 'matched_at']
+			overflow = None if no_wrap else 'fold'
+			table.add_column(
+				key_str,
+				overflow=overflow,
+				min_width=10,
+				no_wrap=no_wrap)
+
+	if not keys:
 		table.add_column(
-			key_str,
-			overflow=overflow,
+			'Extracted values',
+			overflow=False,
 			min_width=10,
-			no_wrap=no_wrap,
-			header_style='bold blue')
+			no_wrap=False)
 
 	# Create table rows
 	for item in items:
 		values = []
-		for key in keys:
-			value = getattr(item, key)
-			value = FORMATTERS.get(key, lambda x: x)(value)
-			if isinstance(value, dict) or isinstance(value, list):
-				value = yaml.dump(value)
-			elif isinstance(value, int) or isinstance(value, float):
-				value = str(value)
-			values.append(value)
+		if keys:
+			for key in keys:
+				value = getattr(item, key) if keys else item
+				value = FORMATTERS.get(key, lambda x: x)(value) if keys else item
+				if isinstance(value, dict) or isinstance(value, list):
+					value = yaml.dump(value)
+				elif isinstance(value, int) or isinstance(value, float):
+					value = str(value)
+				values.append(value)
+		else:
+			values = [item]
 		table.add_row(*values)
 	return table