secator 0.16.2__tar.gz → 0.16.4__tar.gz

{secator-0.16.2 → secator-0.16.4}/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [0.16.4](https://github.com/freelabz/secator/compare/v0.16.3...v0.16.4) (2025-06-13)
+
+
+### Bug Fixes
+
+* **dalfox:** reduce chunk size for dalfox ([#700](https://github.com/freelabz/secator/issues/700)) ([c14be68](https://github.com/freelabz/secator/commit/c14be68427d18072cf75c61fb6ae966f97515d15))
+* **gcs:** add stored_response_path to sent items ([#697](https://github.com/freelabz/secator/issues/697)) ([7c6f992](https://github.com/freelabz/secator/commit/7c6f992b6c7898e956436e169b64af1d9f1d8934))
+* mongodb optimizations ([#699](https://github.com/freelabz/secator/issues/699)) ([c0497a6](https://github.com/freelabz/secator/commit/c0497a67c293680dafdc052eff510ffd17edafe6))
+
+## [0.16.3](https://github.com/freelabz/secator/compare/v0.16.2...v0.16.3) (2025-06-11)
+
+
+### Bug Fixes
+
+* bup file flag ([#690](https://github.com/freelabz/secator/issues/690)) ([83d83d7](https://github.com/freelabz/secator/commit/83d83d72c5c523ea44dcc3cb78f478c935ed4127))
+* **celery:** add no_live_updates to skip backend updates ([#695](https://github.com/freelabz/secator/issues/695)) ([c99599c](https://github.com/freelabz/secator/commit/c99599caa219312e8ca2e190d8353960ca4f2633))
+* **dalfox:** use jsonl option instead of json ([#692](https://github.com/freelabz/secator/issues/692)) ([c406e34](https://github.com/freelabz/secator/commit/c406e346a4011b0c57efd7b6d5e518ee6d7953e6))
+* **mongodb:** add all output types ([#696](https://github.com/freelabz/secator/issues/696)) ([af98935](https://github.com/freelabz/secator/commit/af989352662f53903768c268347e6046db9dd8fd))
+* show info message for mark_runner_started / mark_runner_completed ([#694](https://github.com/freelabz/secator/issues/694)) ([ed0f6cb](https://github.com/freelabz/secator/commit/ed0f6cb1464c962fc6f65381762fd23330f9aa85))
+* **workflow:** put cariddi in crawlers group ([#693](https://github.com/freelabz/secator/issues/693)) ([a9e1afd](https://github.com/freelabz/secator/commit/a9e1afd97256aefc33d179d9e3d4091f24b332b2))
+
 ## [0.16.2](https://github.com/freelabz/secator/compare/v0.16.1...v0.16.2) (2025-06-06)
 
 

{secator-0.16.2 → secator-0.16.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: secator
-Version: 0.16.2
+Version: 0.16.4
 Summary: The pentester's swiss knife.
 Project-URL: Homepage, https://github.com/freelabz/secator
 Project-URL: Issues, https://github.com/freelabz/secator/issues

{secator-0.16.2 → secator-0.16.4}/cloudbuild.yaml

@@ -1,12 +1,13 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: 'bash'
-  args: ['-c', 'docker pull ${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:latest || exit 0']
+  args: ['-c', 'docker pull ${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION} || exit 0']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '-t', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator', '--cache-from', 'europe-west1-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:latest', '.']
+  args: ['build', '-t', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '--cache-from', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '.']
 substitutions:
   _REPOSITORY: secator
   _LOCATION: europe-west1
+  _VERSION: latest
 
 images:
-- '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator'
+- '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}'

{secator-0.16.2 → secator-0.16.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'hatchling.build'
 
 [project]
 name = 'secator'
-version = "0.16.2"
+version = "0.16.4"
 authors = [{ name = 'FreeLabz', email = 'sales@freelabz.com' }]
 readme = 'README.md'
 description = "The pentester's swiss knife."

{secator-0.16.2 → secator-0.16.4}/secator/celery.py

@@ -100,6 +100,8 @@ def update_state(celery_task, task, force=False):
 	"""Update task state to add metadata information."""
 	if not IN_CELERY_WORKER_PROCESS:
 		return
+	if task.no_live_updates:
+		return
 	if not force and not should_update(CONFIG.runners.backend_update_frequency, task.last_updated_celery):
 		return
 	task.last_updated_celery = time()
@@ -212,19 +214,42 @@ def run_command(self, results, name, targets, opts={}):
 		update_state(self, task)
 	update_state(self, task, force=True)
 
+	if CONFIG.addons.mongodb.enabled:
+		return [r._uuid for r in task.results]
 	return task.results
 
 
 @app.task
 def forward_results(results):
+	"""Forward results to the next task (bridge task).
+
+	Args:
+		results (list): Results to forward.
+
+	Returns:
+		list: List of uuids.
+	"""
 	if isinstance(results, list):
 		for ix, item in enumerate(results):
 			if isinstance(item, dict) and 'results' in item:
 				results[ix] = item['results']
 	elif 'results' in results:
 		results = results['results']
+
+	if IN_CELERY_WORKER_PROCESS:
+		console.print(Info(message=f'Deduplicating {len(results)} results'))
+
 	results = flatten(results)
-	results = deduplicate(results, attr='_uuid')
+	if CONFIG.addons.mongodb.enabled:
+		uuids = [r._uuid for r in results if hasattr(r, '_uuid')]
+		uuids.extend([r for r in results if isinstance(r, str)])
+		results = list(set(uuids))
+	else:
+		results = deduplicate(results, attr='_uuid')
+
+	if IN_CELERY_WORKER_PROCESS:
+		console.print(Info(message=f'Forwarded {len(results)} flattened and deduplicated results'))
+
 	return results
 
 
@@ -240,10 +265,17 @@ def mark_runner_started(results, runner, enable_hooks=True):
 	Returns:
 		list: Runner results
 	"""
+	if IN_CELERY_WORKER_PROCESS:
+		console.print(Info(message=f'Runner {runner.unique_name} has started, running mark_started'))
 	debug(f'Runner {runner.unique_name} has started, running mark_started', sub='celery')
 	if results:
-		runner.results = forward_results(results)
+		results = forward_results(results)
 	runner.enable_hooks = enable_hooks
+	if CONFIG.addons.mongodb.enabled:
+		from secator.hooks.mongodb import get_results
+		results = get_results(results)
+	for item in results:
+		runner.add_result(item, print=False)
 	runner.mark_started()
 	return runner.results
 
@@ -260,9 +292,14 @@ def mark_runner_completed(results, runner, enable_hooks=True):
 	Returns:
 		list: Final results
 	"""
+	if IN_CELERY_WORKER_PROCESS:
+		console.print(Info(message=f'Runner {runner.unique_name} has finished, running mark_completed'))
 	debug(f'Runner {runner.unique_name} has finished, running mark_completed', sub='celery')
 	results = forward_results(results)
 	runner.enable_hooks = enable_hooks
+	if CONFIG.addons.mongodb.enabled:
+		from secator.hooks.mongodb import get_results
+		results = get_results(results)
 	for item in results:
 		runner.add_result(item, print=False)
 	runner.mark_completed()

{secator-0.16.2 → secator-0.16.4}/secator/configs/workflows/url_crawl.yaml

@@ -9,7 +9,7 @@ input_types:
 options:
   crawlers:
     type: list
-    help: Crawlers to use (katana, gospider)
+    help: Crawlers to use
     default: ['gau', 'katana']
     internal: True
 
@@ -36,17 +36,14 @@ tasks:
       description: Crawl URLs
       if: "'gospider' in opts.crawlers"
 
-  cariddi:
-    description: Hunt URLs patterns
-    info: True
-    secrets: True
-    errors: True
-    juicy_extensions: 1
-    juicy_endpoints: True
-    targets_:
-    - target.name
-    - url.url
-    if: opts.hunt_patterns
+    cariddi:
+      description: Hunt URLs patterns
+      info: True
+      secrets: True
+      errors: True
+      juicy_extensions: 1
+      juicy_endpoints: True
+      if: opts.hunt_patterns
 
   httpx:
     description: Run HTTP probes on crawled URLs

{secator-0.16.2 → secator-0.16.4}/secator/hooks/gcs.py

@@ -11,7 +11,7 @@ from secator.utils import debug
 
 GCS_BUCKET_NAME = CONFIG.addons.gcs.bucket_name
 ITEMS_TO_SEND = {
-	'url': ['screenshot_path']
+	'url': ['screenshot_path', 'stored_response_path']
 }
 
 

{secator-0.16.2 → secator-0.16.4}/secator/hooks/mongodb.py

@@ -6,7 +6,7 @@ from bson.objectid import ObjectId
 from celery import shared_task
 
 from secator.config import CONFIG
-from secator.output_types import FINDING_TYPES
+from secator.output_types import OUTPUT_TYPES
 from secator.runners import Scan, Task, Workflow
 from secator.utils import debug, escape_mongodb_url
 
@@ -46,6 +46,28 @@ def get_runner_dbg(runner):
 	}
 
 
+def get_results(uuids):
+	"""Get results from MongoDB based on a list of uuids.
+
+	Args:
+		uuids (list[str | Output]): List of uuids, but can also be a mix of uuids and output types.
+
+	Returns:
+		Generator of findings.
+	"""
+	client = get_mongodb_client()
+	db = client.main
+	del_uuids = []
+	for r in uuids:
+		if isinstance(r, tuple(OUTPUT_TYPES)):
+			yield r
+			del_uuids.append(r)
+	uuids = [ObjectId(u) for u in uuids if u not in del_uuids and ObjectId.is_valid(u)]
+	for r in db.findings.find({'_id': {'$in': uuids}}):
+		finding = load_finding(r)
+		yield finding
+
+
 def update_runner(self):
 	client = get_mongodb_client()
 	db = client.main
@@ -78,7 +100,7 @@ def update_runner(self):
 
 
 def update_finding(self, item):
-	if type(item) not in FINDING_TYPES:
+	if type(item) not in OUTPUT_TYPES:
 		return item
 	start_time = time.time()
 	client = get_mongodb_client()
@@ -120,7 +142,7 @@ def find_duplicates(self):
 def load_finding(obj):
 	finding_type = obj['_type']
 	klass = None
-	for otype in FINDING_TYPES:
+	for otype in OUTPUT_TYPES:
 		if finding_type == otype.get_name():
 			klass = otype
 			item = klass.load(obj)

{secator-0.16.2 → secator-0.16.4}/secator/runners/_base.py

@@ -117,6 +117,7 @@ class Runner:
 
 		# Runner process options
 		self.no_poll = self.run_opts.get('no_poll', False)
+		self.no_live_updates = self.run_opts.get('no_live_updates', False)
 		self.no_process = not self.run_opts.get('process', True)
 		self.piped_input = self.run_opts.get('piped_input', False)
 		self.piped_output = self.run_opts.get('piped_output', False)
@@ -177,6 +178,10 @@ class Runner:
 
 		# Add prior results to runner results
 		self.debug(f'adding {len(results)} prior results to runner', sub='init')
+		if CONFIG.addons.mongodb.enabled:
+			self.debug('adding prior results from MongoDB', sub='init')
+			from secator.hooks.mongodb import get_results
+			results = get_results(results)
 		for result in results:
 			self.add_result(result, print=False, output=False, hooks=False, queue=not self.has_parent)
 
@@ -188,8 +193,8 @@ class Runner:
 		for target in targets:
 			self.add_result(target, print=False, output=False)
 
-		# Run extractors on results and targets
-		self._run_extractors(results + targets)
+		# Run extractors on results
+		self._run_extractors()
 		self.debug(f'inputs ({len(self.inputs)})', obj=self.inputs, sub='init')
 		self.debug(f'run opts ({len(self.resolved_opts)})', obj=self.resolved_opts, sub='init')
 		self.debug(f'print opts ({len(self.resolved_print_opts)})', obj=self.resolved_print_opts, sub='init')
@@ -429,12 +434,12 @@ class Runner:
 			if error:
 				self.add_result(error)
 
-	def _run_extractors(self, results):
+	def _run_extractors(self):
 		"""Run extractors on results and targets."""
 		self.debug('running extractors', sub='init')
 		ctx = {'opts': DotMap(self.run_opts), 'targets': self.inputs, 'ancestor_id': self.ancestor_id}
 		inputs, run_opts, errors = run_extractors(
-			results,
+			self.results,
 			self.run_opts,
 			self.inputs,
 			ctx=ctx,

{secator-0.16.2 → secator-0.16.4}/secator/tasks/bup.py

@@ -20,6 +20,7 @@ class bup(Http):
 	output_types = [Url, Progress]
 	tags = ['url', 'bypass']
 	input_flag = '-u'
+	file_flag = '-R'
 	json_flag = '--jsonl'
 	opt_prefix = '--'
 	opts = {

{secator-0.16.2 → secator-0.16.4}/secator/tasks/dalfox.py

@@ -25,9 +25,10 @@ class dalfox(VulnHttp):
 	output_types = [Vulnerability, Url]
 	tags = ['url', 'fuzz']
 	input_flag = 'url'
+	input_chunk_size = 20
 	file_flag = 'file'
 	# input_chunk_size = 1
-	json_flag = '--format json'
+	json_flag = '--format jsonl'
 	version_flag = 'version'
 	opt_prefix = '--'
 	opt_key_map = {
@@ -65,11 +66,6 @@ class dalfox(VulnHttp):
 	proxy_http = True
 	profile = 'cpu'
 
-	@staticmethod
-	def on_line(self, line):
-		line = line.rstrip(',')
-		return line
-
 	@staticmethod
 	def on_json_loaded(self, item):
 		if item.get('type', '') == 'V':