bbot 2.4.2.6677rc0__tar.gz → 2.4.2.6706rc0__tar.gz

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.2.6677rc0
+Version: 2.4.2.6706rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.2.6677rc"
+__version__ = "v2.4.2.6706rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/core/helpers/async_helpers.py

@@ -1,10 +1,10 @@
+import time
 import uuid
 import random
 import asyncio
 import logging
 import functools
-from datetime import datetime
-from .misc import human_timedelta
+from contextlib import suppress
 from cachetools import keys, LRUCache
 from contextlib import asynccontextmanager
 
@@ -63,26 +63,27 @@ class TaskCounter:
             self._lock = asyncio.Lock()
         return self._lock
 
-    def count(self, task_name, n=1, _log=True):
+    def count(self, task_name, n=1, asyncio_task=None, _log=True):
         if callable(task_name):
             task_name = f"{task_name.__qualname__}()"
-        return self.Task(self, task_name, n=n, _log=_log)
+        return self.Task(self, task_name, n=n, _log=_log, asyncio_task=asyncio_task)
 
     class Task:
-        def __init__(self, manager, task_name, n=1, _log=True):
+        def __init__(self, manager, task_name, n=1, _log=True, asyncio_task=None):
             self.manager = manager
             self.task_name = task_name
             self.task_id = None
             self.start_time = None
             self.log = _log
             self.n = n
+            self._asyncio_task = asyncio_task
 
         async def __aenter__(self):
             self.task_id = uuid.uuid4()
             # if self.log:
             #     log.trace(f"Starting task {self.task_name} ({self.task_id})")
             async with self.manager.lock:
-                self.start_time = datetime.now()
+                self.start_time = time.time()
                 self.manager.tasks[self.task_id] = self
             return self
 
@@ -92,9 +93,29 @@ class TaskCounter:
             # if self.log:
             #     log.trace(f"Finished task {self.task_name} ({self.task_id})")
 
+        @property
+        def asyncio_task(self):
+            if self._asyncio_task is None:
+                raise AttributeError("No asyncio task associated with this task")
+            return self._asyncio_task
+
+        @property
+        def function_name(self):
+            with suppress(AttributeError):
+                return self.asyncio_task.get_coro().__name__
+            return ""
+
+        async def cancel(self):
+            self.asyncio_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self.asyncio_task
+
+        @property
+        def running_for(self):
+            return time.time() - self.start_time
+
         def __str__(self):
-            running_for = human_timedelta(datetime.now() - self.start_time)
-            return f"{self.task_name} running for {running_for}"
+            return f"{self.task_name} running for {self.running_for:.1f}s"
 
 
 def get_event_loop():

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/defaults.yml

@@ -133,6 +133,12 @@ deps:
 # Load BBOT modules from these custom paths
 module_dirs: []
 
+# maximum runtime in seconds for each module's handle_event() is 60 minutes
+# when the timeout is reached, the offending handle_event() will be cancelled and the module will move on to the next event
+module_handle_event_timeout: 3600
+# handle_batch() default timeout is 2 hours
+module_handle_batch_timeout: 7200
+
 # Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc.
 speculate: True
 # Passively search event data for URLs, hostnames, emails, etc.

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/apkpure.py

@@ -13,14 +13,16 @@ class apkpure(BaseModule):
         "author": "@domwhewell-sage",
     }
     options = {"output_folder": ""}
-    options_desc = {"output_folder": "Folder to download apk's to"}
+    options_desc = {
+        "output_folder": "Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage."
+    }
 
     async def setup(self):
-        output_folder = self.config.get("output_folder")
+        output_folder = self.config.get("output_folder", "")
         if output_folder:
             self.output_dir = Path(output_folder) / "apk_files"
         else:
-            self.output_dir = self.scan.home / "apk_files"
+            self.output_dir = self.helpers.temp_dir / "apk_files"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/base.py

@@ -61,8 +61,6 @@ class BaseModule:
 
         batch_size (int): Size of batches processed by handle_batch(). Default is 1.
 
-        batch_wait (int): Seconds to wait before force-submitting a batch. Default is 10.
-
         api_failure_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5.
 
         _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False.
@@ -102,7 +100,6 @@ class BaseModule:
 
     _module_threads = 1
     _batch_size = 1
-    batch_wait = 10
 
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
@@ -159,6 +156,13 @@ class BaseModule:
 
         self._tasks = []
         self._event_received = None
+        # maximum runtime for each module's handle_event()
+        self._default_handle_event_timeout = self.scan.config.get("module_handle_event_timeout", 60 * 60)  # 1 hour
+        self._default_handle_batch_timeout = self.scan.config.get(
+            "module_handle_batch_timeout", 60 * 60 * 2
+        )  # 2 hours
+        self._event_handler_watchdog_task = None
+        self._event_handler_watchdog_interval = self.event_handler_timeout / 10
 
         # used for optional "per host" tracking
         self._per_host_tracker = set()
@@ -399,6 +403,13 @@ class BaseModule:
             module_threads = self._module_threads
         return module_threads
 
+    @property
+    def event_handler_timeout(self):
+        module_timeout = self.config.get("module_timeout", None)
+        if module_timeout is not None:
+            return float(module_timeout)
+        return self._default_handle_event_timeout if self.batch_size <= 1 else self._default_handle_batch_timeout
+
     @property
     def auth_secret(self):
         """Indicates if the module is properly configured for authentication.
@@ -446,23 +457,28 @@ class BaseModule:
             - If a "FINISHED" event is found, invokes 'finish()' method of the module.
         """
         finish = False
-        async with self._task_counter.count(f"{self.name}.handle_batch()") as counter:
-            submitted = False
-            if self.batch_size <= 1:
-                return
-            if self.num_incoming_events > 0:
-                events, finish = await self._events_waiting()
-                if events and not self.errored:
-                    counter.n = len(events)
-                    self.verbose(f"Handling batch of {len(events):,} events")
-                    submitted = True
-                    async with self.scan._acatch(f"{self.name}.handle_batch()"):
-                        await self.handle_batch(*events)
-                    self.verbose(f"Finished handling batch of {len(events):,} events")
+        submitted = False
+        if self.batch_size <= 1:
+            return
+        if self.num_incoming_events > 0:
+            events, finish = await self._events_waiting()
+            if events and not self.errored:
+                self.verbose(f"Handling batch of {len(events):,} events")
+                event_types = {}
+                for e in events:
+                    event_types[e.type] = event_types.get(e.type, 0) + 1
+                event_types_sorted = sorted(event_types.items(), key=lambda x: x[1], reverse=True)
+                event_types_str = ", ".join(f"{k}: {v}" for k, v in event_types_sorted)
+                submitted = True
+                context = f"{self.name}.handle_batch({event_types_str})"
+                try:
+                    await self.run_task(self.handle_batch(*events), context, n=len(events))
+                except asyncio.CancelledError:
+                    self.debug(f"{context} was cancelled")
+                self.verbose(f"Finished handling batch of {len(events):,} events")
         if finish:
             context = f"{self.name}.finish()"
-            async with self.scan._acatch(context), self._task_counter.count(context):
-                await self.finish()
+            await self.run_task(self.finish(), context)
         return submitted
 
     def make_event(self, *args, **kwargs):
@@ -594,6 +610,10 @@ class BaseModule:
             asyncio.create_task(self._worker(), name=f"{self.scan.name}.{self.name}._worker()")
             for _ in range(self.module_threads)
         ]
+        self._event_handler_watchdog_task = asyncio.create_task(
+            self._event_handler_watchdog(),
+            name=f"{self.scan.name}.{self.name}._event_handler_watchdog()",
+        )
 
     async def _setup(self):
         """
@@ -662,11 +682,6 @@ class BaseModule:
         async with self.scan._acatch(context=self._worker, unhandled_is_critical=True):
             try:
                 while not self.scan.stopping and not self.errored:
-                    # hold the reigns if our outgoing queue is full
-                    if self._qsize > 0 and self.outgoing_event_queue.qsize() >= self._qsize:
-                        await asyncio.sleep(0.1)
-                        continue
-
                     # if batch wasn't big enough, we wait for the next event before continuing
                     if self.batch_size > 1:
                         submitted = await self._handle_batch()
@@ -689,14 +704,20 @@ class BaseModule:
                         if acceptable:
                             if event.type == "FINISHED":
                                 context = f"{self.name}.finish()"
-                                async with self.scan._acatch(context), self._task_counter.count(context):
-                                    await self.finish()
+                                try:
+                                    await self.run_task(self.finish(), context)
+                                except asyncio.CancelledError:
+                                    self.debug(f"{context} was cancelled")
+                                    continue
                             else:
                                 context = f"{self.name}.handle_event({event})"
                                 self.scan.stats.event_consumed(event, self)
                                 self.debug(f"Handling {event}")
-                                async with self.scan._acatch(context), self._task_counter.count(context):
-                                    await self.handle_event(event)
+                                try:
+                                    await self.run_task(self.handle_event(event), context)
+                                except asyncio.CancelledError:
+                                    self.debug(f"{context} was cancelled")
+                                    continue
                                 self.debug(f"Finished handling {event}")
                         else:
                             self.debug(f"Not accepting {event} because {reason}")
@@ -852,6 +873,36 @@ class BaseModule:
                     async with self.scan._acatch(context), self._task_counter.count(context):
                         await self.helpers.execute_sync_or_async(callback)
 
+    async def run_task(self, coro, name, n=1):
+        """
+        Start a task while tracking it in the module's task counter.
+
+        This lets us keep a detailed module status and selectively cancel tasks when needed, like when handle_event exceeds its max runtime.
+        """
+        task = asyncio.create_task(coro)
+        async with self.scan._acatch(context=name), self._task_counter.count(task_name=name, asyncio_task=task, n=n):
+            return await task
+
+    async def _event_handler_watchdog(self):
+        """
+        Watches handle_event and handle_batch tasks and cancels them if they exceed their max runtime.
+        """
+        while not self.scan.stopping and not self.errored:
+            # if there are events in the outgoing queue, we leave the tasks alone
+            if self.outgoing_event_queue.qsize() > 0:
+                await self.helpers.sleep(self._event_handler_watchdog_interval)
+                continue
+            event_handler_tasks = [
+                t for t in self._task_counter.tasks.values() if t.function_name in ("handle_event", "handle_batch")
+            ]
+            for task in event_handler_tasks:
+                if task.running_for > self.event_handler_timeout:
+                    self.warning(
+                        f"{self.name} Cancelling event handler task {task.task_name} because it's been running for {task.running_for:.1f}s (max timeout is {self.event_handler_timeout})"
+                    )
+                    await task.cancel()
+            await asyncio.sleep(self._event_handler_watchdog_interval)
+
     async def queue_event(self, event):
         """
         Asynchronously queues an incoming event to the module's event queue for further processing.
@@ -1266,7 +1317,7 @@ class BaseModule:
                 new_url, new_kwargs = iter_key(url, page, page_size, offset, **requests_kwargs)
             result = await self.api_request(new_url, **new_kwargs)
             if result is None:
-                self.verbose(f"api_page_iter() got no response for {url}")
+                self.verbose(f"api_page_iter() got no response for {new_url}")
                 break
             try:
                 if _json:
@@ -1708,10 +1759,13 @@ class BaseInterceptModule(BaseModule):
                         context = f"{self.name}.handle_event({event, kwargs})"
                         self.scan.stats.event_consumed(event, self)
                         self.debug(f"Intercepting {event}")
-                        async with self.scan._acatch(context), self._task_counter.count(context):
-                            forward_event = await self.handle_event(event, **kwargs)
-                            with suppress(ValueError, TypeError):
-                                forward_event, forward_event_reason = forward_event
+                        try:
+                            forward_event = await self.run_task(self.handle_event(event, **kwargs), context)
+                        except asyncio.CancelledError:
+                            self.debug(f"{context} was cancelled")
+                            continue
+                        with suppress(ValueError, TypeError):
+                            forward_event, forward_event_reason = forward_event
 
                         if forward_event is False:
                             self.debug(f"Not forwarding {event} because {forward_event_reason}")

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/docker_pull.py

@@ -17,7 +17,7 @@ class docker_pull(BaseModule):
     options = {"all_tags": False, "output_folder": ""}
     options_desc = {
         "all_tags": "Download all tags from each registry (Default False)",
-        "output_folder": "Folder to download docker repositories to",
+        "output_folder": "Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.",
     }
 
     scope_distance_modifier = 2
@@ -34,11 +34,11 @@ class docker_pull(BaseModule):
             )
         }
         self.all_tags = self.config.get("all_tags", True)
-        output_folder = self.config.get("output_folder")
+        output_folder = self.config.get("output_folder", "")
         if output_folder:
             self.output_dir = Path(output_folder) / "docker_images"
         else:
-            self.output_dir = self.scan.home / "docker_images"
+            self.output_dir = self.helpers.temp_dir / "docker_images"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/filedownload.py

@@ -84,12 +84,12 @@ class filedownload(BaseModule):
             "bz2",  #  Bzip2 Compressed File
         ],
         "max_filesize": "10MB",
-        "base_64_encoded_file": "false",
+        "output_folder": "",
     }
     options_desc = {
         "extensions": "File extensions to download",
         "max_filesize": "Cancel download if filesize is greater than this size",
-        "base_64_encoded_file": "Stream the bytes of a file and encode them in base 64 for event data.",
+        "output_folder": "Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage.",
     }
 
     scope_distance_modifier = 3
@@ -97,10 +97,14 @@ class filedownload(BaseModule):
     async def setup(self):
         self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
         self.max_filesize = self.config.get("max_filesize", "10MB")
-        self.download_dir = self.scan.home / "filedownload"
-        self.helpers.mkdir(self.download_dir)
         self.urls_downloaded = set()
         self.files_downloaded = 0
+        output_dir = self.config.get("output_folder", "")
+        if output_dir:
+            self.download_dir = Path(output_dir) / "filedownload"
+        else:
+            self.download_dir = self.helpers.temp_dir / "filedownload"
+        self.helpers.mkdir(self.download_dir)
         self.mime_db_file = await self.helpers.wordlist(
             "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
         )

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/git_clone.py

@@ -13,7 +13,10 @@ class git_clone(github):
         "author": "@domwhewell-sage",
     }
     options = {"api_key": "", "output_folder": ""}
-    options_desc = {"api_key": "Github token", "output_folder": "Folder to clone repositories to"}
+    options_desc = {
+        "api_key": "Github token",
+        "output_folder": "Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.",
+    }
 
     deps_apt = ["git"]
 
@@ -24,7 +27,7 @@ class git_clone(github):
         if output_folder:
             self.output_dir = Path(output_folder) / "git_repos"
         else:
-            self.output_dir = self.scan.home / "git_repos"
+            self.output_dir = self.helpers.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/gitdumper.py

@@ -1,3 +1,4 @@
+import asyncio
 import regex as re
 from pathlib import Path
 from subprocess import CalledProcessError
@@ -19,7 +20,7 @@ class gitdumper(BaseModule):
         "max_semanic_version": 10,
     }
     options_desc = {
-        "output_folder": "Folder to download repositories to",
+        "output_folder": "Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage.",
         "fuzz_tags": "Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version",
         "max_semanic_version": "Maximum version number to fuzz for (default < v10.10.10)",
     }
@@ -28,11 +29,11 @@ class gitdumper(BaseModule):
 
     async def setup(self):
         self.urls_downloaded = set()
-        output_folder = self.config.get("output_folder")
+        output_folder = self.config.get("output_folder", "")
         if output_folder:
             self.output_dir = Path(output_folder) / "git_repos"
         else:
-            self.output_dir = self.scan.home / "git_repos"
+            self.output_dir = self.helpers.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
         self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
         self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
@@ -79,6 +80,7 @@ class gitdumper(BaseModule):
             "staging",
             "test",
             "testing",
+            "trunk",
             "wip",
         ]
         url_patterns = [
@@ -121,7 +123,11 @@ class gitdumper(BaseModule):
         dir_listing = await self.directory_listing_enabled(repo_url)
         if dir_listing:
             urls = await self.recursive_dir_list(dir_listing)
-            result = await self.download_files(urls, repo_folder)
+            try:
+                result = await self.download_files(urls, repo_folder)
+            except asyncio.CancelledError:
+                self.verbose(f"Cancellation requested while downloading files from {repo_url}")
+                result = True
         else:
             result = await self.git_fuzz(repo_url, repo_folder)
         if result:
@@ -171,7 +177,10 @@ class gitdumper(BaseModule):
         result = await self.download_files(url_list, repo_folder)
         if result:
             await self.download_current_branch(repo_url, repo_folder)
-            await self.download_git_objects(repo_url, repo_folder)
+            try:
+                await self.download_git_objects(repo_url, repo_folder)
+            except asyncio.CancelledError:
+                self.verbose(f"Cancellation requested while downloading git objects from {repo_url}")
             await self.download_git_packs(repo_url, repo_folder)
             return True
         else:

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/github_workflows.py

@@ -1,5 +1,6 @@
 import zipfile
 import fnmatch
+from pathlib import Path
 
 from bbot.modules.templates.github import github
 
@@ -13,10 +14,11 @@ class github_workflows(github):
         "created_date": "2024-04-29",
         "author": "@domwhewell-sage",
     }
-    options = {"api_key": "", "num_logs": 1}
+    options = {"api_key": "", "num_logs": 1, "output_folder": ""}
     options_desc = {
         "api_key": "Github token",
         "num_logs": "For each workflow fetch the last N successful runs logs (max 100)",
+        "output_folder": "Folder to download workflow logs and artifacts to",
     }
 
     scope_distance_modifier = 2
@@ -26,7 +28,11 @@ class github_workflows(github):
         if self.num_logs > 100:
             self.log.error("num_logs option is capped at 100")
             return False
-        self.output_dir = self.scan.home / "workflow_logs"
+        output_folder = self.config.get("output_folder", "")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "workflow_logs"
+        else:
+            self.output_dir = self.scan.home / "workflow_logs"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 
@@ -35,9 +41,10 @@ class github_workflows(github):
         return r.is_success or getattr(r, "status_code", 0) == 404
 
     async def filter_event(self, event):
-        if event.type == "CODE_REPOSITORY":
-            if "git" not in event.tags and "github" not in event.data.get("url", ""):
-                return False, "event is not a git repository"
+        if "git" not in event.tags:
+            return False, "event is not a git repository"
+        elif "github.com" not in event.data.get("url", ""):
+            return False, "event is not a github repository"
         return True
 
     async def handle_event(self, event):

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/nuclei.py

@@ -28,6 +28,7 @@ class nuclei(BaseModule):
         "directory_only": True,
         "retries": 0,
         "batch_size": 200,
+        "module_timeout": 21600,  # 6 hours
     }
     options_desc = {
         "version": "nuclei version",
@@ -38,11 +39,12 @@ class nuclei(BaseModule):
         "concurrency": "maximum number of templates to be executed in parallel (default 25)",
         "mode": "manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests",
         "etags": "tags to exclude from the scan",
-        "budget": "Used in budget mode to set the number of requests which will be allotted to the nuclei scan",
+        "budget": "Used in budget mode to set the number of allowed requests per host",
         "silent": "Don't display nuclei's banner or status messages",
         "directory_only": "Filter out 'file' URL event (default True)",
         "retries": "number of times to retry a failed request (default 0)",
         "batch_size": "Number of targets to send to Nuclei per batch (default 200)",
+        "module_timeout": "Max time in seconds to spend handling each batch of events",
     }
     deps_ansible = [
         {
@@ -57,7 +59,7 @@ class nuclei(BaseModule):
     ]
     deps_pip = ["pyyaml~=6.0"]
     in_scope_only = True
-    _batch_size = 25
+    _batch_size = 200
 
     async def setup(self):
         # attempt to update nuclei templates

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/portscan.py

@@ -30,6 +30,7 @@ class portscan(BaseModule):
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
+        "module_timeout": 259200,  # 3 days
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -42,6 +43,7 @@ class portscan(BaseModule):
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
+        "module_timeout": "Max time in seconds to spend handling each batch of events",
     }
     deps_common = ["masscan"]
     batch_size = 1000000

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/postman_download.py

@@ -14,15 +14,18 @@ class postman_download(postman):
         "author": "@domwhewell-sage",
     }
     options = {"output_folder": "", "api_key": ""}
-    options_desc = {"output_folder": "Folder to download postman workspaces to", "api_key": "Postman API Key"}
+    options_desc = {
+        "output_folder": "Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.",
+        "api_key": "Postman API Key",
+    }
     scope_distance_modifier = 2
 
     async def setup(self):
-        output_folder = self.config.get("output_folder")
+        output_folder = self.config.get("output_folder", "")
         if output_folder:
             self.output_dir = Path(output_folder) / "postman_workspaces"
         else:
-            self.output_dir = self.scan.home / "postman_workspaces"
+            self.output_dir = self.helpers.temp_dir / "postman_workspaces"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/modules/trufflehog.py

@@ -88,8 +88,7 @@ class trufflehog(BaseModule):
 
         if event.type == "CODE_REPOSITORY":
             path = event.data["url"]
-            if "git" in event.tags:
-                module = "github-experimental"
+            module = "github-experimental"
         elif event.type == "FILESYSTEM":
             path = event.data["path"]
             if "git" in event.tags:

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/scanner/preset/args.py

@@ -9,9 +9,18 @@ from bbot.core.helpers.misc import chain_lists, get_closest_match, get_keys_in_d
 log = logging.getLogger("bbot.presets.args")
 
 
+universal_module_options = {
+    "batch_size": "The number of events to process in a single batch (only applies to batch modules)",
+    "module_threads": "How many event handlers to run in parallel",
+    "module_timeout": "Max time in seconds to spend handling each event or batch of events",
+}
+
+
 class BBOTArgs:
     # module config options to exclude from validation
-    exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|module_threads)$")
+    exclude_from_validation = re.compile(
+        r".*modules\.[a-z0-9_]+\.(?:" + "|".join(universal_module_options.keys()) + ")$"
+    )
 
     scan_examples = [
         (

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/scanner/preset/preset.py

@@ -276,8 +276,6 @@ class Preset(metaclass=BasePreset):
         self.exclude_flags.update(set(exclude_flags))
         self.require_flags.update(set(require_flags))
 
-        # log.critical(f"{self.name}: verbose: {self.verbose}, debug: {self.debug}, silent: {self.silent}")
-
     @property
     def bbot_home(self):
         return Path(self.config.get("home", "~/.bbot")).expanduser().resolve()

{bbot-2.4.2.6677rc0 → bbot-2.4.2.6706rc0}/bbot/scanner/scanner.py

@@ -245,8 +245,6 @@ class Scanner:
         self._cleanedup = False
         self._omitted_event_types = None
 
-        self.__loop = None
-        self._manager_worker_loop_tasks = []
         self.init_events_task = None
         self.ticker_task = None
         self.dispatcher_tasks = []
@@ -726,6 +724,7 @@ class Scanner:
                             scan_active_status.append(f"        - {task}:")
                     # scan_active_status.append(f"        incoming_queue_size: {m.num_incoming_events}")
                     # scan_active_status.append(f"        outgoing_queue_size: {m.outgoing_event_queue.qsize()}")
+
                 for line in scan_active_status:
                     self.debug(line)
 
@@ -834,8 +833,6 @@ class Scanner:
             tasks.append(self.ticker_task)
         # dispatcher
         tasks += self.dispatcher_tasks
-        # manager worker loops
-        tasks += self._manager_worker_loop_tasks
         self.helpers.cancel_tasks_sync(tasks)
         # process pool
         self.helpers.process_pool.shutdown(cancel_futures=True)