bbot 2.3.0.5538rc0__py3-none-any.whl → 2.3.0.5809rc0__py3-none-any.whl

bbot/modules/templates/postman.py

@@ -14,8 +14,170 @@ class postman(BaseModule):
 
     headers = {
         "Content-Type": "application/json",
-        "X-App-Version": "10.18.8-230926-0808",
+        "X-App-Version": "11.27.4-250109-2338",
         "X-Entity-Team-Id": "0",
         "Origin": "https://www.postman.com",
         "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
     }
+    auth_required = True
+
+    async def setup(self):
+        await super().setup()
+        self.headers = {}
+        api_keys = set()
+        modules_config = self.scan.config.get("modules", {})
+        postman_modules = [m for m in modules_config if str(m).startswith("postman")]
+        for module_name in postman_modules:
+            module_config = modules_config.get(module_name, {})
+            api_key = module_config.get("api_key", "")
+            if isinstance(api_key, str):
+                api_key = [api_key]
+            for key in api_key:
+                key = key.strip()
+                if key:
+                    api_keys.add(key)
+        if not api_keys:
+            if self.auth_required:
+                return None, "No API key set"
+        self.api_key = api_keys
+        if self.api_key:
+            try:
+                await self.ping()
+                self.hugesuccess("API is ready")
+                return True
+            except Exception as e:
+                self.trace()
+                return None, f"Error with API ({str(e).strip()})"
+        return True
+
+    def prepare_api_request(self, url, kwargs):
+        if self.api_key:
+            kwargs["headers"]["X-Api-Key"] = self.api_key
+        return url, kwargs
+
+    async def get_workspace_id(self, repo_url):
+        workspace_id = ""
+        profile = repo_url.split("/")[-2]
+        name = repo_url.split("/")[-1]
+        url = f"{self.base_url}/ws/proxy"
+        json = {
+            "service": "workspaces",
+            "method": "GET",
+            "path": f"/workspaces?handle={profile}&slug={name}",
+        }
+        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
+        if r is None:
+            return workspace_id
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace_id
+        data = json.get("data", [])
+        if len(data) == 1:
+            workspace_id = data[0]["id"]
+        return workspace_id
+
+    async def request_workspace(self, id):
+        data = {"workspace": {}, "environments": [], "collections": []}
+        workspace = await self.get_workspace(id)
+        if workspace:
+            # Main Workspace
+            name = workspace["name"]
+            data["workspace"] = workspace
+
+            # Workspace global variables
+            self.verbose(f"Searching globals for workspace {name}")
+            globals = await self.get_globals(id)
+            data["environments"].append(globals)
+
+            # Workspace Environments
+            workspace_environments = workspace.get("environments", [])
+            if workspace_environments:
+                self.verbose(f"Searching environments for workspace {name}")
+                for _ in workspace_environments:
+                    environment_id = _["uid"]
+                    environment = await self.get_environment(environment_id)
+                    data["environments"].append(environment)
+
+            # Workspace Collections
+            workspace_collections = workspace.get("collections", [])
+            if workspace_collections:
+                self.verbose(f"Searching collections for workspace {name}")
+                for _ in workspace_collections:
+                    collection_id = _["uid"]
+                    collection = await self.get_collection(collection_id)
+                    data["collections"].append(collection)
+        return data
+
+    async def get_workspace(self, workspace_id):
+        workspace = {}
+        workspace_url = f"{self.api_url}/workspaces/{workspace_id}"
+        r = await self.api_request(workspace_url)
+        if r is None:
+            return workspace
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace
+        workspace = json.get("workspace", {})
+        return workspace
+
+    async def get_globals(self, workspace_id):
+        globals = {}
+        globals_url = f"{self.base_url}/workspace/{workspace_id}/globals"
+        r = await self.helpers.request(globals_url, headers=self.headers)
+        if r is None:
+            return globals
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return globals
+        globals = json.get("data", {})
+        return globals
+
+    async def get_environment(self, environment_id):
+        environment = {}
+        environment_url = f"{self.api_url}/environments/{environment_id}"
+        r = await self.api_request(environment_url)
+        if r is None:
+            return environment
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return environment
+        environment = json.get("environment", {})
+        return environment
+
+    async def get_collection(self, collection_id):
+        collection = {}
+        collection_url = f"{self.api_url}/collections/{collection_id}"
+        r = await self.api_request(collection_url)
+        if r is None:
+            return collection
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return collection
+        collection = json.get("collection", {})
+        return collection
+
+    async def validate_workspace(self, workspace, environments, collections):
+        name = workspace.get("name", "")
+        full_wks = str([workspace, environments, collections])
+        in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks)
+        if in_scope_hosts:
+            self.verbose(
+                f'Found in-scope hostname(s): "{in_scope_hosts}" in workspace {name}, it appears to be in-scope'
+            )
+            return True
+        return False

bbot/modules/templates/subdomain_enum.py

@@ -150,7 +150,7 @@ class subdomain_enum(BaseModule):
                     break
                 yield subdomains
         finally:
-            agen.aclose()
+            await agen.aclose()
 
     async def _is_wildcard(self, query):
         rdtypes = ("A", "AAAA", "CNAME")

bbot/modules/templates/webhook.py

@@ -13,41 +13,32 @@ class WebhookOutputModule(BaseOutputModule):
     content_key = "content"
     vuln_severities = ["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CRITICAL"]
 
+    # abort module after 10 failed requests (not including retries)
+    _api_failure_abort_threshold = 10
+    # retry each request up to 10 times, respecting the Retry-After header
+    _api_retries = 10
+
     async def setup(self):
+        self._api_retries = self.config.get("retries", 10)
         self.webhook_url = self.config.get("webhook_url", "")
         self.min_severity = self.config.get("min_severity", "LOW").strip().upper()
-        assert (
-            self.min_severity in self.vuln_severities
-        ), f"min_severity must be one of the following: {','.join(self.vuln_severities)}"
+        assert self.min_severity in self.vuln_severities, (
+            f"min_severity must be one of the following: {','.join(self.vuln_severities)}"
+        )
         self.allowed_severities = self.vuln_severities[self.vuln_severities.index(self.min_severity) :]
         if not self.webhook_url:
             self.warning("Must set Webhook URL")
             return False
-        return True
+        return await super().setup()
 
     async def handle_event(self, event):
-        while 1:
-            message = self.format_message(event)
-            data = {self.content_key: message}
-
-            response = await self.helpers.request(
-                url=self.webhook_url,
-                method="POST",
-                json=data,
-            )
-            status_code = getattr(response, "status_code", 0)
-            if self.evaluate_response(response):
-                break
-            else:
-                response_data = getattr(response, "text", "")
-                try:
-                    retry_after = response.json().get("retry_after", 1)
-                except Exception:
-                    retry_after = 1
-                self.verbose(
-                    f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds"
-                )
-                await self.helpers.sleep(retry_after)
+        message = self.format_message(event)
+        data = {self.content_key: message}
+        await self.api_request(
+            url=self.webhook_url,
+            method="POST",
+            json=data,
+        )
 
     def get_watched_events(self):
         if self._watched_events is None:

bbot/modules/trufflehog.py

@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.88.0",
+        "version": "3.88.2",
         "config": "",
         "only_verified": True,
         "concurrency": 8,
@@ -76,8 +76,8 @@ class trufflehog(BaseModule):
             else:
                 return False, "Deleted forks is not enabled"
         else:
-            if "parsed-folder" in event.tags:
-                return False, "Not accepting parsed-folder events"
+            if "unarchived-folder" in event.tags:
+                return False, "Not accepting unarchived-folder events"
         return True
 
     async def handle_event(self, event):

bbot/modules/wappalyzer.py

@@ -19,7 +19,7 @@ class wappalyzer(BaseModule):
         "created_date": "2022-04-15",
         "author": "@liquidsec",
     }
-    deps_pip = ["python-Wappalyzer~=0.3.1", "aiohttp~=3.9.0b0"]
+    deps_pip = ["python-Wappalyzer~=0.3.1", "aiohttp~=3.9.0b0", "setuptools"]
     # accept all events regardless of scope distance
     scope_distance_modifier = None
     _module_threads = 5

bbot/modules/zoomeye.py

@@ -67,7 +67,7 @@ class zoomeye(subdomain_enum_apikey):
                     break
                 i += 1
         finally:
-            agen.aclose()
+            await agen.aclose()
         return results
 
     async def parse_results(self, r):

bbot/presets/kitchen-sink.yml

@@ -10,7 +10,7 @@ include:
   - paramminer
   - dirbust-light
   - web-screenshots
-  - baddns-thorough
+  - baddns-intense
 
 config:
   modules:

bbot/presets/nuclei/nuclei-budget.yml

@@ -0,0 +1,19 @@
+description: Run nuclei scans against all discovered targets, using budget mode to look for low hanging fruit with greatly reduced number of requests
+
+modules:
+  - httpx
+  - nuclei
+  - portfilter
+
+config:
+  modules:
+    nuclei:
+      mode: budget
+      budget: 10
+      directory_only: true # Do not run nuclei on individual non-directory URLs
+
+conditions:
+  - |
+    {% if config.web.spider_distance != 0 %}
+      {{ warn("Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.") }}
+    {% endif %}

bbot/presets/nuclei/nuclei-intense.yml

@@ -0,0 +1,28 @@
+description: Run nuclei scans against all discovered targets, allowing for spidering, against ALL URLs, and with additional discovery modules.
+
+modules:
+  - httpx
+  - nuclei
+  - robots
+  - urlscan
+  - portfilter
+  - wayback
+
+config:
+  modules:
+    nuclei:
+      directory_only: False # Will run nuclei on ALL discovered URLs - Be careful!
+    wayback:
+      urls: true
+
+conditions:
+  - |
+    {% if config.web.spider_distance == 0 and config.modules.nuclei.directory_only == False %}
+      {{ warn("The 'nuclei-intense' preset turns the 'directory_only' limitation off on the nuclei module. To make the best use of this, you may want to enable spidering with 'spider' or 'spider-intense' preset.") }}
+    {% endif %}
+
+
+# Example for also running a dirbust
+
+#include:
+#  - dirbust-light

bbot/presets/nuclei/nuclei-technology.yml

@@ -0,0 +1,23 @@
+description: Run nuclei scans against all discovered targets, running templates which match discovered technologies
+
+modules:
+  - httpx
+  - nuclei
+  - portfilter
+
+config:
+  modules:
+    nuclei:
+      mode: technology
+      directory_only: True # Do not run nuclei on individual non-directory URLs. This is less unsafe to disable with technology mode.
+
+conditions:
+  - |
+    {% if config.web.spider_distance != 0 %}
+      {{ warn("Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.") }}
+    {% endif %}
+
+# Example for also running a dirbust
+
+#include:
+#  - dirbust-light

bbot/presets/nuclei/nuclei.yml

@@ -0,0 +1,34 @@
+description: Run nuclei scans against all discovered targets
+
+modules:
+  - httpx
+  - nuclei
+  - portfilter
+
+config:
+  modules:
+    nuclei:
+      directory_only: True # Do not run nuclei on individual non-directory URLs
+
+
+conditions:
+  - |
+    {% if config.web.spider_distance != 0 %}
+      {{ warn("Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.") }}
+    {% endif %}
+
+
+
+# Additional Examples:
+
+# Slowing Down Scan
+
+#config:
+#  modules:
+#    nuclei:
+#      ratelimit: 10
+#      concurrency: 5
+
+
+
+

bbot/presets/spider-intense.yml

@@ -0,0 +1,13 @@
+description: Recursive web spider with more aggressive settings
+
+include:
+  - spider
+  
+config:
+  web:
+    # how many links to follow in a row
+    spider_distance: 4
+    # don't follow links whose directory depth is higher than 6
+    spider_depth: 6
+    # maximum number of links to follow per page
+    spider_links_per_page: 50

bbot/scanner/preset/args.py

@@ -133,6 +133,8 @@ class BBOTArgs:
             )
         if self.parsed.event_types:
             args_preset.core.merge_custom({"modules": {"stdout": {"event_types": self.parsed.event_types}}})
+        if self.parsed.exclude_cdn:
+            args_preset.explicit_scan_modules.add("portfilter")
 
         # dependencies
         deps_config = args_preset.core.custom_config.get("deps", {})
@@ -166,6 +168,21 @@ class BBOTArgs:
                 {"modules": {"excavate": {"custom_yara_rules": self.parsed.custom_yara_rules}}}
             )
 
+        # Check if both user_agent and user_agent_suffix are set. If so combine them and merge into the config
+        if self.parsed.user_agent and self.parsed.user_agent_suffix:
+            modified_user_agent = f"{self.parsed.user_agent} {self.parsed.user_agent_suffix}"
+            args_preset.core.merge_custom({"web": {"user_agent": modified_user_agent}})
+
+        # If only user_agent_suffix is set, retrieve the existing user_agent from the merged config and append the suffix
+        elif self.parsed.user_agent_suffix:
+            existing_user_agent = args_preset.core.config.get("web", {}).get("user_agent", "")
+            modified_user_agent = f"{existing_user_agent} {self.parsed.user_agent_suffix}"
+            args_preset.core.merge_custom({"web": {"user_agent": modified_user_agent}})
+
+        # If only user_agent is set, merge it directly
+        elif self.parsed.user_agent:
+            args_preset.core.merge_custom({"web": {"user_agent": self.parsed.user_agent}})
+
         # CLI config options (dot-syntax)
         for config_arg in self.parsed.config:
             try:
@@ -232,7 +249,7 @@ class BBOTArgs:
             "--modules",
             nargs="+",
             default=[],
-            help=f'Modules to enable. Choices: {",".join(sorted(self.preset.module_loader.scan_module_choices))}',
+            help=f"Modules to enable. Choices: {','.join(sorted(self.preset.module_loader.scan_module_choices))}",
             metavar="MODULE",
         )
         modules.add_argument("-l", "--list-modules", action="store_true", help="List available modules.")
@@ -247,7 +264,7 @@ class BBOTArgs:
             "--flags",
             nargs="+",
             default=[],
-            help=f'Enable modules by flag. Choices: {",".join(sorted(self.preset.module_loader.flag_choices))}',
+            help=f"Enable modules by flag. Choices: {','.join(sorted(self.preset.module_loader.flag_choices))}",
             metavar="FLAG",
         )
         modules.add_argument("-lf", "--list-flags", action="store_true", help="List available flags.")
@@ -309,13 +326,19 @@ class BBOTArgs:
             "--output-modules",
             nargs="+",
             default=[],
-            help=f'Output module(s). Choices: {",".join(sorted(self.preset.module_loader.output_module_choices))}',
+            help=f"Output module(s). Choices: {','.join(sorted(self.preset.module_loader.output_module_choices))}",
             metavar="MODULE",
         )
         output.add_argument("-lo", "--list-output-modules", action="store_true", help="List available output modules")
         output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format")
         output.add_argument("--brief", "-br", action="store_true", help="Output only the data itself")
         output.add_argument("--event-types", nargs="+", default=[], help="Choose which event types to display")
+        output.add_argument(
+            "--exclude-cdn",
+            "-ec",
+            action="store_true",
+            help="Filter out unwanted open ports on CDNs/WAFs (80,443 only)",
+        )
 
         deps = p.add_argument_group(
             title="Module dependencies", description="Control how modules install their dependencies"
@@ -340,6 +363,9 @@ class BBOTArgs:
             help="List of custom headers as key value pairs (header=value).",
         )
         misc.add_argument("--custom-yara-rules", "-cy", help="Add custom yara rules to excavate")
+
+        misc.add_argument("--user-agent", "-ua", help="Set the user-agent for all HTTP requests")
+        misc.add_argument("--user-agent-suffix", "-uas", help=argparse.SUPPRESS, metavar="SUFFIX", default=None)
         return p
 
     def sanitize_args(self):

bbot/scanner/preset/preset.py

@@ -24,7 +24,39 @@ _preset_cache = {}
 DEFAULT_PRESETS = None
 
 
-class Preset:
+class BasePreset(type):
+    def __call__(cls, *args, include=None, presets=None, name=None, description=None, _exclude=None, **kwargs):
+        """
+        Handles loading of "included" presets, while preserving the proper load order
+
+        Overriding __call__() allows us to reuse the logic from .merge() without duplicating functionality in __init__().
+        """
+        include_preset = None
+
+        # "presets" is alias to "include"
+        if presets and include:
+            raise ValueError(
+                'Cannot use both "presets" and "include" args at the same time (presets is an alias to include). Please pick one or the other :)'
+            )
+        if presets and not include:
+            include = presets
+        # include other presets
+        if include and not isinstance(include, (list, tuple, set)):
+            include = [include]
+
+        main_preset = type.__call__(cls, *args, name=name, description=description, _exclude=_exclude, **kwargs)
+
+        if include:
+            include_preset = type.__call__(cls, name=name, description=description, _exclude=_exclude)
+            for included_preset in include:
+                include_preset.include_preset(included_preset)
+            include_preset.merge(main_preset)
+            return include_preset
+
+        return main_preset
+
+
+class Preset(metaclass=BasePreset):
     """
     A preset is the central config for a BBOT scan. It contains everything a scan needs to run --
         targets, modules, flags, config options like API keys, etc.
@@ -94,12 +126,10 @@ class Preset:
         exclude_flags=None,
         config=None,
         module_dirs=None,
-        include=None,
-        presets=None,
         output_dir=None,
-        scan_name=None,
         name=None,
         description=None,
+        scan_name=None,
         conditions=None,
         force_start=False,
         verbose=False,
@@ -238,20 +268,6 @@ class Preset:
 
         self._target = None
 
-        # "presets" is alias to "include"
-        if presets and include:
-            raise ValueError(
-                'Cannot use both "presets" and "include" args at the same time (presets is an alias to include). Please pick one or the other :)'
-            )
-        if presets and not include:
-            include = presets
-        # include other presets
-        if include and not isinstance(include, (list, tuple, set)):
-            include = [include]
-        if include:
-            for included_preset in include:
-                self.include_preset(included_preset)
-
         # we don't fill self.modules yet (that happens in .bake())
         self.explicit_scan_modules.update(set(modules))
         self.explicit_output_modules.update(set(output_modules))
@@ -260,6 +276,8 @@ class Preset:
         self.exclude_flags.update(set(exclude_flags))
         self.require_flags.update(set(require_flags))
 
+        # log.critical(f"{self.name}: verbose: {self.verbose}, debug: {self.debug}, silent: {self.silent}")
+
     @property
     def bbot_home(self):
         return Path(self.config.get("home", "~/.bbot")).expanduser().resolve()
@@ -332,6 +350,7 @@ class Preset:
             ['portscan', 'sslcert']
         """
         self.log_debug(f'Merging preset "{other.name}" into "{self.name}"')
+
         # config
         self.core.merge_custom(other.core.custom_config)
         self.module_loader.core = self.core
@@ -671,11 +690,10 @@ class Preset:
             >>> preset.include_preset("/home/user/my_preset.yml")
         """
         self.log_debug(f'Including preset "{filename}"')
-        preset_filename = PRESET_PATH.find(filename)
-        preset_from_yaml = self.from_yaml_file(preset_filename, _exclude=self._preset_files_loaded)
+        preset_from_yaml = self.from_yaml_file(filename, _exclude=self._preset_files_loaded)
         if preset_from_yaml is not False:
             self.merge(preset_from_yaml)
-            self._preset_files_loaded.add(preset_filename)
+            self._preset_files_loaded.add(preset_from_yaml.filename)
 
     @classmethod
     def from_yaml_file(cls, filename, _exclude=None, _log=False):
@@ -687,7 +705,7 @@ class Preset:
         Examples:
             >>> preset = Preset.from_yaml_file("/home/user/my_preset.yml")
         """
-        filename = Path(filename).resolve()
+        filename = PRESET_PATH.find(filename)
         try:
             return _preset_cache[filename]
         except KeyError:
@@ -707,13 +725,14 @@ class Preset:
                 omegaconf.OmegaConf.create(yaml_str), name=filename.stem, _exclude=_exclude, _log=_log
             )
             preset._yaml_str = yaml_str
+            preset.filename = filename
             _preset_cache[filename] = preset
             return preset
 
     @classmethod
     def from_yaml_string(cls, yaml_preset):
         """
-        Create a preset from a YAML file. If the full path is not specified, BBOT will look in all the usual places for it.
+        Create a preset from a YAML string.
 
         The file extension is optional.
 
@@ -870,7 +889,7 @@ class Preset:
                     if f in self.exclude_flags:
                         return False, f'it has excluded flag, "{f}"', preloaded
             if self.require_flags and not all(f in module_flags for f in self.require_flags):
-                return False, f'it doesn\'t have the required flags ({",".join(self.require_flags)})', preloaded
+                return False, f"it doesn't have the required flags ({','.join(self.require_flags)})", preloaded
 
         return True, "", preloaded