bbot 2.6.0.6840rc0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl

bbot/modules/graphql_introspection.py

@@ -27,7 +27,6 @@ class graphql_introspection(BaseModule):
             self.output_dir = Path(output_folder) / "graphql-schemas"
         else:
             self.output_dir = self.scan.home / "graphql-schemas"
-        self.helpers.mkdir(self.output_dir)
         return True
 
     async def filter_event(self, event):
@@ -120,7 +119,10 @@ fragment TypeRef on __Type {
             }
             response = await self.helpers.request(**request_args)
             if not response or response.status_code != 200:
-                self.debug(f"Failed to get GraphQL schema for {url} (status code {response.status_code})")
+                self.debug(
+                    f"Failed to get GraphQL schema for {url} "
+                    f"{f'(status code {response.status_code})' if response else ''}"
+                )
                 continue
             try:
                 response_json = response.json()
@@ -128,6 +130,7 @@ fragment TypeRef on __Type {
                 self.debug(f"Failed to parse JSON for {url}")
                 continue
             if response_json.get("data", {}).get("__schema", {}).get("types", []):
+                self.helpers.mkdir(self.output_dir)
                 filename = f"schema-{self.helpers.tagify(url)}.json"
                 filename = self.output_dir / filename
                 with open(filename, "w") as f:

bbot/modules/httpx.py

@@ -50,6 +50,8 @@ class httpx(BaseModule):
     _shuffle_incoming_queue = False
     _batch_size = 500
     _priority = 2
+    # accept Javascript URLs
+    accept_url_special = True
 
     async def setup(self):
         self.threads = self.config.get("threads", 50)

bbot/modules/iis_shortnames.py

@@ -116,13 +116,6 @@ class iis_shortnames(BaseModule):
 
         return duplicates
 
-    async def threaded_request(self, method, url, affirmative_status_code, c):
-        r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10)
-        if r is not None:
-            if r.status_code == affirmative_status_code:
-                return True, c
-        return None, c
-
     async def solve_valid_chars(self, method, target, affirmative_status_code):
         confirmed_chars = []
         confirmed_exts = []

bbot/modules/internal/cloudcheck.py

@@ -1,3 +1,5 @@
+import asyncio
+import regex as re
 from contextlib import suppress
 
 from bbot.modules.base import BaseInterceptModule
@@ -10,107 +12,98 @@ class CloudCheck(BaseInterceptModule):
         "created_date": "2024-07-07",
         "author": "@TheTechromancer",
     }
-    scope_distance_modifier = 1
+    # tag events up to and including distance-2
+    scope_distance_modifier = 2
     _priority = 3
 
     async def setup(self):
-        from cloudcheck import update
-
-        await update()
-        self.dummy_modules = None
+        self._cloud_hostname_regexes = None
+        self._cloud_hostname_regexes_lock = asyncio.Lock()
         return True
 
-    def make_dummy_modules(self):
-        self.dummy_modules = {}
-        for provider_name in self.helpers.cloud.providers.keys():
-            module = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
-            module.default_discovery_context = "{module} derived {event.type}: {event.host}"
-            self.dummy_modules[provider_name] = module
-
     async def filter_event(self, event):
         if (not event.host) or (event.type in ("IP_RANGE",)):
             return False, "event does not have host attribute"
         return True
 
     async def handle_event(self, event, **kwargs):
-        # don't hold up the event loop loading cloud IPs etc.
-        if self.dummy_modules is None:
-            self.make_dummy_modules()
         # cloud tagging by hosts
         hosts_to_check = set(event.resolved_hosts)
         with suppress(KeyError):
             hosts_to_check.remove(event.host_original)
-        hosts_to_check = [event.host_original] + list(hosts_to_check)
+        hosts_to_check = [str(event.host_original)] + list(hosts_to_check)
 
         for i, host in enumerate(hosts_to_check):
             host_is_ip = self.helpers.is_ip(host)
             try:
-                cloudcheck_results = self.helpers.cloudcheck(host)
+                cloudcheck_results = await self.helpers.cloudcheck.lookup(host)
             except Exception as e:
                 self.trace(f"Error running cloudcheck against {event} (host: {host}): {e}")
                 continue
-            for provider, provider_type, subnet in cloudcheck_results:
-                if provider:
-                    event.add_tag(f"{provider_type}-{provider}")
+            for provider in cloudcheck_results:
+                provider_name = provider["name"].lower()
+                tags = provider.get("tags", [])
+                for tag in tags:
+                    event.add_tag(tag)
+                    event.add_tag(f"{tag}-{provider_name}")
                     if host_is_ip:
-                        event.add_tag(f"{provider_type}-ip")
+                        event.add_tag(f"{provider_name}-ip")
                     else:
                         # if the original hostname is a cloud domain, tag it as such
                         if i == 0:
-                            event.add_tag(f"{provider_type}-domain")
+                            event.add_tag(f"{provider_name}-domain")
                         # any children are tagged as CNAMEs
                         else:
-                            event.add_tag(f"{provider_type}-cname")
+                            event.add_tag(f"{provider_name}-cname")
 
-        found = set()
-        str_hosts_to_check = [str(host) for host in hosts_to_check]
-        # look for cloud assets in hosts, http responses
-        # loop through each provider
-        for provider in self.helpers.cloud.providers.values():
-            provider_name = provider.name.lower()
-            base_kwargs = {
-                "parent": event,
-                "tags": [f"{provider.provider_type}-{provider_name}"],
-                "_provider": provider_name,
-            }
-            # loop through the provider's regex signatures, if any
-            for event_type, sigs in provider.signatures.items():
-                if event_type != "STORAGE_BUCKET":
-                    raise ValueError(f'Unknown cloudcheck event type "{event_type}"')
-                base_kwargs["event_type"] = event_type
-                for sig in sigs:
-                    matches = []
-                    # TODO: convert this to an excavate YARA hook
-                    # if event.type == "HTTP_RESPONSE":
-                    #     matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
-                    if event.type.startswith("DNS_NAME"):
-                        for host in str_hosts_to_check:
-                            match = sig.match(host)
-                            if match:
-                                matches.append(match.groups())
-                    for match in matches:
-                        if match not in found:
-                            found.add(match)
+        # we only generate storage buckets off of in-scope or distance-1 events
+        if event.scope_distance >= self.max_scope_distance:
+            return
 
-                            _kwargs = dict(base_kwargs)
-                            event_type_tag = f"cloud-{event_type}"
-                            _kwargs["tags"].append(event_type_tag)
-                            if event.type.startswith("DNS_NAME"):
-                                event.add_tag(event_type_tag)
+        # see if any of our hosts are storage buckets, etc.
+        regexes = await self.cloud_hostname_regexes()
+        regexes = regexes.get("STORAGE_BUCKET_HOSTNAME", [])
+        for regex_name, regex in regexes.items():
+            for host in hosts_to_check:
+                if match := regex.match(host):
+                    try:
+                        bucket_name, bucket_domain = match.groups()
+                    except Exception as e:
+                        self.error(
+                            f"Bucket regex {regex_name} ({regex}) is not formatted correctly to extract bucket name and domain: {e}"
+                        )
+                        continue
+                    bucket_name, bucket_domain = match.groups()
+                    bucket_url = f"https://{bucket_name}.{bucket_domain}"
+                    await self.emit_event(
+                        {
+                            "name": bucket_name,
+                            "url": bucket_url,
+                            "context": f"{{module}} analyzed {event.type} and found {{event.type}}: {bucket_url}",
+                        },
+                        "STORAGE_BUCKET",
+                        parent=event,
+                    )
 
-                            if event_type == "STORAGE_BUCKET":
-                                bucket_name, bucket_domain = match
-                                bucket_url = f"https://{bucket_name}.{bucket_domain}"
-                                _kwargs["data"] = {
-                                    "name": bucket_name,
-                                    "url": bucket_url,
-                                    "context": f"{{module}} analyzed {event.type} and found {{event.type}}: {bucket_url}",
-                                }
-                                await self.emit_event(**_kwargs)
+    async def cloud_hostname_regexes(self):
+        async with self._cloud_hostname_regexes_lock:
+            if not self._cloud_hostname_regexes:
+                storage_bucket_regexes = {}
+                self._cloud_hostname_regexes = {"STORAGE_BUCKET_HOSTNAME": storage_bucket_regexes}
+                from cloudcheck import providers
 
-    async def emit_event(self, *args, **kwargs):
-        provider_name = kwargs.pop("_provider")
-        dummy_module = self.dummy_modules[provider_name]
-        event = dummy_module.make_event(*args, **kwargs)
-        if event:
-            await super().emit_event(event)
+                for attr in dir(providers):
+                    if attr.startswith("_"):
+                        continue
+                    provider = getattr(providers, attr)
+                    provider_regexes = getattr(provider, "regexes", {})
+                    for regex_name, regexes in provider_regexes.items():
+                        for i, regex in enumerate(regexes):
+                            if not regex_name in ("STORAGE_BUCKET_HOSTNAME"):
+                                continue
+                            try:
+                                storage_bucket_regexes[f"{attr}-{regex_name}-{i}"] = re.compile(regex)
+                            except Exception as e:
+                                self.error(f"Error compiling regex for {attr}-{regex_name}: {e}")
+                                continue
+            return self._cloud_hostname_regexes

bbot/modules/internal/unarchive.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from contextlib import suppress
 from bbot.modules.internal.base import BaseInternalModule
 from bbot.core.helpers.libmagic import get_magic_info, get_compression
 
@@ -62,15 +63,20 @@ class unarchive(BaseInternalModule):
                 context=f'extracted "{path}" to: {output_dir}',
             )
         else:
-            output_dir.rmdir()
+            with suppress(OSError):
+                output_dir.rmdir()
 
     async def extract_file(self, path, output_dir):
         extension, mime_type, description, confidence = get_magic_info(path)
         compression_format = get_compression(mime_type)
         cmd_list = self.compression_methods.get(compression_format, [])
         if cmd_list:
-            if not output_dir.exists():
-                self.helpers.mkdir(output_dir)
+            # output dir must not already exist
+            try:
+                output_dir.mkdir(exist_ok=False)
+            except FileExistsError:
+                self.warning(f"Destination directory {output_dir} already exists, aborting unarchive for {path}")
+                return False
             command = [s.format(filename=path, extract_dir=output_dir) for s in cmd_list]
             try:
                 await self.run_process(command, check=True)

bbot/modules/lightfuzz/lightfuzz.py

@@ -11,7 +11,7 @@ class lightfuzz(BaseModule):
 
     options = {
         "force_common_headers": False,
-        "enabled_submodules": ["sqli", "cmdi", "xss", "path", "ssti", "crypto", "serial"],
+        "enabled_submodules": ["sqli", "cmdi", "xss", "path", "ssti", "crypto", "serial", "esi"],
         "disable_post": False,
     }
     options_desc = {
@@ -34,6 +34,7 @@ class lightfuzz(BaseModule):
         self.event_dict = {}
         self.interactsh_subdomain_tags = {}
         self.interactsh_instance = None
+        self.interactsh_domain = None
         self.disable_post = self.config.get("disable_post", False)
         self.enabled_submodules = self.config.get("enabled_submodules")
         self.interactsh_disable = self.scan.config.get("interactsh_disable", False)
@@ -51,13 +52,16 @@ class lightfuzz(BaseModule):
             self.submodules[submodule_name] = submodule_class
 
         interactsh_needed = any(submodule.uses_interactsh for submodule in self.submodules.values())
-
         if interactsh_needed and not self.interactsh_disable:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
+                if not self.interactsh_domain:
+                    self.warning("Interactsh failure: No domain returned from self.interactsh_instance.register()")
+                    self.interactsh_instance = None
             except InteractshError as e:
                 self.warning(f"Interactsh failure: {e}")
+                self.interactsh_instance = None
         return True
 
     async def interactsh_callback(self, r):

bbot/modules/lightfuzz/submodules/esi.py

@@ -0,0 +1,42 @@
+from .base import BaseLightfuzz
+
+
+class esi(BaseLightfuzz):
+    """
+    Detects Edge Side Includes (ESI) processing vulnerabilities.
+
+    Tests if the server processes ESI tags by sending a payload containing ESI tags
+    and checking if the tags are processed (removed) in the response.
+    """
+
+    # Technique lifted from https://github.com/PortSwigger/active-scan-plus-plus
+
+    friendly_name = "Edge Side Includes"
+
+    async def check_probe(self, cookies, probe, match):
+        """
+        Sends the probe and checks if the expected match string is found in the response.
+        """
+        probe_result = await self.standard_probe(self.event.data["type"], cookies, probe)
+        if probe_result and match in probe_result.text:
+            self.results.append(
+                {
+                    "type": "FINDING",
+                    "description": f"Edge Side Include. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]",
+                }
+            )
+            return True
+        return False
+
+    async def fuzz(self):
+        """
+        Main fuzzing method that sends the ESI test payload and checks for processing.
+        """
+        cookies = self.event.data.get("assigned_cookies", {})
+
+        # ESI test payload: if ESI is processed, <!--esi--> will be removed
+        # leaving AABB<!--esx-->CC in the response
+        payload = "AA<!--esi-->BB<!--esx-->CC"
+        detection_string = "AABB<!--esx-->CC"
+
+        await self.check_probe(cookies, payload, detection_string)

bbot/modules/medusa.py

@@ -1,6 +1,5 @@
 import re
 from bbot.modules.base import BaseModule
-from bbot.errors import WordlistError
 
 
 class medusa(BaseModule):
@@ -102,13 +101,11 @@ class medusa(BaseModule):
         },
     ]
 
-    async def setup(self):
-        # Try to cache wordlist
-        try:
-            self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
-        except WordlistError as e:
-            return False, f"Error retrieving wordlist: {e}"
+    async def setup_deps(self):
+        self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
+        return True
 
+    async def setup(self):
         self.password_match_regex = re.compile(r"Password:\s*(\S+)")
         self.success_indicator_match_regex = re.compile(r"\[([^\]]+)\]\s*$")
 

bbot/modules/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.4.2",
+        "version": "3.6.2",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/modules/otx.py

@@ -1,7 +1,7 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
 
 
-class otx(subdomain_enum):
+class otx(subdomain_enum_apikey):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
@@ -9,10 +9,17 @@ class otx(subdomain_enum):
         "description": "Query otx.alienvault.com for subdomains",
         "created_date": "2022-08-24",
         "author": "@TheTechromancer",
+        "auth_required": True,
     }
+    options = {"api_key": ""}
+    options_desc = {"api_key": "OTX API key"}
 
     base_url = "https://otx.alienvault.com"
 
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["X-OTX-API-KEY"] = self.api_key
+        return url, kwargs
+
     def request_url(self, query):
         url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
         return self.api_request(url)

bbot/modules/output/base.py

@@ -38,26 +38,18 @@ class BaseOutputModule(BaseModule):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, (f"Omitting {event} from output because it's marked as httpx-only")
-
         # omit certain event types
         if event._omit:
-            if "target" in event.tags:
-                reason = "it's a target"
-                self.debug(f"Allowing omitted event: {event} because {reason}")
-            elif event.type in self.get_watched_events():
+            if event.type in self.get_watched_events():
                 reason = "its type is explicitly in watched_events"
                 self.debug(f"Allowing omitted event: {event} because {reason}")
             else:
-                return False, "_omit is True"
+                return False, "its type is omitted in the config"
 
         # internal events like those from speculate, ipneighbor
         # or events that are over our report distance
         if event._internal:
-            return False, "_internal is True"
+            return False, "event is internal and output modules don't accept internal events"
 
         return True, reason
 

bbot/modules/paramminer_headers.py

@@ -82,18 +82,21 @@ class paramminer_headers(BaseModule):
 
     header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$")
 
-    async def setup(self):
-        self.recycle_words = self.config.get("recycle_words", True)
-        self.event_dict = {}
-        self.already_checked = set()
+    async def setup_deps(self):
         wordlist = self.config.get("wordlist", "")
         if not wordlist:
             wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}"
+        self.wordlist_file = await self.helpers.wordlist(wordlist)
         self.debug(f"Using wordlist: [{wordlist}]")
+        return True
+
+    async def setup(self):
+        self.recycle_words = self.config.get("recycle_words", True)
+        self.event_dict = {}
+        self.already_checked = set()
+
         self.wl = {
-            h.strip().lower()
-            for h in self.helpers.read_file(await self.helpers.wordlist(wordlist))
-            if len(h) > 0 and "%" not in h
+            h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and "%" not in h
         }
 
         # check against the boring list (if the option is set)

bbot/modules/portfilter.py

@@ -19,6 +19,8 @@ class portfilter(BaseInterceptModule):
     }
 
     _priority = 4
+    # we consume URLs but we don't want to automatically enable httpx
+    _disable_auto_module_deps = True
 
     async def setup(self):
         self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]

bbot/modules/postman_download.py

@@ -7,7 +7,7 @@ from bbot.modules.templates.postman import postman
 class postman_download(postman):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum", "download"]
     meta = {
         "description": "Download workspaces, collections, requests from Postman",
         "created_date": "2024-09-07",