bbot 2.4.2__py3-none-any.whl → 2.4.2.6590rc0__py3-none-any.whl

bbot/core/helpers/web/envelopes.py

@@ -0,0 +1,352 @@
+import json
+import base64
+import binascii
+import xmltodict
+from contextlib import suppress
+from urllib.parse import unquote, quote
+from xml.parsers.expat import ExpatError
+
+from bbot.core.helpers.misc import is_printable
+
+
+# TODO: This logic is perfect for extracting params. We should expand it outwards to include other higher-level envelopes:
+#    - QueryStringEnvelope
+#    - MultipartFormEnvelope
+#    - HeaderEnvelope
+#    - CookieEnvelope
+#
+# Once we start ingesting HTTP_REQUEST events, this will make them instantly fuzzable
+
+
+class EnvelopeChildTracker(type):
+    """
+    Keeps track of all the child envelope classes
+    """
+
+    children = []
+
+    def __new__(mcs, name, bases, class_dict):
+        # Create the class
+        cls = super().__new__(mcs, name, bases, class_dict)
+        # Don't register the base class itself
+        if bases and not name.startswith("Base"):  # Only register if it has base classes (i.e., is a child)
+            EnvelopeChildTracker.children.append(cls)
+            EnvelopeChildTracker.children.sort(key=lambda x: x.priority)
+        return cls
+
+
+class BaseEnvelope(metaclass=EnvelopeChildTracker):
+    __slots__ = ["subparams", "selected_subparam", "singleton"]
+
+    # determines the order of the envelope detection
+    priority = 5
+    # whether the envelope is the final format, e.g. raw text/binary
+    end_format = False
+    ignore_exceptions = (Exception,)
+    envelope_classes = EnvelopeChildTracker.children
+    # transparent envelopes (i.e. TextEnvelope) are not counted as envelopes or included in the finding descriptions
+    transparent = False
+
+    def __init__(self, s):
+        unpacked_data = self.unpack(s)
+
+        if self.end_format:
+            inner_envelope = unpacked_data
+        else:
+            inner_envelope = self.detect(unpacked_data)
+
+        self.selected_subparam = None
+        # if we have subparams, our inner envelope will be a dictionary
+        if isinstance(inner_envelope, dict):
+            self.subparams = inner_envelope
+            self.singleton = False
+        # otherwise if we just have one value, we make a dictionary with a default key
+        else:
+            self.subparams = {"__default__": inner_envelope}
+            self.singleton = True
+
+    @property
+    def final_envelope(self):
+        try:
+            return self.unpacked_data(recursive=False).final_envelope
+        except AttributeError:
+            return self
+
+    @property
+    def friendly_name(self):
+        if self.friendly_name:
+            return self.friendly_name
+        else:
+            return self.name
+
+    def pack(self, data=None):
+        if data is None:
+            data = self.unpacked_data(recursive=False)
+            with suppress(AttributeError):
+                data = data.pack()
+        return self._pack(data)
+
+    def unpack(self, s):
+        return self._unpack(s)
+
+    def _pack(self, s):
+        """
+        Encodes the string using the class's unique encoder (adds the outer envelope)
+        """
+        raise NotImplementedError("Envelope.pack() must be implemented")
+
+    def _unpack(self, s):
+        """
+        Decodes the string using the class's unique encoder (removes the outer envelope)
+        """
+        raise NotImplementedError("Envelope.unpack() must be implemented")
+
+    def unpacked_data(self, recursive=True):
+        try:
+            unpacked = self.subparams["__default__"]
+            if recursive:
+                with suppress(AttributeError):
+                    return unpacked.unpacked_data(recursive=recursive)
+            return unpacked
+        except KeyError:
+            return self.subparams
+
+    @classmethod
+    def detect(cls, s):
+        """
+        Detects the type of envelope used to encode the packed_data
+        """
+        if not isinstance(s, str):
+            raise ValueError(f"Invalid data passed to detect(): {s} ({type(s)})")
+        # if the value is empty, we just return the text envelope
+        if not s.strip():
+            return TextEnvelope(s)
+        for envelope_class in cls.envelope_classes:
+            with suppress(*envelope_class.ignore_exceptions):
+                envelope = envelope_class(s)
+                if envelope is not False:
+                    # make sure the envelope is not just the original string, to prevent unnecessary envelope detection. For example, "10" is technically valid JSON, but nothing is being encapsulated
+                    if str(envelope.unpacked_data()) == s:
+                        return TextEnvelope(s)
+                    else:
+                        return envelope
+                del envelope
+        raise Exception(f"No envelope detected for data: '{s}' ({type(s)})")
+
+    def get_subparams(self, key=None, data=None, recursive=True):
+        if data is None:
+            data = self.unpacked_data(recursive=recursive)
+        if key is None:
+            key = []
+
+        if isinstance(data, dict):
+            for k, v in data.items():
+                full_key = key + [k]
+                if isinstance(v, dict):
+                    yield from self.get_subparams(full_key, v)
+                else:
+                    yield full_key, v
+        else:
+            yield [], data
+
+    def get_subparam(self, key=None, recursive=True):
+        if key is None:
+            key = self.selected_subparam
+        envelope = self
+        if recursive:
+            envelope = self.final_envelope
+        data = envelope.unpacked_data(recursive=False)
+        if key is None:
+            if envelope.singleton:
+                key = []
+            else:
+                raise ValueError("No subparam selected")
+        else:
+            for segment in key:
+                data = data[segment]
+        return data
+
+    def set_subparam(self, key=None, value=None, recursive=True):
+        envelope = self
+        if recursive:
+            envelope = self.final_envelope
+
+        # if there's only one value to set, we can just set it directly
+        if envelope.singleton:
+            envelope.subparams["__default__"] = value
+            return
+
+        # if key isn't specified, use the selected subparam
+        if key is None:
+            key = self.selected_subparam
+        if key is None:
+            raise ValueError(f"{self} -> {envelope}: No subparam selected")
+
+        data = envelope.unpacked_data(recursive=False)
+        for segment in key[:-1]:
+            data = data[segment]
+        data[key[-1]] = value
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    @property
+    def num_envelopes(self):
+        num_envelopes = 0 if self.transparent else 1
+        if self.end_format:
+            return num_envelopes
+        for envelope in self.subparams.values():
+            with suppress(AttributeError):
+                num_envelopes += envelope.num_envelopes
+        return num_envelopes
+
+    @property
+    def summary(self):
+        if self.transparent:
+            return ""
+        self_string = f"{self.friendly_name}"
+        with suppress(AttributeError):
+            child_envelope = self.unpacked_data(recursive=False)
+            child_summary = child_envelope.summary
+            if child_summary:
+                self_string += f" -> {child_summary}"
+
+        if self.selected_subparam:
+            self_string += f" [{'.'.join(self.selected_subparam)}]"
+        return self_string
+
+    def to_dict(self):
+        return self.summary
+
+    def __str__(self):
+        return self.summary
+
+    __repr__ = __str__
+
+
+class HexEnvelope(BaseEnvelope):
+    """
+    Hexadecimal encoding
+    """
+
+    friendly_name = "Hexadecimal-Encoded"
+
+    ignore_exceptions = (ValueError, UnicodeDecodeError)
+
+    def _pack(self, s):
+        return s.encode().hex()
+
+    def _unpack(self, s):
+        return bytes.fromhex(s).decode()
+
+
+class B64Envelope(BaseEnvelope):
+    """
+    Base64 encoding
+    """
+
+    friendly_name = "Base64-Encoded"
+
+    ignore_exceptions = (binascii.Error, UnicodeDecodeError, ValueError)
+
+    def unpack(self, s):
+        # it's easy to have a small value that accidentally decodes to base64
+        if len(s) < 8 and not s.endswith("="):
+            raise ValueError("Data is too small to be sure")
+        return super().unpack(s)
+
+    def _pack(self, s):
+        return base64.b64encode(s.encode()).decode()
+
+    def _unpack(self, s):
+        return base64.b64decode(s).decode()
+
+
+class URLEnvelope(BaseEnvelope):
+    """
+    URL encoding
+    """
+
+    friendly_name = "URL-Encoded"
+
+    def unpack(self, s):
+        unpacked = super().unpack(s)
+        if unpacked == s:
+            raise ValueError("Data is not URL-encoded")
+        return unpacked
+
+    def _pack(self, s):
+        return quote(s)
+
+    def _unpack(self, s):
+        return unquote(s)
+
+
+class TextEnvelope(BaseEnvelope):
+    """
+    Text encoding
+    """
+
+    end_format = True
+    # lowest priority means text is the ultimate fallback
+    priority = 10
+    transparent = True
+    ignore_exceptions = ()
+
+    def _pack(self, s):
+        return s
+
+    def _unpack(self, s):
+        if not is_printable(s):
+            raise ValueError(f"Non-printable data detected in TextEnvelope: '{s}' ({type(s)})")
+        return s
+
+
+# class BinaryEnvelope(BaseEnvelope):
+#     """
+#     Binary encoding
+#     """
+#     end_format = True
+
+#     def pack(self, s):
+#         return s
+
+#     def unpack(self, s):
+#         if is_printable(s):
+#             raise Exception("Non-binary data detected in BinaryEnvelope")
+#         return s
+
+
+class JSONEnvelope(BaseEnvelope):
+    """
+    JSON encoding
+    """
+
+    friendly_name = "JSON-formatted"
+    end_format = True
+    priority = 8
+    ignore_exceptions = (json.JSONDecodeError,)
+
+    def _pack(self, s):
+        return json.dumps(s)
+
+    def _unpack(self, s):
+        return json.loads(s)
+
+
+class XMLEnvelope(BaseEnvelope):
+    """
+    XML encoding
+    """
+
+    friendly_name = "XML-formatted"
+    end_format = True
+    priority = 9
+    ignore_exceptions = (ExpatError,)
+
+    def _pack(self, s):
+        return xmltodict.unparse(s)
+
+    def _unpack(self, s):
+        return xmltodict.parse(s)

bbot/core/helpers/web/web.py

@@ -349,6 +349,7 @@ class WebHelper(EngineClient):
             curl_command.append("-k")
 
         headers = kwargs.get("headers", {})
+        cookies = kwargs.get("cookies", {})
 
         ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False)
 
@@ -362,10 +363,17 @@ class WebHelper(EngineClient):
             if "User-Agent" not in headers:
                 headers["User-Agent"] = user_agent
 
-            # only add custom headers if the URL is in-scope
+            # only add custom headers / cookies if the URL is in-scope
             if self.parent_helper.preset.in_scope(url):
                 for hk, hv in self.web_config.get("http_headers", {}).items():
-                    headers[hk] = hv
+                    # Only add the header if it doesn't already exist in the headers dictionary
+                    if hk not in headers:
+                        headers[hk] = hv
+
+                for ck, cv in self.web_config.get("http_cookies", {}).items():
+                    # don't clobber cookies
+                    if ck not in cookies:
+                        cookies[ck] = cv
 
         # add the timeout
         if "timeout" not in kwargs:

bbot/core/helpers/yara_helper.py

@@ -0,0 +1,50 @@
+import yara
+
+
+class YaraHelper:
+    def __init__(self, parent_helper):
+        self.parent_helper = parent_helper
+
+    def compile_strings(self, strings: list[str], nocase=False):
+        """
+        Compile a list of strings into a YARA rule
+        """
+        # Format each string as a YARA string definition
+        yara_strings = []
+        for i, s in enumerate(strings):
+            s = s.replace('"', '\\"')
+            yara_string = f'$s{i} = "{s}"'
+            if nocase:
+                yara_string += " nocase"
+            yara_strings.append(yara_string)
+        yara_strings = "\n        ".join(yara_strings)
+
+        # Create the complete YARA rule
+        yara_rule = f"""
+rule strings_match
+{{
+    strings:
+        {yara_strings}
+    condition:
+        any of them
+}}
+"""
+        # Compile and return the rule
+        return self.compile(source=yara_rule)
+
+    def compile(self, *args, **kwargs):
+        return yara.compile(*args, **kwargs)
+
+    async def match(self, compiled_rules, text):
+        """
+        Given a compiled YARA rule and a body of text, return a list of strings that match the rule
+        """
+        matched_strings = []
+        matches = await self.parent_helper.run_in_executor(compiled_rules.match, data=text)
+        if matches:
+            for match in matches:
+                for string_match in match.strings:
+                    for instance in string_match.instances:
+                        matched_string = instance.matched_data.decode("utf-8")
+                        matched_strings.append(matched_string)
+        return matched_strings

bbot/core/modules.py

@@ -104,8 +104,9 @@ class ModuleLoader:
 
     def file_filter(self, file):
         file = file.resolve()
-        if "templates" in file.parts:
-            return False
+        for part in file.parts:
+            if part.endswith("_submodules") or part == "templates":
+                return False
         return file.suffix.lower() == ".py" and file.stem not in ["base", "__init__"]
 
     def preload(self, module_dirs=None):
@@ -158,12 +159,11 @@ class ModuleLoader:
                         namespace = f"bbot.modules.{module_dir.name}"
                     try:
                         preloaded = self.preload_module(module_file)
+                        if preloaded is None:
+                            continue
                         module_type = "scan"
                         if module_dir.name in ("output", "internal"):
                             module_type = str(module_dir.name)
-                        elif module_dir.name not in ("modules"):
-                            flags = set(preloaded["flags"] + [module_dir.name])
-                            preloaded["flags"] = sorted(flags)
 
                         # derive module dependencies from watched event types (only for scan modules)
                         if module_type == "scan":
@@ -327,12 +327,28 @@ class ModuleLoader:
         deps_apt = []
         deps_common = []
         ansible_tasks = []
+        config = {}
+        options_desc = {}
         python_code = open(module_file).read()
         # take a hash of the code so we can keep track of when it changes
         module_hash = sha1(python_code).hexdigest()
         parsed_code = ast.parse(python_code)
-        config = {}
-        options_desc = {}
+
+        # discard if the module isn't a valid BBOT module
+        is_bbot_module = False
+        for root_element in parsed_code.body:
+            if type(root_element) == ast.ClassDef:
+                for class_attr in root_element.body:
+                    if type(class_attr) == ast.Assign and any(
+                        target.id in ("watched_events", "produced_events") for target in class_attr.targets
+                    ):
+                        is_bbot_module = True
+                        break
+
+        if not is_bbot_module:
+            log.debug(f"Skipping {module_file} as it is not a valid BBOT module")
+            return
+
         for root_element in parsed_code.body:
             # look for classes
             if type(root_element) == ast.ClassDef:

bbot/defaults.yml

@@ -227,7 +227,32 @@ url_extension_static:
   - bz2
   - 7z
   - rar
-  
+
+parameter_blacklist:
+  - __VIEWSTATE
+  - __EVENTARGUMENT
+  - __EVENTVALIDATION
+  - __EVENTTARGET
+  - __EVENTARGUMENT
+  - __VIEWSTATEGENERATOR
+  - __SCROLLPOSITIONY
+  - __SCROLLPOSITIONX
+  - ASP.NET_SessionId
+  - PHPSESSID
+  - __cf_bm
+  - f5_cspm
+
+parameter_blacklist_prefixes:
+  - TS01
+  - BIGipServer
+  - incap_
+  - visid_incap_
+  - AWSALB
+  - utm_
+  - ApplicationGatewayAffinity
+  - JSESSIONID
+  - ARRAffinity
+
 # Don't output these types of events (they are still distributed to modules)
 omit_event_types:
   - HTTP_RESPONSE

bbot/modules/base.py

@@ -535,8 +535,10 @@ class BaseModule:
             if v is not None:
                 emit_kwargs[o] = v
         event = self.make_event(*args, **event_kwargs)
-        if event:
-            await self.queue_outgoing_event(event, **emit_kwargs)
+        if event is not None:
+            children = event.children
+            for e in [event] + children:
+                await self.queue_outgoing_event(e, **emit_kwargs)
         return event
 
     async def _events_waiting(self, batch_size=None):

bbot/modules/{deadly/dastardly.py → dastardly.py}

@@ -5,7 +5,7 @@ from bbot.modules.base import BaseModule
 class dastardly(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING", "VULNERABILITY"]
-    flags = ["active", "aggressive", "slow", "web-thorough"]
+    flags = ["active", "aggressive", "slow", "web-thorough", "deadly"]
     meta = {
         "description": "Lightweight web application security scanner",
         "created_date": "2023-12-11",

bbot/modules/{deadly/ffuf.py → ffuf.py}

@@ -9,7 +9,7 @@ import base64
 class ffuf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
-    flags = ["aggressive", "active"]
+    flags = ["aggressive", "active", "deadly"]
     meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@liquidsec"}
 
     options = {

bbot/modules/ffuf_shortnames.py

@@ -3,7 +3,7 @@ import re
 import random
 import string
 
-from bbot.modules.deadly.ffuf import ffuf
+from bbot.modules.ffuf import ffuf
 
 
 class ffuf_shortnames(ffuf):

bbot/modules/httpx.py

@@ -3,6 +3,8 @@ import orjson
 import tempfile
 import subprocess
 from pathlib import Path
+from http.cookies import SimpleCookie
+
 from bbot.modules.base import BaseModule
 
 
@@ -137,8 +139,20 @@ class httpx(BaseModule):
         if self.probe_all_ips:
             command += ["-probe-all-ips"]
 
+        # Add custom HTTP headers
         for hk, hv in self.scan.custom_http_headers.items():
             command += ["-header", f"{hk}: {hv}"]
+
+        # Add custom HTTP cookies as a single header
+        if self.scan.custom_http_cookies:
+            cookie = SimpleCookie()
+            for ck, cv in self.scan.custom_http_cookies.items():
+                cookie[ck] = cv
+
+            # Build the cookie header
+            cookie_header = f"Cookie: {cookie.output(header='', sep='; ').strip()}"
+            command += ["-header", cookie_header]
+
         proxy = self.scan.http_proxy
         if proxy:
             command += ["-http-proxy", proxy]

bbot/modules/hunt.py

@@ -284,11 +284,29 @@ class hunt(BaseModule):
 
     async def handle_event(self, event):
         p = event.data["name"]
+        matching_categories = []
+
+        # Collect all matching categories
         for k in hunt_param_dict.keys():
             if p.lower() in hunt_param_dict[k]:
-                description = f"Found potential {k.upper()} parameter [{p}]"
-                data = {"host": str(event.host), "description": description}
-                url = event.data.get("url", "")
-                if url:
-                    data["url"] = url
-                await self.emit_event(data, "FINDING", event)
+                matching_categories.append(k)
+
+        if matching_categories:
+            # Create a comma-separated string of categories
+            category_str = ", ".join(matching_categories)
+            description = f"Found potentially interesting parameter. Name: [{p}] Parameter Type: [{event.data['type']}] Categories: [{category_str}]"
+
+            if (
+                "original_value" in event.data.keys()
+                and event.data["original_value"] != ""
+                and event.data["original_value"] is not None
+            ):
+                description += (
+                    f" Original Value: [{self.helpers.truncate_string(str(event.data['original_value']), 200)}]"
+                )
+
+            data = {"host": str(event.host), "description": description}
+            url = event.data.get("url", "")
+            if url:
+                data["url"] = url
+            await self.emit_event(data, "FINDING", event)

bbot/modules/internal/aggregate.py

@@ -2,6 +2,7 @@ from bbot.modules.report.base import BaseReportModule
 
 
 class aggregate(BaseReportModule):
+    watched_events = []
     flags = ["passive", "safe"]
     meta = {
         "description": "Summarize statistics at the end of a scan",