bbot 2.3.0.5546rc0__py3-none-any.whl → 2.3.1__py3-none-any.whl

bbot/modules/output/teams.py

@@ -8,36 +8,21 @@ class Teams(WebhookOutputModule):
         "created_date": "2023-08-14",
         "author": "@TheTechromancer",
     }
-    options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
+    options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW", "retries": 10}
     options_desc = {
         "webhook_url": "Teams webhook URL",
         "event_types": "Types of events to send",
         "min_severity": "Only allow VULNERABILITY events of this severity or higher",
+        "retries": "Number of times to retry sending the message before skipping the event",
     }
-    _module_threads = 5
 
     async def handle_event(self, event):
-        while 1:
-            data = self.format_message(event)
-
-            response = await self.helpers.request(
-                url=self.webhook_url,
-                method="POST",
-                json=data,
-            )
-            status_code = getattr(response, "status_code", 0)
-            if self.evaluate_response(response):
-                break
-            else:
-                response_data = getattr(response, "text", "")
-                try:
-                    retry_after = response.json().get("retry_after", 1)
-                except Exception:
-                    retry_after = 1
-                self.verbose(
-                    f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds"
-                )
-                await self.helpers.sleep(retry_after)
+        data = self.format_message(event)
+        await self.api_request(
+            url=self.webhook_url,
+            method="POST",
+            json=data,
+        )
 
     def trim_message(self, message):
         if len(message) > self.message_size_limit:
@@ -62,7 +47,7 @@ class Teams(WebhookOutputModule):
     def get_severity_color(self, event):
         color = "Accent"
         if event.type == "VULNERABILITY":
-            severity = event.data.get("severity", "UNKNOWN")
+            severity = event.data.get("severity", "INFO")
             if severity == "CRITICAL":
                 color = "Attention"
             elif severity == "HIGH":
@@ -96,7 +81,7 @@ class Teams(WebhookOutputModule):
         if event.type in ("VULNERABILITY", "FINDING"):
             subheading = {
                 "type": "TextBlock",
-                "text": event.data.get("severity", "UNKNOWN"),
+                "text": event.data.get("severity", "INFO"),
                 "spacing": "None",
                 "size": "Large",
                 "wrap": True,

bbot/modules/output/web_parameters.py

@@ -0,0 +1,55 @@
+from contextlib import suppress
+from collections import defaultdict
+
+from bbot.modules.output.base import BaseOutputModule
+
+
+class Web_parameters(BaseOutputModule):
+    watched_events = ["WEB_PARAMETER"]
+    meta = {
+        "description": "Output WEB_PARAMETER names to a file",
+        "created_date": "2025-01-25",
+        "author": "@liquidsec",
+    }
+    options = {"output_file": "", "include_count": False}
+    options_desc = {
+        "output_file": "Output to file",
+        "include_count": "Include the count of each parameter in the output",
+    }
+
+    output_filename = "web_parameters.txt"
+
+    async def setup(self):
+        self._prep_output_dir(self.output_filename)
+        self.parameter_counts = defaultdict(int)
+        return True
+
+    async def handle_event(self, event):
+        parameter_name = event.data.get("name", "")
+        if parameter_name:
+            self.parameter_counts[parameter_name] += 1
+
+    async def cleanup(self):
+        if getattr(self, "_file", None) is not None:
+            with suppress(Exception):
+                self.file.close()
+
+    async def report(self):
+        include_count = self.config.get("include_count", False)
+
+        # Sort behavior:
+        # - If include_count is True, sort by count (descending) and then alphabetically by name
+        # - If include_count is False, sort alphabetically by name only
+        sorted_parameters = sorted(
+            self.parameter_counts.items(), key=lambda x: (-x[1], x[0]) if include_count else x[0]
+        )
+        for param, count in sorted_parameters:
+            if include_count:
+                # Include the count of each parameter in the output
+                self.file.write(f"{count}\t{param}\n")
+            else:
+                # Only include the parameter name, effectively deduplicating by name
+                self.file.write(f"{param}\n")
+        self.file.flush()
+        if getattr(self, "_file", None) is not None:
+            self.info(f"Saved web parameters to {self.output_file}")

bbot/modules/paramminer_headers.py

@@ -140,7 +140,7 @@ class paramminer_headers(BaseModule):
                 tags = ["http_reflection"]
             description = f"[Paramminer] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}] Reflection: [{str(reflection)}]"
             reflected = "reflected " if reflection else ""
-            self.extracted_words_master.add(result)
+
             await self.emit_event(
                 {
                     "host": str(event.host),
@@ -159,9 +159,12 @@ class paramminer_headers(BaseModule):
         # If recycle words is enabled, we will collect WEB_PARAMETERS we find to build our list in finish()
         # We also collect any parameters of type "SPECULATIVE"
         if event.type == "WEB_PARAMETER":
+            parameter_name = event.data.get("name")
             if self.recycle_words or (event.data.get("type") == "SPECULATIVE"):
-                parameter_name = event.data.get("name")
-                if self.config.get("skip_boring_words", True) and parameter_name not in self.boring_words:
+                if self.config.get("skip_boring_words", True) and parameter_name in self.boring_words:
+                    return
+                if parameter_name not in self.wl:  # Ensure it's not already in the wordlist
+                    self.debug(f"Adding {parameter_name} to wordlist")
                     self.extracted_words_master.add(parameter_name)
 
         elif event.type == "HTTP_RESPONSE":
@@ -238,27 +241,29 @@ class paramminer_headers(BaseModule):
         return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1))
 
     async def finish(self):
-        untested_matches = sorted(self.extracted_words_master.copy())
         for url, (event, batch_size) in list(self.event_dict.items()):
             try:
                 compare_helper = self.helpers.http_compare(url)
             except HttpCompareError as e:
                 self.debug(f"Error initializing compare helper: {e}")
                 continue
-            untested_matches_copy = untested_matches.copy()
-            for i in untested_matches:
-                h = hash(i + url)
-                if h in self.already_checked:
-                    untested_matches_copy.remove(i)
+            words_to_process = {
+                i for i in self.extracted_words_master.copy() if hash(i + url) not in self.already_checked
+            }
             try:
-                results = await self.do_mining(untested_matches_copy, url, batch_size, compare_helper)
+                results = await self.do_mining(words_to_process, url, batch_size, compare_helper)
             except HttpCompareError as e:
                 self.debug(f"Encountered HttpCompareError: [{e}] for URL [{url}]")
                 continue
             await self.process_results(event, results)
 
     async def filter_event(self, event):
+        # Filter out static endpoints
+        if event.data.get("url").endswith(tuple(f".{ext}" for ext in self.config.get("url_extension_static", []))):
+            return False
+
         # We don't need to look at WEB_PARAMETERS that we produced
         if str(event.module).startswith("paramminer"):
             return False
+
         return True

bbot/modules/portfilter.py

@@ -0,0 +1,41 @@
+from bbot.modules.base import BaseInterceptModule
+
+
+class portfilter(BaseInterceptModule):
+    watched_events = ["OPEN_TCP_PORT"]
+    flags = ["passive", "safe"]
+    meta = {
+        "description": "Filter out unwanted open ports from cloud/CDN targets",
+        "created_date": "2025-01-06",
+        "author": "@TheTechromancer",
+    }
+    options = {
+        "cdn_tags": "cdn-",
+        "allowed_cdn_ports": "80,443",
+    }
+    options_desc = {
+        "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,cloud'",
+        "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
+    }
+
+    async def setup(self):
+        self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
+        self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", "").strip()
+        if self.allowed_cdn_ports:
+            try:
+                self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(",")]
+            except Exception as e:
+                return False, f"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}"
+        return True
+
+    async def handle_event(self, event):
+        # if the port isn't in our list of allowed CDN ports
+        if event.port not in self.allowed_cdn_ports:
+            for cdn_tag in self.cdn_tags:
+                # and if any of the event's tags match our CDN filter
+                if any(t.startswith(str(cdn_tag)) for t in event.tags):
+                    return (
+                        False,
+                        f"one of the event's tags matches the tag '{cdn_tag}' and the port is not in the allowed list",
+                    )
+        return True

bbot/modules/portscan.py

@@ -30,8 +30,6 @@ class portscan(BaseModule):
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
-        "cdn_tags": "cdn-",
-        "allowed_cdn_ports": None,
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -44,8 +42,6 @@ class portscan(BaseModule):
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
-        "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,cloud'",
-        "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
     deps_common = ["masscan"]
     batch_size = 1000000
@@ -68,13 +64,6 @@ class portscan(BaseModule):
                 self.helpers.parse_port_string(self.ports)
             except ValueError as e:
                 return False, f"Error parsing ports '{self.ports}': {e}"
-        self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
-        self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", None)
-        if self.allowed_cdn_ports is not None:
-            try:
-                self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(",")]
-            except Exception as e:
-                return False, f"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}"
 
         # whether we've finished scanning our original scan targets
         self.scanned_initial_targets = False
@@ -243,19 +232,9 @@ class portscan(BaseModule):
             context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}",
         )
 
-        await self.emit_event(event, abort_if=self.abort_if)
+        await self.emit_event(event)
         return event
 
-    def abort_if(self, event):
-        if self.allowed_cdn_ports is not None:
-            # if the host is a CDN
-            for cdn_tag in self.cdn_tags:
-                if any(t.startswith(str(cdn_tag)) for t in event.tags):
-                    # and if its port isn't in the list of allowed CDN ports
-                    if event.port not in self.allowed_cdn_ports:
-                        return True, "event is a CDN and port is not in the allowed list"
-        return False
-
     def parse_json_line(self, line):
         try:
             j = json.loads(line)

bbot/modules/postman.py

@@ -10,52 +10,64 @@ class postman(postman):
         "created_date": "2024-09-07",
         "author": "@domwhewell-sage",
     }
-
+    options = {"api_key": ""}
+    options_desc = {"api_key": "Postman API Key"}
     reject_wildcards = False
 
     async def handle_event(self, event):
         # Handle postman profile
         if event.type == "SOCIAL":
-            await self.handle_profile(event)
+            owner = event.data.get("profile_name", "")
+            in_scope_workspaces = await self.process_workspaces(user=owner)
         elif event.type == "ORG_STUB":
-            await self.handle_org_stub(event)
-
-    async def handle_profile(self, event):
-        profile_name = event.data.get("profile_name", "")
-        self.verbose(f"Searching for postman workspaces, collections, requests belonging to {profile_name}")
-        for item in await self.query(profile_name):
-            workspace = item["document"]
-            name = workspace["slug"]
-            profile = workspace["publisherHandle"]
-            if profile_name.lower() == profile.lower():
-                self.verbose(f"Got {name}")
-                workspace_url = f"{self.html_url}/{profile}/{name}"
+            owner = event.data
+            in_scope_workspaces = await self.process_workspaces(org=owner)
+        if in_scope_workspaces:
+            for workspace in in_scope_workspaces:
+                repo_url = workspace["url"]
+                repo_name = workspace["repo_name"]
+                if event.type == "SOCIAL":
+                    context = f'{{module}} searched postman.com for workspaces belonging to "{owner}" and found "{repo_name}" at {{event.type}}: {repo_url}'
+                elif event.type == "ORG_STUB":
+                    context = f'{{module}} searched postman.com for "{owner}" and found matching workspace "{repo_name}" at {{event.type}}: {repo_url}'
                 await self.emit_event(
-                    {"url": workspace_url},
+                    {"url": repo_url},
                     "CODE_REPOSITORY",
                     tags="postman",
                     parent=event,
-                    context=f'{{module}} searched postman.com for workspaces belonging to "{profile_name}" and found "{name}" at {{event.type}}: {workspace_url}',
+                    context=context,
                 )
 
-    async def handle_org_stub(self, event):
-        org_name = event.data
-        self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}")
-        for item in await self.query(org_name):
-            workspace = item["document"]
-            name = workspace["slug"]
-            profile = workspace["publisherHandle"]
-            self.verbose(f"Got {name}")
-            workspace_url = f"{self.html_url}/{profile}/{name}"
-            await self.emit_event(
-                {"url": workspace_url},
-                "CODE_REPOSITORY",
-                tags="postman",
-                parent=event,
-                context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
-            )
+    async def process_workspaces(self, user=None, org=None):
+        in_scope_workspaces = []
+        owner = user or org
+        if owner:
+            self.verbose(f"Searching for postman workspaces, collections, requests for {owner}")
+            for item in await self.query(owner):
+                workspace = item["document"]
+                slug = workspace["slug"]
+                profile = workspace["publisherHandle"]
+                repo_url = f"{self.html_url}/{profile}/{slug}"
+                workspace_id = await self.get_workspace_id(repo_url)
+                if (org and workspace_id) or (user and owner.lower() == profile.lower()):
+                    self.verbose(f"Found workspace ID {workspace_id} for {repo_url}")
+                    data = await self.request_workspace(workspace_id)
+                    in_scope = await self.validate_workspace(
+                        data["workspace"], data["environments"], data["collections"]
+                    )
+                    if in_scope:
+                        in_scope_workspaces.append({"url": repo_url, "repo_name": slug})
+                    else:
+                        self.verbose(
+                            f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails"
+                        )
+        return in_scope_workspaces
 
     async def query(self, query):
+        def api_page_iter(url, page, page_size, offset, **kwargs):
+            kwargs["json"]["body"]["from"] = offset
+            return url, kwargs
+
         data = []
         url = f"{self.base_url}/ws/proxy"
         json = {
@@ -67,7 +79,7 @@ class postman(postman):
                     "collaboration.workspace",
                 ],
                 "queryText": self.helpers.quote(query),
-                "size": 100,
+                "size": 25,
                 "from": 0,
                 "clientTraceId": "",
                 "requestOrigin": "srp",
@@ -76,13 +88,19 @@ class postman(postman):
                 "domain": "public",
             },
         }
-        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
-        if r is None:
-            return data
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return None
-        return json.get("data", [])
+
+        agen = self.api_page_iter(
+            url, page_size=25, method="POST", iter_key=api_page_iter, json=json, _json=False, headers=self.headers
+        )
+        async for r in agen:
+            status_code = getattr(r, "status_code", 0)
+            if status_code != 200:
+                self.debug(f"Reached end of postman search results (url: {r.url}) with status code {status_code}")
+                break
+            try:
+                data.extend(r.json().get("data", []))
+            except Exception as e:
+                self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                return None
+
+        return data

bbot/modules/postman_download.py

@@ -24,11 +24,7 @@ class postman_download(postman):
         else:
             self.output_dir = self.scan.home / "postman_workspaces"
         self.helpers.mkdir(self.output_dir)
-        return await self.require_api_key()
-
-    def prepare_api_request(self, url, kwargs):
-        kwargs["headers"]["X-Api-Key"] = self.api_key
-        return url, kwargs
+        return await super().setup()
 
     async def filter_event(self, event):
         if event.type == "CODE_REPOSITORY":
@@ -45,149 +41,16 @@ class postman_download(postman):
             workspace = data["workspace"]
             environments = data["environments"]
             collections = data["collections"]
-            in_scope = await self.validate_workspace(workspace, environments, collections)
-            if in_scope:
-                workspace_path = self.save_workspace(workspace, environments, collections)
-                if workspace_path:
-                    self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}")
-                    codebase_event = self.make_event(
-                        {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event
-                    )
-                    await self.emit_event(
-                        codebase_event,
-                        context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}",
-                    )
-            else:
-                self.verbose(
-                    f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails, skipping download"
+            workspace_path = self.save_workspace(workspace, environments, collections)
+            if workspace_path:
+                self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}")
+                codebase_event = self.make_event(
+                    {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event
+                )
+                await self.emit_event(
+                    codebase_event,
+                    context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}",
                 )
-
-    async def get_workspace_id(self, repo_url):
-        workspace_id = ""
-        profile = repo_url.split("/")[-2]
-        name = repo_url.split("/")[-1]
-        url = f"{self.base_url}/ws/proxy"
-        json = {
-            "service": "workspaces",
-            "method": "GET",
-            "path": f"/workspaces?handle={profile}&slug={name}",
-        }
-        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
-        if r is None:
-            return workspace_id
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return workspace_id
-        data = json.get("data", [])
-        if len(data) == 1:
-            workspace_id = data[0]["id"]
-        return workspace_id
-
-    async def request_workspace(self, id):
-        data = {"workspace": {}, "environments": [], "collections": []}
-        workspace = await self.get_workspace(id)
-        if workspace:
-            # Main Workspace
-            name = workspace["name"]
-            data["workspace"] = workspace
-
-            # Workspace global variables
-            self.verbose(f"Downloading globals for workspace {name}")
-            globals = await self.get_globals(id)
-            data["environments"].append(globals)
-
-            # Workspace Environments
-            workspace_environments = workspace.get("environments", [])
-            if workspace_environments:
-                self.verbose(f"Downloading environments for workspace {name}")
-                for _ in workspace_environments:
-                    environment_id = _["uid"]
-                    environment = await self.get_environment(environment_id)
-                    data["environments"].append(environment)
-
-            # Workspace Collections
-            workspace_collections = workspace.get("collections", [])
-            if workspace_collections:
-                self.verbose(f"Downloading collections for workspace {name}")
-                for _ in workspace_collections:
-                    collection_id = _["uid"]
-                    collection = await self.get_collection(collection_id)
-                    data["collections"].append(collection)
-        return data
-
-    async def get_workspace(self, workspace_id):
-        workspace = {}
-        workspace_url = f"{self.api_url}/workspaces/{workspace_id}"
-        r = await self.api_request(workspace_url)
-        if r is None:
-            return workspace
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return workspace
-        workspace = json.get("workspace", {})
-        return workspace
-
-    async def get_globals(self, workspace_id):
-        globals = {}
-        globals_url = f"{self.base_url}/workspace/{workspace_id}/globals"
-        r = await self.helpers.request(globals_url, headers=self.headers)
-        if r is None:
-            return globals
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return globals
-        globals = json.get("data", {})
-        return globals
-
-    async def get_environment(self, environment_id):
-        environment = {}
-        environment_url = f"{self.api_url}/environments/{environment_id}"
-        r = await self.api_request(environment_url)
-        if r is None:
-            return environment
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return environment
-        environment = json.get("environment", {})
-        return environment
-
-    async def get_collection(self, collection_id):
-        collection = {}
-        collection_url = f"{self.api_url}/collections/{collection_id}"
-        r = await self.api_request(collection_url)
-        if r is None:
-            return collection
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return collection
-        collection = json.get("collection", {})
-        return collection
-
-    async def validate_workspace(self, workspace, environments, collections):
-        name = workspace.get("name", "")
-        full_wks = str([workspace, environments, collections])
-        in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks)
-        if in_scope_hosts:
-            self.verbose(
-                f'Found in-scope hostname(s): "{in_scope_hosts}" in workspace {name}, it appears to be in-scope'
-            )
-            return True
-        return False
 
     def save_workspace(self, workspace, environments, collections):
         zip_path = None

bbot/modules/sitedossier.py

@@ -36,7 +36,7 @@ class sitedossier(subdomain_enum):
         base_url = f"{self.base_url}/{self.helpers.quote(query)}"
         url = str(base_url)
         for i, page in enumerate(range(1, 100 * self.max_pages + 2, 100)):
-            self.verbose(f"Fetching page #{i+1} for {query}")
+            self.verbose(f"Fetching page #{i + 1} for {query}")
             if page > 1:
                 url = f"{base_url}/{page}"
             response = await self.helpers.request(url)

bbot/modules/skymem.py

@@ -51,5 +51,5 @@ class skymem(emailformat):
                     email,
                     "EMAIL_ADDRESS",
                     parent=event,
-                    context=f'{{module}} searched skymem.info for "{query}" and found {{event.type}} on page {i+1}: {{event.data}}',
+                    context=f'{{module}} searched skymem.info for "{query}" and found {{event.type}} on page {i + 1}: {{event.data}}',
                 )