bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/modules/output/sqlite.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class SQLite(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a SQLite database"}
+    options = {
+        "database": "",
+    }
+    options_desc = {
+        "database": "The path to the sqlite database file",
+    }
+    deps_pip = ["sqlmodel", "aiosqlite"]
+
+    async def setup(self):
+        db_file = self.config.get("database", "")
+        if not db_file:
+            db_file = self.scan.home / "output.sqlite"
+        db_file = Path(db_file)
+        if not db_file.is_absolute():
+            db_file = self.scan.home / db_file
+        self.db_file = db_file
+        self.db_file.parent.mkdir(parents=True, exist_ok=True)
+        return await super().setup()
+
+    def connection_string(self, mask_password=False):
+        return f"sqlite+aiosqlite:///{self.db_file}"

bbot/modules/output/stdout.py

@@ -20,7 +20,7 @@ class Stdout(BaseOutputModule):
 
     async def setup(self):
         self.text_format = self.config.get("format", "text").strip().lower()
-        if not self.text_format in self.format_choices:
+        if self.text_format not in self.format_choices:
             return (
                 False,
                 f'Invalid text format choice, "{self.text_format}" (choices: {",".join(self.format_choices)})',
@@ -33,7 +33,7 @@ class Stdout(BaseOutputModule):
 
     async def filter_event(self, event):
         if self.accept_event_types:
-            if not event.type in self.accept_event_types:
+            if event.type not in self.accept_event_types:
                 return False, f'Event type "{event.type}" is not in the allowed event_types'
         return True
 

bbot/modules/output/teams.py

@@ -10,14 +10,115 @@ class Teams(WebhookOutputModule):
     }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
-        "webhook_url": "Discord webhook URL",
+        "webhook_url": "Teams webhook URL",
         "event_types": "Types of events to send",
         "min_severity": "Only allow VULNERABILITY events of this severity or higher",
     }
     _module_threads = 5
-    good_status_code = 200
-    content_key = "text"
 
-    def evaluate_response(self, response):
-        text = getattr(response, "text", "")
-        return text == "1"
+    async def handle_event(self, event):
+        while 1:
+            data = self.format_message(event)
+
+            response = await self.helpers.request(
+                url=self.webhook_url,
+                method="POST",
+                json=data,
+            )
+            status_code = getattr(response, "status_code", 0)
+            if self.evaluate_response(response):
+                break
+            else:
+                response_data = getattr(response, "text", "")
+                try:
+                    retry_after = response.json().get("retry_after", 1)
+                except Exception:
+                    retry_after = 1
+                self.verbose(
+                    f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds"
+                )
+                await self.helpers.sleep(retry_after)
+
+    def trim_message(self, message):
+        if len(message) > self.message_size_limit:
+            message = message[: self.message_size_limit - 3] + "..."
+        return message
+
+    def format_message_str(self, event):
+        items = []
+        msg = self.trim_message(event.data)
+        items.append({"type": "TextBlock", "text": f"{msg}", "wrap": True})
+        items.append({"type": "FactSet", "facts": [{"title": "Tags:", "value": ", ".join(event.tags)}]})
+        return items
+
+    def format_message_other(self, event):
+        items = [{"type": "FactSet", "facts": []}]
+        for key, value in event.data.items():
+            if key != "severity":
+                msg = self.trim_message(str(value))
+                items[0]["facts"].append({"title": f"{key}:", "value": msg})
+        return items
+
+    def get_severity_color(self, event):
+        color = "Accent"
+        if event.type == "VULNERABILITY":
+            severity = event.data.get("severity", "UNKNOWN")
+            if severity == "CRITICAL":
+                color = "Attention"
+            elif severity == "HIGH":
+                color = "Attention"
+            elif severity == "MEDIUM":
+                color = "Warning"
+            elif severity == "LOW":
+                color = "Good"
+        return color
+
+    def format_message(self, event):
+        adaptive_card = {
+            "type": "message",
+            "attachments": [
+                {
+                    "contentType": "application/vnd.microsoft.card.adaptive",
+                    "contentUrl": None,
+                    "content": {
+                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                        "type": "AdaptiveCard",
+                        "version": "1.2",
+                        "msteams": {"width": "full"},
+                        "body": [],
+                    },
+                }
+            ],
+        }
+        heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "size": "Large", "style": "heading"}
+        body = adaptive_card["attachments"][0]["content"]["body"]
+        body.append(heading)
+        if event.type in ("VULNERABILITY", "FINDING"):
+            subheading = {
+                "type": "TextBlock",
+                "text": event.data.get("severity", "UNKNOWN"),
+                "spacing": "None",
+                "size": "Large",
+                "wrap": True,
+            }
+            subheading["color"] = self.get_severity_color(event)
+            body.append(subheading)
+        main_text = {
+            "type": "ColumnSet",
+            "separator": True,
+            "spacing": "Medium",
+            "columns": [
+                {
+                    "type": "Column",
+                    "width": "stretch",
+                    "items": [],
+                }
+            ],
+        }
+        if isinstance(event.data, str):
+            items = self.format_message_str(event)
+        else:
+            items = self.format_message_other(event)
+        main_text["columns"][0]["items"] = items
+        body.append(main_text)
+        return adaptive_card

bbot/modules/paramminer_headers.py

@@ -15,7 +15,7 @@ class paramminer_headers(BaseModule):
     meta = {
         "description": "Use smart brute-force to check for common HTTP header parameters",
         "created_date": "2022-04-15",
-        "author": "@pmueller",
+        "author": "@liquidsec",
     }
     options = {
         "wordlist": "",  # default is defined within setup function
@@ -82,7 +82,6 @@ class paramminer_headers(BaseModule):
     header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$")
 
     async def setup(self):
-
         self.recycle_words = self.config.get("recycle_words", True)
         self.event_dict = {}
         self.already_checked = set()
@@ -90,11 +89,11 @@ class paramminer_headers(BaseModule):
         if not wordlist:
             wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}"
         self.debug(f"Using wordlist: [{wordlist}]")
-        self.wl = set(
+        self.wl = {
             h.strip().lower()
             for h in self.helpers.read_file(await self.helpers.wordlist(wordlist))
             if len(h) > 0 and "%" not in h
-        )
+        }
 
         # check against the boring list (if the option is set)
         if self.config.get("skip_boring_words", True):
@@ -157,7 +156,6 @@ class paramminer_headers(BaseModule):
             )
 
     async def handle_event(self, event):
-
         # If recycle words is enabled, we will collect WEB_PARAMETERS we find to build our list in finish()
         # We also collect any parameters of type "SPECULATIVE"
         if event.type == "WEB_PARAMETER":
@@ -174,7 +172,7 @@ class paramminer_headers(BaseModule):
                 self.debug(f"Error initializing compare helper: {e}")
                 return
             batch_size = await self.count_test(url)
-            if batch_size == None or batch_size <= 0:
+            if batch_size is None or batch_size <= 0:
                 self.debug(f"Failed to get baseline max {self.compare_mode} count, aborting")
                 return
             self.debug(f"Resolved batch_size at {str(batch_size)}")
@@ -197,11 +195,11 @@ class paramminer_headers(BaseModule):
         baseline = await self.helpers.request(url)
         if baseline is None:
             return
-        if str(baseline.status_code)[0] in ("4", "5"):
+        if str(baseline.status_code)[0] in {"4", "5"}:
             return
         for count, args, kwargs in self.gen_count_args(url):
             r = await self.helpers.request(*args, **kwargs)
-            if r is not None and not ((str(r.status_code)[0] in ("4", "5"))):
+            if r is not None and str(r.status_code)[0] not in {"4", "5"}:
                 return count
 
     def gen_count_args(self, url):
@@ -224,7 +222,7 @@ class paramminer_headers(BaseModule):
         elif len(group) > 1 or (len(group) == 1 and len(reasons) == 0):
             for group_slice in self.helpers.split_list(group):
                 match, reasons, reflection, subject_response = await self.check_batch(compare_helper, url, group_slice)
-                if match == False:
+                if match is False:
                     async for r in self.binary_search(compare_helper, url, group_slice, reasons, reflection):
                         yield r
         else:
@@ -240,8 +238,7 @@ class paramminer_headers(BaseModule):
         return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1))
 
     async def finish(self):
-
-        untested_matches = sorted(list(self.extracted_words_master.copy()))
+        untested_matches = sorted(self.extracted_words_master.copy())
         for url, (event, batch_size) in list(self.event_dict.items()):
             try:
                 compare_helper = self.helpers.http_compare(url)

bbot/modules/passivetotal.py

@@ -11,36 +11,36 @@ class passivetotal(subdomain_enum_apikey):
         "author": "@TheTechromancer",
         "auth_required": True,
     }
-    options = {"username": "", "api_key": ""}
-    options_desc = {"username": "RiskIQ Username", "api_key": "RiskIQ API Key"}
+    options = {"api_key": ""}
+    options_desc = {"api_key": "PassiveTotal API Key in the format of 'username:api_key'"}
 
     base_url = "https://api.passivetotal.org/v2"
 
     async def setup(self):
-        self.username = self.config.get("username", "")
-        self.api_key = self.config.get("api_key", "")
-        self.auth = (self.username, self.api_key)
         return await super().setup()
 
     async def ping(self):
         url = f"{self.base_url}/account/quota"
-        j = (await self.request_with_fail_count(url, auth=self.auth)).json()
+        j = (await self.api_request(url)).json()
         limit = j["user"]["limits"]["search_api"]
         used = j["user"]["counts"]["search_api"]
         assert used < limit, "No quota remaining"
 
+    def prepare_api_request(self, url, kwargs):
+        api_username, api_key = self.api_key.split(":", 1)
+        kwargs["auth"] = (api_username, api_key)
+        return url, kwargs
+
     async def abort_if(self, event):
         # RiskIQ is famous for their junk data
         return await super().abort_if(event) or "unresolved" in event.tags
 
     async def request_url(self, query):
         url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url, auth=self.auth)
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for subdomain in r.json().get("subdomains", []):
-            yield f"{subdomain}.{query}"
-
-    @property
-    def auth_secret(self):
-        return self.username and self.api_key
+            results.add(f"{subdomain}.{query}")
+        return results

bbot/modules/portscan.py

@@ -6,6 +6,9 @@ from radixtarget import RadixTarget
 from bbot.modules.base import BaseModule
 
 
+# TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN.
+
+
 class portscan(BaseModule):
     flags = ["active", "portscan", "safe"]
     watched_events = ["IP_ADDRESS", "IP_RANGE", "DNS_NAME"]
@@ -27,6 +30,8 @@ class portscan(BaseModule):
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
+        "cdn_tags": "cdn-",
+        "allowed_cdn_ports": None,
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -39,6 +44,8 @@ class portscan(BaseModule):
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
+        "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,cloud'",
+        "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
     deps_common = ["masscan"]
     batch_size = 1000000
@@ -60,7 +67,15 @@ class portscan(BaseModule):
             try:
                 self.helpers.parse_port_string(self.ports)
             except ValueError as e:
-                return False, f"Error parsing ports: {e}"
+                return False, f"Error parsing ports '{self.ports}': {e}"
+        self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
+        self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", None)
+        if self.allowed_cdn_ports is not None:
+            try:
+                self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(",")]
+            except Exception as e:
+                return False, f"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}"
+
         # whether we've finished scanning our original scan targets
         self.scanned_initial_targets = False
         # keeps track of individual scanned IPs and their open ports
@@ -84,17 +99,17 @@ class portscan(BaseModule):
             return False, "Masscan failed to run"
         returncode = getattr(ipv6_result, "returncode", 0)
         if returncode and "failed to detect IPv6 address" in ipv6_result.stderr:
-            self.warning(f"It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
+            self.warning("It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
             self.ipv6_support = False
         return True
 
     async def handle_batch(self, *events):
-        # on our first run, we automatically include all our intial scan targets
+        # on our first run, we automatically include all our initial scan targets
         if not self.scanned_initial_targets:
             self.scanned_initial_targets = True
             events = set(events)
             events.update(
-                set([e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")])
+                {e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")}
             )
 
         # ping scan
@@ -227,9 +242,20 @@ class portscan(BaseModule):
             parent=parent_event,
             context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}",
         )
-        await self.emit_event(event)
+
+        await self.emit_event(event, abort_if=self.abort_if)
         return event
 
+    def abort_if(self, event):
+        if self.allowed_cdn_ports is not None:
+            # if the host is a CDN
+            for cdn_tag in self.cdn_tags:
+                if any(t.startswith(str(cdn_tag)) for t in event.tags):
+                    # and if its port isn't in the list of allowed CDN ports
+                    if event.port not in self.allowed_cdn_ports:
+                        return True, "event is a CDN and port is not in the allowed list"
+        return False
+
     def parse_json_line(self, line):
         try:
             j = json.loads(line)
@@ -308,7 +334,7 @@ class portscan(BaseModule):
         if "FAIL" in s:
             self.warning(s)
             self.warning(
-                f'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
+                'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
             )
         else:
             self.verbose(s)

bbot/modules/postman.py

@@ -1,36 +1,62 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
+from bbot.modules.templates.postman import postman
 
 
-class postman(subdomain_enum):
-    watched_events = ["DNS_NAME"]
-    produced_events = ["URL_UNVERIFIED"]
+class postman(postman):
+    watched_events = ["ORG_STUB", "SOCIAL"]
+    produced_events = ["CODE_REPOSITORY"]
     flags = ["passive", "subdomain-enum", "safe", "code-enum"]
     meta = {
-        "description": "Query Postman's API for related workspaces, collections, requests",
-        "created_date": "2023-12-23",
+        "description": "Query Postman's API for related workspaces, collections, requests and download them",
+        "created_date": "2024-09-07",
         "author": "@domwhewell-sage",
     }
 
-    base_url = "https://www.postman.com/_api"
-
-    headers = {
-        "Content-Type": "application/json",
-        "X-App-Version": "10.18.8-230926-0808",
-        "X-Entity-Team-Id": "0",
-        "Origin": "https://www.postman.com",
-        "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
-    }
-
     reject_wildcards = False
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}")
-        for url, context in await self.query(query):
-            await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe", context=context)
+        # Handle postman profile
+        if event.type == "SOCIAL":
+            await self.handle_profile(event)
+        elif event.type == "ORG_STUB":
+            await self.handle_org_stub(event)
+
+    async def handle_profile(self, event):
+        profile_name = event.data.get("profile_name", "")
+        self.verbose(f"Searching for postman workspaces, collections, requests belonging to {profile_name}")
+        for item in await self.query(profile_name):
+            workspace = item["document"]
+            name = workspace["slug"]
+            profile = workspace["publisherHandle"]
+            if profile_name.lower() == profile.lower():
+                self.verbose(f"Got {name}")
+                workspace_url = f"{self.html_url}/{profile}/{name}"
+                await self.emit_event(
+                    {"url": workspace_url},
+                    "CODE_REPOSITORY",
+                    tags="postman",
+                    parent=event,
+                    context=f'{{module}} searched postman.com for workspaces belonging to "{profile_name}" and found "{name}" at {{event.type}}: {workspace_url}',
+                )
+
+    async def handle_org_stub(self, event):
+        org_name = event.data
+        self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}")
+        for item in await self.query(org_name):
+            workspace = item["document"]
+            name = workspace["slug"]
+            profile = workspace["publisherHandle"]
+            self.verbose(f"Got {name}")
+            workspace_url = f"{self.html_url}/{profile}/{name}"
+            await self.emit_event(
+                {"url": workspace_url},
+                "CODE_REPOSITORY",
+                tags="postman",
+                parent=event,
+                context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
+            )
 
     async def query(self, query):
-        interesting_urls = []
+        data = []
         url = f"{self.base_url}/ws/proxy"
         json = {
             "service": "search",
@@ -39,11 +65,6 @@ class postman(subdomain_enum):
             "body": {
                 "queryIndices": [
                     "collaboration.workspace",
-                    "runtime.collection",
-                    "runtime.request",
-                    "adp.api",
-                    "flow.flow",
-                    "apinetwork.team",
                 ],
                 "queryText": self.helpers.quote(query),
                 "size": 100,
@@ -57,108 +78,11 @@ class postman(subdomain_enum):
         }
         r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
         if r is None:
-            return interesting_urls
+            return data
         status_code = getattr(r, "status_code", 0)
         try:
             json = r.json()
         except Exception as e:
             self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return interesting_urls
-        workspaces = []
-        for item in json.get("data", {}):
-            for workspace in item.get("document", {}).get("workspaces", []):
-                if workspace not in workspaces:
-                    workspaces.append(workspace)
-        for item in workspaces:
-            id = item.get("id", "")
-            name = item.get("name", "")
-            tldextract = self.helpers.tldextract(query)
-            if tldextract.domain.lower() in name.lower():
-                self.verbose(f"Discovered workspace {name} ({id})")
-                workspace_url = f"{self.base_url}/workspace/{id}"
-                interesting_urls.append(
-                    (
-                        workspace_url,
-                        f'{{module}} searched postman.com for "{query}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
-                    )
-                )
-                environments, collections = await self.search_workspace(id)
-                globals_url = f"{self.base_url}/workspace/{id}/globals"
-                interesting_urls.append(
-                    (
-                        globals_url,
-                        f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, and found globals at {{event.type}}: {globals_url}',
-                    )
-                )
-                for e_id in environments:
-                    env_url = f"{self.base_url}/environment/{e_id}"
-                    interesting_urls.append(
-                        (
-                            env_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated environments, and found {{event.type}}: {env_url}',
-                        )
-                    )
-                for c_id in collections:
-                    collection_url = f"{self.base_url}/collection/{c_id}"
-                    interesting_urls.append(
-                        (
-                            collection_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated collections, and found {{event.type}}: {collection_url}',
-                        )
-                    )
-                requests = await self.search_collections(id)
-                for r_id in requests:
-                    request_url = f"{self.base_url}/request/{r_id}"
-                    interesting_urls.append(
-                        (
-                            request_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated requests, and found {{event.type}}: {request_url}',
-                        )
-                    )
-            else:
-                self.verbose(f"Skipping workspace {name} ({id}) as it does not appear to be in scope")
-        return interesting_urls
-
-    async def search_workspace(self, id):
-        url = f"{self.base_url}/workspace/{id}"
-        r = await self.helpers.request(url)
-        if r is None:
-            return [], []
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-            if not isinstance(json, dict):
-                raise ValueError(f"Got unexpected value for JSON: {json}")
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return [], []
-        environments = json.get("data", {}).get("dependencies", {}).get("environments", [])
-        collections = json.get("data", {}).get("dependencies", {}).get("collections", [])
-        return environments, collections
-
-    async def search_collections(self, id):
-        request_ids = []
-        url = f"{self.base_url}/list/collection?workspace={id}"
-        r = await self.helpers.request(url, method="POST")
-        if r is None:
-            return request_ids
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return request_ids
-        for item in json.get("data", {}):
-            request_ids.extend(await self.parse_collection(item))
-        return request_ids
-
-    async def parse_collection(self, json):
-        request_ids = []
-        folders = json.get("folders", [])
-        requests = json.get("requests", [])
-        for folder in folders:
-            request_ids.extend(await self.parse_collection(folder))
-        for request in requests:
-            r_id = request.get("id", "")
-            request_ids.append(r_id)
-        return request_ids
+            return None
+        return json.get("data", [])