bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/passivetotal.py

@@ -11,36 +11,36 @@ class passivetotal(subdomain_enum_apikey):
         "author": "@TheTechromancer",
         "auth_required": True,
     }
-    options = {"username": "", "api_key": ""}
-    options_desc = {"username": "RiskIQ Username", "api_key": "RiskIQ API Key"}
+    options = {"api_key": ""}
+    options_desc = {"api_key": "PassiveTotal API Key in the format of 'username:api_key'"}
 
     base_url = "https://api.passivetotal.org/v2"
 
     async def setup(self):
-        self.username = self.config.get("username", "")
-        self.api_key = self.config.get("api_key", "")
-        self.auth = (self.username, self.api_key)
         return await super().setup()
 
     async def ping(self):
         url = f"{self.base_url}/account/quota"
-        j = (await self.request_with_fail_count(url, auth=self.auth)).json()
+        j = (await self.api_request(url)).json()
         limit = j["user"]["limits"]["search_api"]
         used = j["user"]["counts"]["search_api"]
         assert used < limit, "No quota remaining"
 
+    def prepare_api_request(self, url, kwargs):
+        api_username, api_key = self.api_key.split(":", 1)
+        kwargs["auth"] = (api_username, api_key)
+        return url, kwargs
+
     async def abort_if(self, event):
         # RiskIQ is famous for their junk data
         return await super().abort_if(event) or "unresolved" in event.tags
 
     async def request_url(self, query):
         url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url, auth=self.auth)
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for subdomain in r.json().get("subdomains", []):
-            yield f"{subdomain}.{query}"
-
-    @property
-    def auth_secret(self):
-        return self.username and self.api_key
+            results.add(f"{subdomain}.{query}")
+        return results

bbot/modules/portscan.py

@@ -6,6 +6,9 @@ from radixtarget import RadixTarget
 from bbot.modules.base import BaseModule
 
 
+# TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN.
+
+
 class portscan(BaseModule):
     flags = ["active", "portscan", "safe"]
     watched_events = ["IP_ADDRESS", "IP_RANGE", "DNS_NAME"]
@@ -27,6 +30,8 @@ class portscan(BaseModule):
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
+        "cdn_tags": "cdn-",
+        "allowed_cdn_ports": None,
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -39,6 +44,8 @@ class portscan(BaseModule):
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
+        "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,cloud'",
+        "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
     deps_common = ["masscan"]
     batch_size = 1000000
@@ -60,7 +67,15 @@ class portscan(BaseModule):
             try:
                 self.helpers.parse_port_string(self.ports)
             except ValueError as e:
-                return False, f"Error parsing ports: {e}"
+                return False, f"Error parsing ports '{self.ports}': {e}"
+        self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
+        self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", None)
+        if self.allowed_cdn_ports is not None:
+            try:
+                self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(",")]
+            except Exception as e:
+                return False, f"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}"
+
         # whether we've finished scanning our original scan targets
         self.scanned_initial_targets = False
         # keeps track of individual scanned IPs and their open ports
@@ -84,17 +99,17 @@ class portscan(BaseModule):
             return False, "Masscan failed to run"
         returncode = getattr(ipv6_result, "returncode", 0)
         if returncode and "failed to detect IPv6 address" in ipv6_result.stderr:
-            self.warning(f"It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
+            self.warning("It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
             self.ipv6_support = False
         return True
 
     async def handle_batch(self, *events):
-        # on our first run, we automatically include all our intial scan targets
+        # on our first run, we automatically include all our initial scan targets
         if not self.scanned_initial_targets:
             self.scanned_initial_targets = True
             events = set(events)
             events.update(
-                set([e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")])
+                {e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")}
             )
 
         # ping scan
@@ -227,9 +242,20 @@ class portscan(BaseModule):
             parent=parent_event,
             context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}",
         )
-        await self.emit_event(event)
+
+        await self.emit_event(event, abort_if=self.abort_if)
         return event
 
+    def abort_if(self, event):
+        if self.allowed_cdn_ports is not None:
+            # if the host is a CDN
+            for cdn_tag in self.cdn_tags:
+                if any(t.startswith(str(cdn_tag)) for t in event.tags):
+                    # and if its port isn't in the list of allowed CDN ports
+                    if event.port not in self.allowed_cdn_ports:
+                        return True, "event is a CDN and port is not in the allowed list"
+        return False
+
     def parse_json_line(self, line):
         try:
             j = json.loads(line)
@@ -308,7 +334,7 @@ class portscan(BaseModule):
         if "FAIL" in s:
             self.warning(s)
             self.warning(
-                f'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
+                'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
             )
         else:
             self.verbose(s)

bbot/modules/postman.py

@@ -1,36 +1,62 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
+from bbot.modules.templates.postman import postman
 
 
-class postman(subdomain_enum):
-    watched_events = ["DNS_NAME"]
-    produced_events = ["URL_UNVERIFIED"]
+class postman(postman):
+    watched_events = ["ORG_STUB", "SOCIAL"]
+    produced_events = ["CODE_REPOSITORY"]
     flags = ["passive", "subdomain-enum", "safe", "code-enum"]
     meta = {
-        "description": "Query Postman's API for related workspaces, collections, requests",
-        "created_date": "2023-12-23",
+        "description": "Query Postman's API for related workspaces, collections, requests and download them",
+        "created_date": "2024-09-07",
         "author": "@domwhewell-sage",
     }
 
-    base_url = "https://www.postman.com/_api"
-
-    headers = {
-        "Content-Type": "application/json",
-        "X-App-Version": "10.18.8-230926-0808",
-        "X-Entity-Team-Id": "0",
-        "Origin": "https://www.postman.com",
-        "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
-    }
-
     reject_wildcards = False
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}")
-        for url, context in await self.query(query):
-            await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe", context=context)
+        # Handle postman profile
+        if event.type == "SOCIAL":
+            await self.handle_profile(event)
+        elif event.type == "ORG_STUB":
+            await self.handle_org_stub(event)
+
+    async def handle_profile(self, event):
+        profile_name = event.data.get("profile_name", "")
+        self.verbose(f"Searching for postman workspaces, collections, requests belonging to {profile_name}")
+        for item in await self.query(profile_name):
+            workspace = item["document"]
+            name = workspace["slug"]
+            profile = workspace["publisherHandle"]
+            if profile_name.lower() == profile.lower():
+                self.verbose(f"Got {name}")
+                workspace_url = f"{self.html_url}/{profile}/{name}"
+                await self.emit_event(
+                    {"url": workspace_url},
+                    "CODE_REPOSITORY",
+                    tags="postman",
+                    parent=event,
+                    context=f'{{module}} searched postman.com for workspaces belonging to "{profile_name}" and found "{name}" at {{event.type}}: {workspace_url}',
+                )
+
+    async def handle_org_stub(self, event):
+        org_name = event.data
+        self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}")
+        for item in await self.query(org_name):
+            workspace = item["document"]
+            name = workspace["slug"]
+            profile = workspace["publisherHandle"]
+            self.verbose(f"Got {name}")
+            workspace_url = f"{self.html_url}/{profile}/{name}"
+            await self.emit_event(
+                {"url": workspace_url},
+                "CODE_REPOSITORY",
+                tags="postman",
+                parent=event,
+                context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
+            )
 
     async def query(self, query):
-        interesting_urls = []
+        data = []
         url = f"{self.base_url}/ws/proxy"
         json = {
             "service": "search",
@@ -39,11 +65,6 @@ class postman(subdomain_enum):
             "body": {
                 "queryIndices": [
                     "collaboration.workspace",
-                    "runtime.collection",
-                    "runtime.request",
-                    "adp.api",
-                    "flow.flow",
-                    "apinetwork.team",
                 ],
                 "queryText": self.helpers.quote(query),
                 "size": 100,
@@ -57,108 +78,11 @@ class postman(subdomain_enum):
         }
         r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
         if r is None:
-            return interesting_urls
+            return data
         status_code = getattr(r, "status_code", 0)
         try:
             json = r.json()
         except Exception as e:
             self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return interesting_urls
-        workspaces = []
-        for item in json.get("data", {}):
-            for workspace in item.get("document", {}).get("workspaces", []):
-                if workspace not in workspaces:
-                    workspaces.append(workspace)
-        for item in workspaces:
-            id = item.get("id", "")
-            name = item.get("name", "")
-            tldextract = self.helpers.tldextract(query)
-            if tldextract.domain.lower() in name.lower():
-                self.verbose(f"Discovered workspace {name} ({id})")
-                workspace_url = f"{self.base_url}/workspace/{id}"
-                interesting_urls.append(
-                    (
-                        workspace_url,
-                        f'{{module}} searched postman.com for "{query}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
-                    )
-                )
-                environments, collections = await self.search_workspace(id)
-                globals_url = f"{self.base_url}/workspace/{id}/globals"
-                interesting_urls.append(
-                    (
-                        globals_url,
-                        f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, and found globals at {{event.type}}: {globals_url}',
-                    )
-                )
-                for e_id in environments:
-                    env_url = f"{self.base_url}/environment/{e_id}"
-                    interesting_urls.append(
-                        (
-                            env_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated environments, and found {{event.type}}: {env_url}',
-                        )
-                    )
-                for c_id in collections:
-                    collection_url = f"{self.base_url}/collection/{c_id}"
-                    interesting_urls.append(
-                        (
-                            collection_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated collections, and found {{event.type}}: {collection_url}',
-                        )
-                    )
-                requests = await self.search_collections(id)
-                for r_id in requests:
-                    request_url = f"{self.base_url}/request/{r_id}"
-                    interesting_urls.append(
-                        (
-                            request_url,
-                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated requests, and found {{event.type}}: {request_url}',
-                        )
-                    )
-            else:
-                self.verbose(f"Skipping workspace {name} ({id}) as it does not appear to be in scope")
-        return interesting_urls
-
-    async def search_workspace(self, id):
-        url = f"{self.base_url}/workspace/{id}"
-        r = await self.helpers.request(url)
-        if r is None:
-            return [], []
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-            if not isinstance(json, dict):
-                raise ValueError(f"Got unexpected value for JSON: {json}")
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return [], []
-        environments = json.get("data", {}).get("dependencies", {}).get("environments", [])
-        collections = json.get("data", {}).get("dependencies", {}).get("collections", [])
-        return environments, collections
-
-    async def search_collections(self, id):
-        request_ids = []
-        url = f"{self.base_url}/list/collection?workspace={id}"
-        r = await self.helpers.request(url, method="POST")
-        if r is None:
-            return request_ids
-        status_code = getattr(r, "status_code", 0)
-        try:
-            json = r.json()
-        except Exception as e:
-            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-            return request_ids
-        for item in json.get("data", {}):
-            request_ids.extend(await self.parse_collection(item))
-        return request_ids
-
-    async def parse_collection(self, json):
-        request_ids = []
-        folders = json.get("folders", [])
-        requests = json.get("requests", [])
-        for folder in folders:
-            request_ids.extend(await self.parse_collection(folder))
-        for request in requests:
-            r_id = request.get("id", "")
-            request_ids.append(r_id)
-        return request_ids
+            return None
+        return json.get("data", [])

bbot/modules/postman_download.py

@@ -0,0 +1,220 @@
+import zipfile
+import json
+from pathlib import Path
+from bbot.modules.templates.postman import postman
+
+
+class postman_download(postman):
+    watched_events = ["CODE_REPOSITORY"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    meta = {
+        "description": "Download workspaces, collections, requests from Postman",
+        "created_date": "2024-09-07",
+        "author": "@domwhewell-sage",
+    }
+    options = {"output_folder": "", "api_key": ""}
+    options_desc = {"output_folder": "Folder to download postman workspaces to", "api_key": "Postman API Key"}
+    scope_distance_modifier = 2
+
+    async def setup(self):
+        output_folder = self.config.get("output_folder")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "postman_workspaces"
+        else:
+            self.output_dir = self.scan.home / "postman_workspaces"
+        self.helpers.mkdir(self.output_dir)
+        return await self.require_api_key()
+
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["X-Api-Key"] = self.api_key
+        return url, kwargs
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "postman" not in event.tags:
+                return False, "event is not a postman workspace"
+        return True
+
+    async def handle_event(self, event):
+        repo_url = event.data.get("url")
+        workspace_id = await self.get_workspace_id(repo_url)
+        if workspace_id:
+            self.verbose(f"Found workspace ID {workspace_id} for {repo_url}")
+            data = await self.request_workspace(workspace_id)
+            workspace = data["workspace"]
+            environments = data["environments"]
+            collections = data["collections"]
+            in_scope = await self.validate_workspace(workspace, environments, collections)
+            if in_scope:
+                workspace_path = self.save_workspace(workspace, environments, collections)
+                if workspace_path:
+                    self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}")
+                    codebase_event = self.make_event(
+                        {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event
+                    )
+                    await self.emit_event(
+                        codebase_event,
+                        context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}",
+                    )
+            else:
+                self.verbose(
+                    f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails, skipping download"
+                )
+
+    async def get_workspace_id(self, repo_url):
+        workspace_id = ""
+        profile = repo_url.split("/")[-2]
+        name = repo_url.split("/")[-1]
+        url = f"{self.base_url}/ws/proxy"
+        json = {
+            "service": "workspaces",
+            "method": "GET",
+            "path": f"/workspaces?handle={profile}&slug={name}",
+        }
+        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
+        if r is None:
+            return workspace_id
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace_id
+        data = json.get("data", [])
+        if len(data) == 1:
+            workspace_id = data[0]["id"]
+        return workspace_id
+
+    async def request_workspace(self, id):
+        data = {"workspace": {}, "environments": [], "collections": []}
+        workspace = await self.get_workspace(id)
+        if workspace:
+            # Main Workspace
+            name = workspace["name"]
+            data["workspace"] = workspace
+
+            # Workspace global variables
+            self.verbose(f"Downloading globals for workspace {name}")
+            globals = await self.get_globals(id)
+            data["environments"].append(globals)
+
+            # Workspace Environments
+            workspace_environments = workspace.get("environments", [])
+            if workspace_environments:
+                self.verbose(f"Downloading environments for workspace {name}")
+                for _ in workspace_environments:
+                    environment_id = _["uid"]
+                    environment = await self.get_environment(environment_id)
+                    data["environments"].append(environment)
+
+            # Workspace Collections
+            workspace_collections = workspace.get("collections", [])
+            if workspace_collections:
+                self.verbose(f"Downloading collections for workspace {name}")
+                for _ in workspace_collections:
+                    collection_id = _["uid"]
+                    collection = await self.get_collection(collection_id)
+                    data["collections"].append(collection)
+        return data
+
+    async def get_workspace(self, workspace_id):
+        workspace = {}
+        workspace_url = f"{self.api_url}/workspaces/{workspace_id}"
+        r = await self.api_request(workspace_url)
+        if r is None:
+            return workspace
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace
+        workspace = json.get("workspace", {})
+        return workspace
+
+    async def get_globals(self, workspace_id):
+        globals = {}
+        globals_url = f"{self.base_url}/workspace/{workspace_id}/globals"
+        r = await self.helpers.request(globals_url, headers=self.headers)
+        if r is None:
+            return globals
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return globals
+        globals = json.get("data", {})
+        return globals
+
+    async def get_environment(self, environment_id):
+        environment = {}
+        environment_url = f"{self.api_url}/environments/{environment_id}"
+        r = await self.api_request(environment_url)
+        if r is None:
+            return environment
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return environment
+        environment = json.get("environment", {})
+        return environment
+
+    async def get_collection(self, collection_id):
+        collection = {}
+        collection_url = f"{self.api_url}/collections/{collection_id}"
+        r = await self.api_request(collection_url)
+        if r is None:
+            return collection
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return collection
+        collection = json.get("collection", {})
+        return collection
+
+    async def validate_workspace(self, workspace, environments, collections):
+        name = workspace.get("name", "")
+        full_wks = str([workspace, environments, collections])
+        in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks)
+        if in_scope_hosts:
+            self.verbose(
+                f'Found in-scope hostname(s): "{in_scope_hosts}" in workspace {name}, it appears to be in-scope'
+            )
+            return True
+        return False
+
+    def save_workspace(self, workspace, environments, collections):
+        zip_path = None
+        # Create a folder for the workspace
+        name = workspace["name"]
+        id = workspace["id"]
+        folder = self.output_dir / name
+        self.helpers.mkdir(folder)
+        zip_path = folder / f"{id}.zip"
+
+        # Main Workspace
+        self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json")
+
+        # Workspace Environments
+        if environments:
+            for environment in environments:
+                environment_id = environment["id"]
+                self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json")
+
+            # Workspace Collections
+            if collections:
+                for collection in collections:
+                    collection_name = collection["info"]["name"]
+                    self.add_json_to_zip(zip_path, collection, f"{collection_name}.postman_collection.json")
+        return zip_path
+
+    def add_json_to_zip(self, zip_path, data, filename):
+        with zipfile.ZipFile(zip_path, "a") as zipf:
+            json_content = json.dumps(data, indent=4)
+            zipf.writestr(filename, json_content)

bbot/modules/rapiddns.py

@@ -15,14 +15,9 @@ class rapiddns(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result"
-        response = await self.request_with_fail_count(url, timeout=self.http_timeout + 10)
+        response = await self.api_request(url, timeout=self.http_timeout + 10)
         return response
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)