bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/modules/postman_download.py

@@ -0,0 +1,220 @@
+import zipfile
+import json
+from pathlib import Path
+from bbot.modules.templates.postman import postman
+
+
+class postman_download(postman):
+    watched_events = ["CODE_REPOSITORY"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    meta = {
+        "description": "Download workspaces, collections, requests from Postman",
+        "created_date": "2024-09-07",
+        "author": "@domwhewell-sage",
+    }
+    options = {"output_folder": "", "api_key": ""}
+    options_desc = {"output_folder": "Folder to download postman workspaces to", "api_key": "Postman API Key"}
+    scope_distance_modifier = 2
+
+    async def setup(self):
+        output_folder = self.config.get("output_folder")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "postman_workspaces"
+        else:
+            self.output_dir = self.scan.home / "postman_workspaces"
+        self.helpers.mkdir(self.output_dir)
+        return await self.require_api_key()
+
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["X-Api-Key"] = self.api_key
+        return url, kwargs
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "postman" not in event.tags:
+                return False, "event is not a postman workspace"
+        return True
+
+    async def handle_event(self, event):
+        repo_url = event.data.get("url")
+        workspace_id = await self.get_workspace_id(repo_url)
+        if workspace_id:
+            self.verbose(f"Found workspace ID {workspace_id} for {repo_url}")
+            data = await self.request_workspace(workspace_id)
+            workspace = data["workspace"]
+            environments = data["environments"]
+            collections = data["collections"]
+            in_scope = await self.validate_workspace(workspace, environments, collections)
+            if in_scope:
+                workspace_path = self.save_workspace(workspace, environments, collections)
+                if workspace_path:
+                    self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}")
+                    codebase_event = self.make_event(
+                        {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event
+                    )
+                    await self.emit_event(
+                        codebase_event,
+                        context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}",
+                    )
+            else:
+                self.verbose(
+                    f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails, skipping download"
+                )
+
+    async def get_workspace_id(self, repo_url):
+        workspace_id = ""
+        profile = repo_url.split("/")[-2]
+        name = repo_url.split("/")[-1]
+        url = f"{self.base_url}/ws/proxy"
+        json = {
+            "service": "workspaces",
+            "method": "GET",
+            "path": f"/workspaces?handle={profile}&slug={name}",
+        }
+        r = await self.helpers.request(url, method="POST", json=json, headers=self.headers)
+        if r is None:
+            return workspace_id
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace_id
+        data = json.get("data", [])
+        if len(data) == 1:
+            workspace_id = data[0]["id"]
+        return workspace_id
+
+    async def request_workspace(self, id):
+        data = {"workspace": {}, "environments": [], "collections": []}
+        workspace = await self.get_workspace(id)
+        if workspace:
+            # Main Workspace
+            name = workspace["name"]
+            data["workspace"] = workspace
+
+            # Workspace global variables
+            self.verbose(f"Downloading globals for workspace {name}")
+            globals = await self.get_globals(id)
+            data["environments"].append(globals)
+
+            # Workspace Environments
+            workspace_environments = workspace.get("environments", [])
+            if workspace_environments:
+                self.verbose(f"Downloading environments for workspace {name}")
+                for _ in workspace_environments:
+                    environment_id = _["uid"]
+                    environment = await self.get_environment(environment_id)
+                    data["environments"].append(environment)
+
+            # Workspace Collections
+            workspace_collections = workspace.get("collections", [])
+            if workspace_collections:
+                self.verbose(f"Downloading collections for workspace {name}")
+                for _ in workspace_collections:
+                    collection_id = _["uid"]
+                    collection = await self.get_collection(collection_id)
+                    data["collections"].append(collection)
+        return data
+
+    async def get_workspace(self, workspace_id):
+        workspace = {}
+        workspace_url = f"{self.api_url}/workspaces/{workspace_id}"
+        r = await self.api_request(workspace_url)
+        if r is None:
+            return workspace
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return workspace
+        workspace = json.get("workspace", {})
+        return workspace
+
+    async def get_globals(self, workspace_id):
+        globals = {}
+        globals_url = f"{self.base_url}/workspace/{workspace_id}/globals"
+        r = await self.helpers.request(globals_url, headers=self.headers)
+        if r is None:
+            return globals
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return globals
+        globals = json.get("data", {})
+        return globals
+
+    async def get_environment(self, environment_id):
+        environment = {}
+        environment_url = f"{self.api_url}/environments/{environment_id}"
+        r = await self.api_request(environment_url)
+        if r is None:
+            return environment
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return environment
+        environment = json.get("environment", {})
+        return environment
+
+    async def get_collection(self, collection_id):
+        collection = {}
+        collection_url = f"{self.api_url}/collections/{collection_id}"
+        r = await self.api_request(collection_url)
+        if r is None:
+            return collection
+        status_code = getattr(r, "status_code", 0)
+        try:
+            json = r.json()
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return collection
+        collection = json.get("collection", {})
+        return collection
+
+    async def validate_workspace(self, workspace, environments, collections):
+        name = workspace.get("name", "")
+        full_wks = str([workspace, environments, collections])
+        in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks)
+        if in_scope_hosts:
+            self.verbose(
+                f'Found in-scope hostname(s): "{in_scope_hosts}" in workspace {name}, it appears to be in-scope'
+            )
+            return True
+        return False
+
+    def save_workspace(self, workspace, environments, collections):
+        zip_path = None
+        # Create a folder for the workspace
+        name = workspace["name"]
+        id = workspace["id"]
+        folder = self.output_dir / name
+        self.helpers.mkdir(folder)
+        zip_path = folder / f"{id}.zip"
+
+        # Main Workspace
+        self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json")
+
+        # Workspace Environments
+        if environments:
+            for environment in environments:
+                environment_id = environment["id"]
+                self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json")
+
+            # Workspace Collections
+            if collections:
+                for collection in collections:
+                    collection_name = collection["info"]["name"]
+                    self.add_json_to_zip(zip_path, collection, f"{collection_name}.postman_collection.json")
+        return zip_path
+
+    def add_json_to_zip(self, zip_path, data, filename):
+        with zipfile.ZipFile(zip_path, "a") as zipf:
+            json_content = json.dumps(data, indent=4)
+            zipf.writestr(filename, json_content)

bbot/modules/rapiddns.py

@@ -15,14 +15,9 @@ class rapiddns(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result"
-        response = await self.request_with_fail_count(url, timeout=self.http_timeout + 10)
+        response = await self.api_request(url, timeout=self.http_timeout + 10)
         return response
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/report/asn.py

@@ -38,7 +38,7 @@ class asn(BaseReportModule):
 
     async def handle_event(self, event):
         host = event.host
-        if self.cache_get(host) == False:
+        if self.cache_get(host) is False:
             asns, source = await self.get_asn(host)
             if not asns:
                 self.cache_put(self.unknown_asn)
@@ -96,7 +96,7 @@ class asn(BaseReportModule):
         for p in self.helpers.ip_network_parents(ip):
             try:
                 self.asn_counts[p] += 1
-                if ret == False:
+                if ret is False:
                     ret = p
             except KeyError:
                 continue
@@ -112,7 +112,7 @@ class asn(BaseReportModule):
             for i, source in enumerate(list(self.sources)):
                 get_asn_fn = getattr(self, f"get_asn_{source}")
                 res = await get_asn_fn(ip)
-                if res == False:
+                if res is False:
                     # demote the current source to lowest priority since it just failed
                     self.sources.append(self.sources.pop(i))
                     self.verbose(f"Failed to contact {source}, retrying")
@@ -125,7 +125,7 @@ class asn(BaseReportModule):
         url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}"
         response = await self.get_url(url, "ASN")
         asns = []
-        if response == False:
+        if response is False:
             return False
         data = response.get("data", {})
         if not data:
@@ -138,7 +138,7 @@ class asn(BaseReportModule):
             asn_numbers = []
         for number in asn_numbers:
             asn = await self.get_asn_metadata_ripe(number)
-            if asn == False:
+            if asn is False:
                 return False
             asn["subnet"] = prefix
             asns.append(asn)
@@ -155,7 +155,7 @@ class asn(BaseReportModule):
             }
             url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
             response = await self.get_url(url, "ASN Metadata", cache=True)
-            if response == False:
+            if response is False:
                 return False
             data = response.get("data", {})
             if not data:
@@ -187,7 +187,7 @@ class asn(BaseReportModule):
         data = await self.get_url(url, "ASN")
         asns = []
         asns_tried = set()
-        if data == False:
+        if data is False:
             return False
         data = data.get("data", {})
         prefixes = data.get("prefixes", [])
@@ -201,13 +201,20 @@ class asn(BaseReportModule):
             description = details.get("description") or prefix.get("description") or ""
             country = details.get("country_code") or prefix.get("country_code") or ""
             emails = []
-            if not asn in asns_tried:
+            if asn not in asns_tried:
                 emails = await self.get_emails_bgpview(asn)
-                if emails == False:
+                if emails is False:
                     return False
                 asns_tried.add(asn)
             asns.append(
-                dict(asn=asn, subnet=subnet, name=name, description=description, country=country, emails=emails)
+                {
+                    "asn": asn,
+                    "subnet": subnet,
+                    "name": name,
+                    "description": description,
+                    "country": country,
+                    "emails": emails,
+                }
             )
         if not asns:
             self.debug(f'No results for "{ip}"')
@@ -217,7 +224,7 @@ class asn(BaseReportModule):
         contacts = []
         url = f"https://api.bgpview.io/asn/{asn}"
         data = await self.get_url(url, "ASN metadata", cache=True)
-        if data == False:
+        if data is False:
             return False
         data = data.get("data", {})
         if not data:

bbot/modules/robots.py

@@ -33,14 +33,14 @@ class robots(BaseModule):
                 for l in lines:
                     if len(l) > 0:
                         split_l = l.split(": ")
-                        if (split_l[0].lower() == "allow" and self.config.get("include_allow") == True) or (
-                            split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True
+                        if (split_l[0].lower() == "allow" and self.config.get("include_allow") is True) or (
+                            split_l[0].lower() == "disallow" and self.config.get("include_disallow") is True
                         ):
                             unverified_url = f"{host}{split_l[1].lstrip('/')}".replace(
                                 "*", self.helpers.rand_string(4)
                             )
 
-                        elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") == True:
+                        elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") is True:
                             unverified_url = split_l[1]
                         else:
                             continue

bbot/modules/securitytrails.py

@@ -15,24 +15,21 @@ class securitytrails(subdomain_enum_apikey):
     options_desc = {"api_key": "SecurityTrails API key"}
 
     base_url = "https://api.securitytrails.com/v1"
+    ping_url = f"{base_url}/ping?apikey={{api_key}}"
 
     async def setup(self):
         self.limit = 100
         return await super().setup()
 
-    async def ping(self):
-        url = f"{self.base_url}/ping?apikey={self.api_key}"
-        r = await self.request_with_fail_count(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def request_url(self, query):
-        url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}"
-        response = await self.request_with_fail_count(url)
+        url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}"
+        response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for host in j.get("subdomains", []):
-                yield f"{host}.{query}"
+                results.add(f"{host}.{query}")
+        return results

bbot/modules/securitytxt.py

@@ -121,7 +121,7 @@ class securitytxt(BaseModule):
                         start, end = match.span()
                         found_url = v[start:end]
 
-                        if found_url != url and self._urls == True:
+                        if found_url != url and self._urls is True:
                             await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags)
 
 

bbot/modules/shodan_dns.py

@@ -16,13 +16,11 @@ class shodan_dns(shodan):
 
     base_url = "https://api.shodan.io"
 
-    async def request_url(self, query):
-        url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}"
-        response = await self.request_with_fail_count(url)
-        return response
+    async def handle_event(self, event):
+        await self.handle_event_paginated(event)
 
-    def parse_results(self, r, query):
-        json = r.json()
-        if json:
-            for hostname in json.get("subdomains", []):
-                yield f"{hostname}.{query}"
+    def make_url(self, query):
+        return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
+
+    async def parse_results(self, json, query):
+        return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

bbot/modules/sitedossier.py

@@ -52,5 +52,5 @@ class sitedossier(subdomain_enum):
                     results.add(hostname)
                     yield hostname
             if '<a href="/parentdomain/' not in response.text:
-                self.debug(f"Next page not found")
+                self.debug("Next page not found")
                 break

bbot/modules/skymem.py

@@ -24,7 +24,7 @@ class skymem(emailformat):
         _, query = self.helpers.split_domain(event.data)
         # get first page
         url = f"{self.base_url}/srch?q={self.helpers.quote(query)}"
-        r = await self.request_with_fail_count(url)
+        r = await self.api_request(url)
         if not r:
             return
         responses = [r]
@@ -34,7 +34,7 @@ class skymem(emailformat):
         if domain_ids:
             domain_id = domain_ids[0]
             for page in range(2, 22):
-                r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
+                r2 = await self.api_request(f"{self.base_url}/domain/{domain_id}?p={page}")
                 if not r2:
                     continue
                 responses.append(r2)

bbot/modules/social.py

@@ -25,6 +25,7 @@ class social(BaseModule):
         "discord": (r"discord.gg/([a-zA-Z0-9_-]+)", True),
         "docker": (r"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)", False),
         "huggingface": (r"huggingface.co/([a-zA-Z0-9_-]+)", False),
+        "postman": (r"www.postman.com/([a-zA-Z0-9_-]+)", False),
     }
 
     scope_distance_modifier = 1
@@ -44,7 +45,7 @@ class social(BaseModule):
                 url = f"https://{url}"
                 event_data = {"platform": platform, "url": url, "profile_name": profile_name}
                 # only emit if the same event isn't already in the parent chain
-                if not any([e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()]):
+                if not any(e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()):
                     social_event = self.make_event(
                         event_data,
                         "SOCIAL",

bbot/modules/subdomaincenter.py

@@ -33,7 +33,7 @@ class subdomaincenter(subdomain_enum):
                 break
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

bbot/modules/subdomainradar.py

@@ -0,0 +1,160 @@
+import time
+import asyncio
+
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class SubdomainRadar(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query the Subdomain API for subdomains",
+        "created_date": "2022-07-08",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "group": "fast", "timeout": 120}
+    options_desc = {
+        "api_key": "SubDomainRadar.io API key",
+        "group": "The enumeration group to use. Choose from fast, medium, deep",
+        "timeout": "Timeout in seconds",
+    }
+
+    base_url = "https://api.subdomainradar.io"
+    ping_url = f"{base_url}/profile"
+    group_choices = ("fast", "medium", "deep")
+
+    # set this really high so the poll loop finishes as soon as possible
+    _qsize = 9999999
+
+    async def setup(self):
+        self.group = self.config.get("group", "fast").strip().lower()
+        self.timeout = self.config.get("timeout", 120)
+        if self.group not in self.group_choices:
+            return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}'
+        success, reason = await self.require_api_key()
+        if not success:
+            return success, reason
+        # convert groups to enumerators
+        enumerators = {}
+        response = await self.api_request(f"{self.base_url}/enumerators/groups")
+        status_code = getattr(response, "status_code", 0)
+        if status_code != 200:
+            return False, f"Failed to get enumerators: (HTTP status code: {status_code})"
+        else:
+            try:
+                j = response.json()
+            except Exception:
+                return False, "Failed to get enumerators: failed to parse response as JSON"
+            for group in j:
+                group_name = group.get("name", "").strip().lower()
+                if group_name:
+                    group_enumerators = []
+                    for enumerator in group.get("enumerators", []):
+                        enumerator_name = enumerator.get("display_name", "")
+                        if enumerator_name:
+                            group_enumerators.append(enumerator_name)
+                    if group_enumerators:
+                        enumerators[group_name] = group_enumerators
+
+        self.enumerators = enumerators.get(self.group, [])
+        if not self.enumerators:
+            return False, f'No enumerators found for group: "{self.group}" ({self.enumerators})'
+
+        self.enum_tasks = {}
+        self.poll_task = asyncio.create_task(self.task_poll_loop())
+
+        return True
+
+    def prepare_api_request(self, url, kwargs):
+        if self.api_key:
+            kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"}
+        return url, kwargs
+
+    async def handle_event(self, event):
+        query = self.make_query(event)
+        # start enumeration task
+        url = f"{self.base_url}/enumerate"
+        response = await self.api_request(
+            url, method="POST", json={"domains": [query], "enumerators": self.enumerators}
+        )
+        try:
+            j = response.json()
+        except Exception:
+            self.warning(f"Failed to parse response as JSON: {getattr(response, 'text', '')}")
+            return
+        task_id = j.get("tasks", {}).get(query, "")
+        if not task_id:
+            self.warning(f"Failed to start enumeration for {query}")
+            return
+        self.enum_tasks[query] = (task_id, time.time(), event)
+        self.debug(f"Started enumeration task for {query}; task id: {task_id}")
+
+    async def task_poll_loop(self):
+        # async with self._task_counter.count(f"{self.name}.task_poll_loop()"):
+        while 1:
+            for query, (task_id, start_time, event) in list(self.enum_tasks.items()):
+                url = f"{self.base_url}/tasks/{task_id}"
+                response = await self.api_request(url)
+                if getattr(response, "status_code", 0) == 200:
+                    finished = await self.parse_response(response, query, event)
+                    if finished:
+                        self.enum_tasks.pop(query)
+                        continue
+                # if scan is finishing, consider timeout
+                if self.scan.status == "FINISHING":
+                    if start_time + self.timeout < time.time():
+                        self.enum_tasks.pop(query)
+                        self.info(f"Enumeration task for {query} timed out")
+
+            if self.scan.status == "FINISHING" and not self.enum_tasks:
+                break
+            await self.helpers.sleep(5)
+
+    async def parse_response(self, response, query, event):
+        j = response.json()
+        status = j.get("status", "")
+        if status.lower() == "completed":
+            for subdomain in j.get("subdomains", []):
+                hostname = subdomain.get("subdomain", "")
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched SubDomainRadar.io API for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+            return True
+        return False
+
+    async def finish(self):
+        start_time = time.time()
+        while self.enum_tasks and not self.poll_task.done():
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= self.timeout:
+                self.warning(f"Timed out waiting for the following tasks to finish: {self.enum_tasks}")
+                for query, (task_id, _, _) in list(self.enum_tasks.items()):
+                    url = f"{self.base_url}/tasks/{task_id}"
+                    self.warning(f"    - {query} ({url})")
+                break
+
+            self.verbose(
+                f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)"
+            )
+
+            try:
+                # Wait for the task to complete or for 10 seconds, whichever comes first
+                await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10)
+            except asyncio.TimeoutError:
+                # This just means our 10-second check has elapsed, not that the task failed
+                pass
+
+        # Cancel the poll_task if it's still running
+        if not self.poll_task.done():
+            self.poll_task.cancel()
+            try:
+                await self.poll_task
+            except asyncio.CancelledError:
+                pass