bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/modules/telerik.py

@@ -173,8 +173,8 @@ class telerik(BaseModule):
             webresource = "Telerik.Web.UI.WebResource.axd?type=rau"
             result, _ = await self.test_detector(event.data, webresource)
             if result:
-                if "RadAsyncUpload handler is registered succesfully" in result.text:
-                    self.debug(f"Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
+                if "RadAsyncUpload handler is registered successfully" in result.text:
+                    self.debug("Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
 
                     probe_data = {
                         "rauPostData": (
@@ -216,7 +216,7 @@ class telerik(BaseModule):
                         event,
                         context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik RAU AXD Handler",
                     )
-                    if self.config.get("exploit_RAU_crypto") == True:
+                    if self.config.get("exploit_RAU_crypto") is True:
                         hostname = urlparse(event.data).netloc
                         if hostname not in self.RAUConfirmed:
                             self.RAUConfirmed.append(hostname)
@@ -270,7 +270,7 @@ class telerik(BaseModule):
                 else:
                     if "Cannot deserialize dialog parameters" in response.text:
                         self.debug(f"Detected Telerik UI instance ({dh})")
-                        description = f"Telerik DialogHandler detected"
+                        description = "Telerik DialogHandler detected"
                         await self.emit_event(
                             {"host": str(event.host), "url": f"{event.data}{dh}", "description": description},
                             "FINDING",
@@ -289,8 +289,8 @@ class telerik(BaseModule):
                 self.debug(validate_result)
                 validate_status_code = getattr(validate_result, "status_code", 0)
                 if validate_status_code not in (0, 500):
-                    self.debug(f"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
-                    description = f"Telerik SpellCheckHandler detected"
+                    self.debug("Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
+                    description = "Telerik SpellCheckHandler detected"
                     await self.emit_event(
                         {
                             "host": str(event.host),
@@ -334,7 +334,7 @@ class telerik(BaseModule):
                         },
                         "FINDING",
                         event,
-                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik ChartImage AXD Handler",
+                        context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik ChartImage AXD Handler",
                     )
                 elif '"_serializedConfiguration":"' in resp_body:
                     await self.emit_event(
@@ -345,7 +345,7 @@ class telerik(BaseModule):
                         },
                         "FINDING",
                         event,
-                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik AsyncUpload",
+                        context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik AsyncUpload",
                     )
 
         # Check for RAD Controls in URL

bbot/modules/templates/bucket.py

@@ -159,7 +159,7 @@ class bucket_template(BaseModule):
         valid = self.cloud_helper.is_valid_bucket_name(bucket_name)
         if valid and not self.helpers.is_ip(bucket_name):
             bucket_hash = hash(bucket_name)
-            if not bucket_hash in self.buckets_tried:
+            if bucket_hash not in self.buckets_tried:
                 self.buckets_tried.add(bucket_hash)
                 return True
         return False

bbot/modules/templates/github.py

@@ -1,3 +1,5 @@
+import traceback
+
 from bbot.modules.base import BaseModule
 
 
@@ -9,30 +11,36 @@ class github(BaseModule):
 
     _qsize = 1
     base_url = "https://api.github.com"
+    ping_url = f"{base_url}/zen"
+
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["Authorization"] = f"token {self.api_key}"
+        return url, kwargs
 
     async def setup(self):
         await super().setup()
-        self.api_key = None
         self.headers = {}
-        for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
-            module_config = self.scan.config.get("modules", {}).get(module_name, {})
+        api_keys = set()
+        modules_config = self.scan.config.get("modules", {})
+        git_modules = [m for m in modules_config if str(m).startswith("git")]
+        for module_name in git_modules:
+            module_config = modules_config.get(module_name, {})
             api_key = module_config.get("api_key", "")
-            if api_key:
-                self.api_key = api_key
-                self.headers = {"Authorization": f"token {self.api_key}"}
-                break
-        if not self.api_key:
+            if isinstance(api_key, str):
+                api_key = [api_key]
+            for key in api_key:
+                key = key.strip()
+                if key:
+                    api_keys.add(key)
+        if not api_keys:
             if self.auth_required:
                 return None, "No API key set"
+        self.api_key = api_keys
         try:
             await self.ping()
-            self.hugesuccess(f"API is ready")
+            self.hugesuccess("API is ready")
             return True
         except Exception as e:
+            self.trace(traceback.format_exc())
             return None, f"Error with API ({str(e).strip()})"
         return True
-
-    async def ping(self):
-        url = f"{self.base_url}/zen"
-        response = await self.helpers.request(url, headers=self.headers)
-        assert getattr(response, "status_code", 0) == 200, response.text

bbot/modules/templates/postman.py

@@ -0,0 +1,21 @@
+from bbot.modules.base import BaseModule
+
+
+class postman(BaseModule):
+    """
+    A template module for use of the GitHub API
+    Inherited by several other github modules.
+    """
+
+    base_url = "https://www.postman.com/_api"
+    api_url = "https://api.getpostman.com"
+    html_url = "https://www.postman.com"
+    ping_url = f"{api_url}/me"
+
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Version": "10.18.8-230926-0808",
+        "X-Entity-Team-Id": "0",
+        "Origin": "https://www.postman.com",
+        "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
+    }

bbot/modules/templates/shodan.py

@@ -1,3 +1,5 @@
+import traceback
+
 from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
@@ -6,29 +8,28 @@ class shodan(subdomain_enum):
     options_desc = {"api_key": "Shodan API key"}
 
     base_url = "https://api.shodan.io"
+    ping_url = f"{base_url}/api-info?key={{api_key}}"
 
     async def setup(self):
         await super().setup()
-        self.api_key = None
+        api_keys = set()
         for module_name in ("shodan", "shodan_dns", "shodan_port"):
             module_config = self.scan.config.get("modules", {}).get(module_name, {})
             api_key = module_config.get("api_key", "")
-            if api_key:
-                self.api_key = api_key
-                break
-        if not self.api_key:
+            if isinstance(api_key, str):
+                api_key = [api_key]
+            for key in api_key:
+                key = key.strip()
+                if key:
+                    api_keys.add(key)
+        if not api_keys:
             if self.auth_required:
                 return None, "No API key set"
+        self.api_key = api_keys
         try:
             await self.ping()
-            self.hugesuccess(f"API is ready")
+            self.hugesuccess("API is ready")
             return True
         except Exception as e:
+            self.trace(traceback.format_exc())
             return None, f"Error with API ({str(e).strip()})"
-        return True
-
-    async def ping(self):
-        url = f"{self.base_url}/api-info?key={self.api_key}"
-        r = await self.request_with_fail_count(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content

bbot/modules/templates/sql.py

@@ -0,0 +1,95 @@
+from contextlib import suppress
+from sqlmodel import SQLModel
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+
+from bbot.db.sql.models import Event, Scan, Target
+from bbot.modules.output.base import BaseOutputModule
+
+
+class SQLTemplate(BaseOutputModule):
+    meta = {"description": "SQL output module template"}
+    options = {
+        "database": "bbot",
+        "username": "",
+        "password": "",
+        "host": "127.0.0.1",
+        "port": 0,
+    }
+    options_desc = {
+        "database": "The database to use",
+        "username": "The username to use to connect to the database",
+        "password": "The password to use to connect to the database",
+        "host": "The host to use to connect to the database",
+        "port": "The port to use to connect to the database",
+    }
+
+    protocol = ""
+
+    async def setup(self):
+        self.database = self.config.get("database", "bbot")
+        self.username = self.config.get("username", "")
+        self.password = self.config.get("password", "")
+        self.host = self.config.get("host", "127.0.0.1")
+        self.port = self.config.get("port", 0)
+
+        await self.init_database()
+        return True
+
+    async def handle_event(self, event):
+        event_obj = Event(**event.json()).validated
+
+        async with self.async_session() as session:
+            async with session.begin():
+                # insert event
+                session.add(event_obj)
+
+                # if it's a SCAN event, create/update the scan and target
+                if event_obj.type == "SCAN":
+                    event_data = event_obj.get_data()
+                    if not isinstance(event_data, dict):
+                        raise ValueError(f"Invalid data for SCAN event: {event_data}")
+                    scan = Scan(**event_data).validated
+                    await session.merge(scan)  # Insert or update scan
+
+                    target_data = event_data.get("target", {})
+                    if not isinstance(target_data, dict):
+                        raise ValueError(f"Invalid target for SCAN event: {target_data}")
+                    target = Target(**target_data).validated
+                    await session.merge(target)  # Insert or update target
+
+                await session.commit()
+
+    async def create_database(self):
+        pass
+
+    async def init_database(self):
+        await self.create_database()
+
+        # Now create the engine for the actual database
+        self.engine = create_async_engine(self.connection_string())
+        # Create a session factory bound to the engine
+        self.async_session = sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)
+
+        # Use the engine directly to create all tables
+        async with self.engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+    def connection_string(self, mask_password=False):
+        connection_string = f"{self.protocol}://"
+        if self.username:
+            password = self.password
+            if mask_password:
+                password = "****"
+            connection_string += f"{self.username}:{password}"
+        if self.host:
+            connection_string += f"@{self.host}"
+            if self.port:
+                connection_string += f":{self.port}"
+        if self.database:
+            connection_string += f"/{self.database}"
+        return connection_string
+
+    async def cleanup(self):
+        with suppress(Exception):
+            await self.engine.dispose()

bbot/modules/templates/subdomain_enum.py

@@ -20,8 +20,8 @@ class subdomain_enum(BaseModule):
     # whether to reject wildcard DNS_NAMEs
     reject_wildcards = "strict"
 
-    # set qsize to 10. this helps combat rate limiting by ensuring that a query doesn't execute
-    # until the queue is ready to receive its results
+    # set qsize to 10. this helps combat rate limiting by ensuring the next query doesn't execute
+    # until the result from the previous queue have been consumed by the scan
     # we don't use 1 because it causes delays due to the asyncio.sleep; 10 gives us reasonable buffer room
     _qsize = 10
 
@@ -31,6 +31,11 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    # how many results to request per API call
+    page_size = 100
+    # arguments to pass to api_page_iter
+    api_page_iter_kwargs = {}
+
     @property
     def source_pretty_name(self):
         return f"{self.__class__.__name__} API"
@@ -61,9 +66,30 @@ class subdomain_enum(BaseModule):
                             context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
                         )
 
+    async def handle_event_paginated(self, event):
+        query = self.make_query(event)
+        async for result_batch in self.query_paginated(query):
+            for hostname in set(result_batch):
+                try:
+                    hostname = self.helpers.validators.validate_host(hostname)
+                except ValueError as e:
+                    self.verbose(e)
+                    continue
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+
     async def request_url(self, query):
-        url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url)
+        url = self.make_url(query)
+        return await self.api_request(url)
+
+    def make_url(self, query):
+        return f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
 
     def make_query(self, event):
         query = event.data
@@ -78,30 +104,26 @@ class subdomain_enum(BaseModule):
             if self.scan.in_scope(p):
                 query = p
                 break
-        try:
-            return ".".join([s for s in query.split(".") if s != "_wildcard"])
-        except Exception:
-            self.critical(query)
-            raise
+        return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for hostname in json:
                 yield hostname
 
-    async def query(self, query, parse_fn=None, request_fn=None):
-        if parse_fn is None:
-            parse_fn = self.parse_results
+    async def query(self, query, request_fn=None, parse_fn=None):
         if request_fn is None:
             request_fn = self.request_url
+        if parse_fn is None:
+            parse_fn = self.parse_results
         try:
             response = await request_fn(query)
             if response is None:
                 self.info(f'Query "{query}" failed (no response)')
                 return []
             try:
-                results = list(parse_fn(response, query))
+                results = list(await parse_fn(response, query))
             except Exception as e:
                 if response:
                     self.info(
@@ -117,10 +139,23 @@ class subdomain_enum(BaseModule):
         except Exception as e:
             self.info(f"Error retrieving results for {query}: {e}", trace=True)
 
+    async def query_paginated(self, query):
+        url = self.make_url(query)
+        agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
+        try:
+            async for response in agen:
+                subdomains = await self.parse_results(response, query)
+                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
+                if not subdomains:
+                    break
+                yield subdomains
+        finally:
+            agen.aclose()
+
     async def _is_wildcard(self, query):
         rdtypes = ("A", "AAAA", "CNAME")
         if self.helpers.is_dns_name(query):
-            for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).items():
+            for wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).values():
                 if any(t in wildcard_rdtypes for t in rdtypes):
                     return True
         return False
@@ -134,7 +169,7 @@ class subdomain_enum(BaseModule):
         if any(t.startswith("cloud-") for t in event.tags):
             is_cloud = True
         # reject if it's a cloud resource and not in our target
-        if is_cloud and event not in self.scan.target:
+        if is_cloud and event not in self.scan.target.whitelist:
             return False, "Event is a cloud resource and not a direct target"
         # optionally reject events with wildcards / errors
         if self.reject_wildcards:

bbot/modules/templates/webhook.py

@@ -9,7 +9,6 @@ class WebhookOutputModule(BaseOutputModule):
     """
 
     accept_dupes = False
-    good_status_code = 204
     message_size_limit = 2000
     content_key = "content"
     vuln_severities = ["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CRITICAL"]
@@ -61,7 +60,7 @@ class WebhookOutputModule(BaseOutputModule):
     async def filter_event(self, event):
         if event.type == "VULNERABILITY":
             severity = event.data.get("severity", "UNKNOWN")
-            if not severity in self.allowed_severities:
+            if severity not in self.allowed_severities:
                 return False, f"{severity} is below min_severity threshold"
         return True
 
@@ -94,5 +93,4 @@ class WebhookOutputModule(BaseOutputModule):
         return msg
 
     def evaluate_response(self, response):
-        status_code = getattr(response, "status_code", 0)
-        return status_code == self.good_status_code
+        return getattr(response, "is_success", False)

bbot/modules/trickest.py

@@ -19,53 +19,24 @@ class Trickest(subdomain_enum_apikey):
     }
 
     base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be"
+    ping_url = f"{base_url}/dataset"
     dataset_id = "a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc"
     page_size = 50
 
-    async def ping(self):
-        self.headers = {"Authorization": f"Token {self.api_key}"}
-        url = f"{self.base_url}/dataset"
-        response = await self.helpers.request(url, headers=self.headers)
-        status_code = getattr(response, "status_code", 0)
-        if status_code != 200:
-            response_text = getattr(response, "text", "no response from server")
-            return False, response_text
-        return True
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["Authorization"] = f"Token {self.api_key}"
+        return url, kwargs
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        async for result_batch in self.query(query):
-            for hostname in set(result_batch):
-                try:
-                    hostname = self.helpers.validators.validate_host(hostname)
-                except ValueError as e:
-                    self.verbose(e)
-                    continue
-                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                    await self.emit_event(
-                        hostname,
-                        "DNS_NAME",
-                        event,
-                        abort_if=self.abort_if,
-                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
-                    )
+        await self.handle_event_paginated(event)
 
-    async def query(self, query):
+    def make_url(self, query):
         url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22"
         url += f"&dataset_id={self.dataset_id}"
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, page_size=self.page_size)
-        try:
-            async for response in agen:
-                subdomains = self.parse_results(response)
-                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
-                if not subdomains:
-                    break
-                yield subdomains
-        finally:
-            agen.aclose()
+        return url
 
-    def parse_results(self, j):
+    async def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

bbot/modules/trufflehog.py

@@ -1,5 +1,4 @@
 import json
-from pathlib import Path
 from bbot.modules.base import BaseModule
 
 
@@ -14,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.81.10",
+        "version": "3.84.1",
         "config": "",
         "only_verified": True,
         "concurrency": 8,
@@ -31,7 +30,7 @@ class trufflehog(BaseModule):
         {
             "name": "Download trufflehog",
             "unarchive": {
-                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
+                "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
                 "include": "trufflehog",
                 "dest": "#{BBOT_TOOLS}",
                 "remote_src": True,
@@ -52,7 +51,7 @@ class trufflehog(BaseModule):
         self.github_token = ""
         if self.deleted_forks:
             self.warning(
-                f"Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
+                "Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours."
             )
             for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
                 module_config = self.scan.config.get("modules", {}).get(module_name, {})
@@ -65,7 +64,6 @@ class trufflehog(BaseModule):
             if not self.github_token:
                 self.deleted_forks = False
                 return None, "A github api_key must be provided to the github modules for deleted forks to be scanned"
-        self.processed = set()
         return True
 
     async def filter_event(self, event):
@@ -78,12 +76,8 @@ class trufflehog(BaseModule):
             else:
                 return False, "Deleted forks is not enabled"
         else:
-            path = event.data["path"]
-            for processed in self.processed:
-                processed_path = Path(processed)
-                new_path = Path(path)
-                if new_path.is_relative_to(processed_path):
-                    return False, "Parent folder has already been processed"
+            if "parsed-folder" in event.tags:
+                return False, "Not accepting parsed-folder events"
         return True
 
     async def handle_event(self, event):
@@ -94,11 +88,12 @@ class trufflehog(BaseModule):
                 module = "github-experimental"
         else:
             path = event.data["path"]
-            self.processed.add(path)
             if "git" in event.tags:
                 module = "git"
             elif "docker" in event.tags:
                 module = "docker"
+            elif "postman" in event.tags:
+                module = "postman"
             else:
                 module = "filesystem"
         if event.type == "CODE_REPOSITORY":
@@ -164,6 +159,9 @@ class trufflehog(BaseModule):
         elif module == "docker":
             command.append("docker")
             command.append("--image=file://" + path)
+        elif module == "postman":
+            command.append("postman")
+            command.append("--workspace-paths=" + path)
         elif module == "filesystem":
             command.append("filesystem")
             command.append(path)

bbot/modules/url_manipulation.py

@@ -69,11 +69,11 @@ class url_manipulation(BaseModule):
 
             if subject_response:
                 subject_content = "".join([str(x) for x in subject_response.headers])
-                if subject_response.text != None:
+                if subject_response.text is not None:
                     subject_content += subject_response.text
 
                 if self.rand_string not in subject_content:
-                    if match == False:
+                    if match is False:
                         if str(subject_response.status_code).startswith("2"):
                             if "body" in reasons:
                                 reported_signature = f"Modified URL: {sig[1]}"
@@ -98,7 +98,7 @@ class url_manipulation(BaseModule):
         return False
 
     def format_signature(self, sig, event):
-        if sig[2] == True:
+        if sig[2] is True:
             cleaned_path = event.parsed_url.path.strip("/")
         else:
             cleaned_path = event.parsed_url.path.lstrip("/")

bbot/modules/urlscan.py

@@ -78,5 +78,5 @@ class urlscan(subdomain_enum):
             else:
                 self.debug(f'No results for "{query}"')
         except Exception:
-            self.verbose(f"Error retrieving urlscan results")
+            self.verbose("Error retrieving urlscan results")
         return results

bbot/modules/viewdns.py

@@ -48,7 +48,7 @@ class viewdns(BaseModule):
 
         html = self.helpers.beautifulsoup(content, "html.parser")
         if html is False:
-            self.debug(f"BeautifulSoup returned False")
+            self.debug("BeautifulSoup returned False")
             return results
         found = set()
         for table_row in html.findAll("tr"):