bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/censys.py

@@ -15,25 +15,21 @@ class censys(subdomain_enum_apikey):
         "author": "@TheTechromancer",
         "auth_required": True,
     }
-    options = {"api_id": "", "api_secret": "", "max_pages": 5}
+    options = {"api_key": "", "max_pages": 5}
     options_desc = {
-        "api_id": "Censys.io API ID",
-        "api_secret": "Censys.io API Secret",
+        "api_key": "Censys.io API Key in the format of 'key:secret'",
         "max_pages": "Maximum number of pages to fetch (100 results per page)",
     }
 
     base_url = "https://search.censys.io/api"
 
     async def setup(self):
-        self.api_id = self.config.get("api_id", "")
-        self.api_secret = self.config.get("api_secret", "")
-        self.auth = (self.api_id, self.api_secret)
         self.max_pages = self.config.get("max_pages", 5)
         return await super().setup()
 
     async def ping(self):
         url = f"{self.base_url}/v1/account"
-        resp = await self.helpers.request(url, auth=self.auth)
+        resp = await self.api_request(url)
         d = resp.json()
         assert isinstance(d, dict), f"Invalid response from {url}: {resp}"
         quota = d.get("quota", {})
@@ -41,6 +37,11 @@ class censys(subdomain_enum_apikey):
         allowance = int(quota.get("allowance", 0))
         assert used < allowance, "No quota remaining"
 
+    def prepare_api_request(self, url, kwargs):
+        api_id, api_secret = self.api_key.split(":", 1)
+        kwargs["auth"] = (api_id, api_secret)
+        return url, kwargs
+
     async def query(self, query):
         results = set()
         cursor = ""
@@ -52,11 +53,10 @@ class censys(subdomain_enum_apikey):
             }
             if cursor:
                 json_data.update({"cursor": cursor})
-            resp = await self.helpers.request(
+            resp = await self.api_request(
                 url,
                 method="POST",
                 json=json_data,
-                auth=self.auth,
             )
 
             if resp is None:
@@ -96,7 +96,3 @@ class censys(subdomain_enum_apikey):
                     break
 
         return results
-
-    @property
-    def auth_secret(self):
-        return self.api_id and self.api_secret

bbot/modules/certspotter.py

@@ -15,11 +15,13 @@ class certspotter(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
-        return self.request_with_fail_count(url, timeout=self.http_timeout + 30)
+        return self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         json = r.json()
         if json:
             for r in json:
                 for dns_name in r.get("dns_names", []):
-                    yield dns_name.lstrip(".*").rstrip(".")
+                    results.add(dns_name.lstrip(".*").rstrip("."))
+        return results

bbot/modules/chaos.py

@@ -15,18 +15,19 @@ class chaos(subdomain_enum_apikey):
     options_desc = {"api_key": "Chaos API key"}
 
     base_url = "https://dns.projectdiscovery.io/dns"
+    ping_url = f"{base_url}/example.com"
 
-    async def ping(self):
-        url = f"{self.base_url}/example.com"
-        response = await self.request_with_fail_count(url, headers={"Authorization": self.api_key})
-        assert response.json()["domain"] == "example.com"
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["Authorization"] = self.api_key
+        return url, kwargs
 
     async def request_url(self, query):
         _, domain = self.helpers.split_domain(query)
         url = f"{self.base_url}/{domain}/subdomains"
-        return await self.request_with_fail_count(url, headers={"Authorization": self.api_key})
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):
@@ -39,4 +40,5 @@ class chaos(subdomain_enum_apikey):
                 for s in subdomains_set:
                     full_subdomain = f"{s}.{domain}"
                     if full_subdomain and full_subdomain.endswith(f".{query}"):
-                        yield full_subdomain
+                        results.add(full_subdomain)
+        return results

bbot/modules/code_repository.py

@@ -19,6 +19,7 @@ class code_repository(BaseModule):
             (r"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
         ],
         "docker": (r"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+        "postman": (r"www.postman.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
     }
 
     scope_distance_modifier = 1

bbot/modules/columbus.py

@@ -15,11 +15,11 @@ class columbus(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}?days=365"
-        return await self.request_with_fail_count(url)
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):
-            return set([f"{s.lower()}.{query}" for s in json])
+            return {f"{s.lower()}.{query}" for s in json}
         return results

bbot/modules/crt.py

@@ -21,9 +21,10 @@ class crt(subdomain_enum):
     async def request_url(self, query):
         params = {"q": f"%.{query}", "output": "json"}
         url = self.helpers.add_get_params(self.base_url, params).geturl()
-        return await self.request_with_fail_count(url, timeout=self.http_timeout + 30)
+        return await self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         for cert_info in j:
             if not type(cert_info) == dict:
@@ -35,4 +36,5 @@ class crt(subdomain_enum):
                     domain = cert_info.get("name_value")
                     if domain:
                         for d in domain.splitlines():
-                            yield d.lower()
+                            results.add(d.lower())
+        return results

bbot/modules/deadly/dastardly.py

@@ -90,7 +90,7 @@ class dastardly(BaseModule):
     def parse_dastardly_xml(self, xml_file):
         try:
             with open(xml_file, "rb") as f:
-                et = etree.parse(f, parser=etree.XMLParser(recover=True))
+                et = etree.parse(f, parser=etree.XMLParser(recover=True, resolve_entities=False))
                 for testsuite in et.iter("testsuite"):
                     yield TestSuite(testsuite)
         except FileNotFoundError:

bbot/modules/deadly/ffuf.py

@@ -10,7 +10,7 @@ class ffuf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["aggressive", "active"]
-    meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@pmueller"}
+    meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@liquidsec"}
 
     options = {
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt",
@@ -28,7 +28,7 @@ class ffuf(BaseModule):
 
     deps_common = ["ffuf"]
 
-    banned_characters = set([" "])
+    banned_characters = {" "}
     blacklist = ["images", "css", "image"]
 
     in_scope_only = True
@@ -52,7 +52,7 @@ class ffuf(BaseModule):
 
     async def handle_event(self, event):
         if self.helpers.url_depth(event.data) > self.config.get("max_depth"):
-            self.debug(f"Exceeded max depth, aborting event")
+            self.debug("Exceeded max depth, aborting event")
             return
 
         # only FFUF against a directory
@@ -122,7 +122,7 @@ class ffuf(BaseModule):
                 continue
 
             # if the codes are different, we should abort, this should also be a warning, as it is highly unusual behavior
-            if len(set(d["status"] for d in canary_results)) != 1:
+            if len({d["status"] for d in canary_results}) != 1:
                 self.warning("Got different codes for each baseline. This could indicate load balancing")
                 filters[ext] = ["ABORT", "BASELINE_CHANGED_CODES"]
                 continue
@@ -148,7 +148,7 @@ class ffuf(BaseModule):
                 continue
 
             # we start by seeing if all of the baselines have the same character count
-            if len(set(d["length"] for d in canary_results)) == 1:
+            if len({d["length"] for d in canary_results}) == 1:
                 self.debug("All baseline results had the same char count, we can make a filter on that")
                 filters[ext] = [
                     "-fc",
@@ -161,7 +161,7 @@ class ffuf(BaseModule):
                 continue
 
             # if that doesn't work we can try words
-            if len(set(d["words"] for d in canary_results)) == 1:
+            if len({d["words"] for d in canary_results}) == 1:
                 self.debug("All baseline results had the same word count, we can make a filter on that")
                 filters[ext] = [
                     "-fc",
@@ -174,7 +174,7 @@ class ffuf(BaseModule):
                 continue
 
             # as a last resort we will try lines
-            if len(set(d["lines"] for d in canary_results)) == 1:
+            if len({d["lines"] for d in canary_results}) == 1:
                 self.debug("All baseline results had the same word count, we can make a filter on that")
                 filters[ext] = [
                     "-fc",
@@ -252,7 +252,7 @@ class ffuf(BaseModule):
                         self.warning(f"Exiting from FFUF run early, received an ABORT filter: [{filters[ext][1]}]")
                         continue
 
-                    elif filters[ext] == None:
+                    elif filters[ext] is None:
                         pass
 
                     else:
@@ -282,7 +282,7 @@ class ffuf(BaseModule):
                         else:
                             if mode == "normal":
                                 # before emitting, we are going to send another baseline. This will immediately catch things like a WAF flipping blocking on us mid-scan
-                                if baseline == False:
+                                if baseline is False:
                                     pre_emit_temp_canary = [
                                         f
                                         async for f in self.execute_ffuf(

bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.0",
+        "version": "3.3.6",
         "tags": "",
         "templates": "",
         "severity": "",
@@ -226,8 +226,8 @@ class nuclei(BaseModule):
                 command.append(f"-{cli_option}")
                 command.append(option)
 
-        if self.scan.config.get("interactsh_disable") == True:
-            self.info("Disbling interactsh in accordance with global settings")
+        if self.scan.config.get("interactsh_disable") is True:
+            self.info("Disabling interactsh in accordance with global settings")
             command.append("-no-interactsh")
 
         if self.mode == "technology":

bbot/modules/deadly/vhost.py

@@ -23,6 +23,7 @@ class vhost(ffuf):
     }
 
     deps_common = ["ffuf"]
+    banned_characters = {" ", "."}
 
     in_scope_only = True
 
@@ -72,7 +73,7 @@ class vhost(ffuf):
 
     async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=False):
         filters = await self.baseline_ffuf(f"{host}/", exts=[""], suffix=basehost, mode="hostheader")
-        self.debug(f"Baseline completed and returned these filters:")
+        self.debug("Baseline completed and returned these filters:")
         self.debug(filters)
         if not wordlist:
             wordlist = self.tempfile
@@ -89,7 +90,7 @@ class vhost(ffuf):
                     parent=event,
                     context=f"{{module}} brute-forced virtual hosts for {event.data} and found {{event.type}}: {vhost_str}",
                 )
-                if skip_dns_host == False:
+                if skip_dns_host is False:
                     await self.emit_event(
                         f"{vhost_dict['vhost']}{basehost}",
                         "DNS_NAME",
@@ -103,7 +104,7 @@ class vhost(ffuf):
     def mutations_check(self, vhost):
         mutations_list = []
         for mutation in self.helpers.word_cloud.mutations(vhost):
-            for i in ["", ".", "-"]:
+            for i in ["", "-"]:
                 mutations_list.append(i.join(mutation))
         mutations_list_file = self.helpers.tempfile(mutations_list, pipe=False)
         return mutations_list_file

bbot/modules/dehashed.py

@@ -90,7 +90,7 @@ class dehashed(subdomain_enum):
         url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
         page = 0
         num_entries = 0
-        agen = self.helpers.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False)
+        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False)
         async for result in agen:
             result_json = {}
             with suppress(Exception):

bbot/modules/digitorus.py

@@ -19,7 +19,7 @@ class digitorus(subdomain_enum):
         url = f"{self.base_url}/{self.helpers.quote(query)}"
         return await self.helpers.request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         content = getattr(r, "text", "")
         extract_regex = re.compile(r"[\w.-]+\." + query, re.I)

bbot/modules/dnsbimi.py

@@ -0,0 +1,145 @@
+# bimi.py
+#
+# Checks for and parses common BIMI DNS TXT records, e.g. default._bimi.target.domain
+#
+# Example TXT record: "v=BIMI1; l=https://example.com/brand/logo.svg; a=https://example.com/brand/certificate.pem"
+#
+# BIMI records may contain a link to an SVG format brand authorised image, which may be useful for:
+#  1. Sub-domain or otherwise unknown content hosting locations
+#  2. Brand impersonation
+#  3. May not be formatted/stripped of metadata correctly leading to some (low value probably) information exposure
+#
+# BIMI records may also contain a link to a PEM format X.509 VMC certificate, which may be similarly useful.
+#
+# We simply extract any URL's as URL_UNVERIFIED, no further parsing or download is done by this module in order to remain passive.
+#
+# The domain portion of any URL's is also passively checked and added as appropriate, for additional inspection by other modules.
+#
+# Files may be downloaded by other modules which respond to URL_UNVERIFIED events, if you have configured bbot to do so.
+#
+# NOTE: .svg file extensions are filtered from inclusion by default, modify "url_extension_blacklist" appropriately if you want the .svg image to be considered for download.
+#
+# NOTE: use the "filedownload" module if you to download .svg and .pem files. .pem will be downloaded by default, .svg will require a customised configuration for that module.
+#
+# The domain portion of any URL_UNVERIFIED's will be extracted by the various internal modules if .svg is not filtered.
+#
+
+from bbot.modules.base import BaseModule
+from bbot.core.helpers.dns.helpers import service_record
+
+import re
+
+# Handle "v=BIMI1; l=; a=;" == RFC conformant explicit declination to publish, e.g. useful on a sub-domain if you don't want the sub-domain to have a BIMI logo, yet your registered domain does?
+# Handle "v=BIMI1; l=; a=" == RFC non-conformant explicit declination to publish
+# Handle "v=BIMI1; l=;" == RFC non-conformant explicit declination to publish
+# Handle "v=BIMI1; l=" == RFC non-conformant explicit declination to publish
+# Handle "v=BIMI1;" == RFC non-conformant explicit declination to publish
+# Handle "v=BIMI1" == RFC non-conformant explicit declination to publish
+# Handle "v=BIMI1;l=https://bimi.entrust.net/example.com/logo.svg;"
+# Handle "v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;"
+# Handle "v=BIMI1;l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem"
+# Handle "v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem;"
+_bimi_regex = r"^v=(?P<v>BIMI1);* *(l=(?P<l>https*://[^;]*|)|);*( *a=((?P<a>https://[^;]*|)|);*)*$"
+bimi_regex = re.compile(_bimi_regex, re.I)
+
+
+class dnsbimi(BaseModule):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["URL_UNVERIFIED", "RAW_DNS_RECORD"]
+    flags = ["subdomain-enum", "cloud-enum", "passive", "safe"]
+    meta = {
+        "description": "Check DNS_NAME's for BIMI records to find image and certificate hosting URL's",
+        "author": "@colin-stubbs",
+        "created_date": "2024-11-15",
+    }
+    options = {
+        "emit_raw_dns_records": False,
+        "emit_urls": True,
+        "selectors": "default,email,mail,bimi",
+    }
+    options_desc = {
+        "emit_raw_dns_records": "Emit RAW_DNS_RECORD events",
+        "emit_urls": "Emit URL_UNVERIFIED events",
+        "selectors": "CSV list of BIMI selectors to check",
+    }
+
+    async def setup(self):
+        self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False)
+        self.emit_urls = self.config.get("emit_urls", True)
+        self._selectors = self.config.get("selectors", "").replace(", ", ",").split(",")
+
+        return await super().setup()
+
+    def _incoming_dedup_hash(self, event):
+        # dedupe by parent
+        parent_domain = self.helpers.parent_domain(event.data)
+        return hash(parent_domain), "already processed parent domain"
+
+    async def filter_event(self, event):
+        if "_wildcard" in str(event.host).split("."):
+            return False, "event is wildcard"
+
+        # there's no value in inspecting service records
+        if service_record(event.host) is True:
+            return False, "service record detected"
+
+        return True
+
+    async def inspectBIMI(self, event, domain):
+        parent_domain = self.helpers.parent_domain(event.data)
+        rdtype = "TXT"
+
+        for selector in self._selectors:
+            tags = ["bimi-record", f"bimi-{selector}"]
+            hostname = f"{selector}._bimi.{parent_domain}"
+
+            r = await self.helpers.resolve_raw(hostname, type=rdtype)
+
+            if r:
+                raw_results, errors = r
+
+                for answer in raw_results:
+                    if self.emit_raw_dns_records:
+                        await self.emit_event(
+                            {
+                                "host": hostname,
+                                "type": rdtype,
+                                "answer": answer.to_text(),
+                            },
+                            "RAW_DNS_RECORD",
+                            parent=event,
+                            tags=tags.append(f"{rdtype.lower()}-record"),
+                            context=f"{rdtype} lookup on {hostname} produced {{event.type}}",
+                        )
+
+                    # we need to strip surrounding quotes and whitespace, as well as fix TXT data that may have been split across two different rdata's
+                    # e.g. we will get a single string, but within that string we may have two parts such as:
+                    # answer = '"part 1 that was really long" "part 2 that did not fit in part 1"'
+                    s = answer.to_text().strip('"').strip().replace('" "', "")
+
+                    bimi_match = bimi_regex.search(s)
+
+                    if bimi_match and bimi_match.group("v") and "bimi" in bimi_match.group("v").lower():
+                        if bimi_match.group("l") and bimi_match.group("l") != "":
+                            if self.emit_urls:
+                                await self.emit_event(
+                                    bimi_match.group("l"),
+                                    "URL_UNVERIFIED",
+                                    parent=event,
+                                    tags=tags.append("bimi-location"),
+                                )
+
+                        if bimi_match.group("a") and bimi_match.group("a") != "":
+                            if self.emit_urls:
+                                await self.emit_event(
+                                    bimi_match.group("a"),
+                                    "URL_UNVERIFIED",
+                                    parent=event,
+                                    tags=tags.append("bimi-authority"),
+                                )
+
+    async def handle_event(self, event):
+        await self.inspectBIMI(event, event.host)
+
+
+# EOF

bbot/modules/dnscaa.py

@@ -2,7 +2,7 @@
 #
 # Checks for and parses CAA DNS TXT records for IODEF reporting destination email addresses and/or URL's.
 #
-# NOTE: when the target domain is initially resolved basic "dns_name_regex" matched targets will be extracted so we do not perform that again here.
+# NOTE: when the target domain is initially resolved basic "dns_name_extraction_regex" matched targets will be extracted so we do not perform that again here.
 #
 # Example CAA records,
 #   0 iodef "mailto:dnsadmin@example.com"
@@ -23,7 +23,7 @@ from bbot.modules.base import BaseModule
 
 import re
 
-from bbot.core.helpers.regexes import dns_name_regex, email_regex, url_regexes
+from bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes
 
 # Handle '0 iodef "mailto:support@hcaptcha.com"'
 # Handle '1 iodef "https://some.host.tld/caa;"'
@@ -109,7 +109,7 @@ class dnscaa(BaseModule):
 
                     elif caa_match.group("property").lower().startswith("issue"):
                         if self._dns_names:
-                            for match in dns_name_regex.finditer(caa_match.group("text")):
+                            for match in dns_name_extraction_regex.finditer(caa_match.group("text")):
                                 start, end = match.span()
                                 name = caa_match.group("text")[start:end]
 

bbot/modules/dnsdumpster.py

@@ -18,7 +18,7 @@ class dnsdumpster(subdomain_enum):
     async def query(self, domain):
         ret = []
         # first, get the CSRF tokens
-        res1 = await self.request_with_fail_count(self.base_url)
+        res1 = await self.api_request(self.base_url)
         status_code = getattr(res1, "status_code", 0)
         if status_code in [429]:
             self.verbose(f'Too many requests "{status_code}"')
@@ -31,7 +31,7 @@ class dnsdumpster(subdomain_enum):
 
         html = self.helpers.beautifulsoup(res1.content, "html.parser")
         if html is False:
-            self.verbose(f"BeautifulSoup returned False")
+            self.verbose("BeautifulSoup returned False")
             return ret
 
         csrftoken = None
@@ -62,7 +62,7 @@ class dnsdumpster(subdomain_enum):
 
         # Otherwise, do the needful
         subdomains = set()
-        res2 = await self.request_with_fail_count(
+        res2 = await self.api_request(
             f"{self.base_url}/",
             method="POST",
             cookies={"csrftoken": csrftoken},
@@ -82,7 +82,7 @@ class dnsdumpster(subdomain_enum):
             return ret
         html = self.helpers.beautifulsoup(res2.content, "html.parser")
         if html is False:
-            self.verbose(f"BeautifulSoup returned False")
+            self.verbose("BeautifulSoup returned False")
             return ret
         escaped_domain = re.escape(domain)
         match_pattern = re.compile(r"^[\w\.-]+\." + escaped_domain + r"$")