bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/report/asn.py

@@ -38,7 +38,7 @@ class asn(BaseReportModule):
 
     async def handle_event(self, event):
         host = event.host
-        if self.cache_get(host) == False:
+        if self.cache_get(host) is False:
             asns, source = await self.get_asn(host)
             if not asns:
                 self.cache_put(self.unknown_asn)
@@ -96,7 +96,7 @@ class asn(BaseReportModule):
         for p in self.helpers.ip_network_parents(ip):
             try:
                 self.asn_counts[p] += 1
-                if ret == False:
+                if ret is False:
                     ret = p
             except KeyError:
                 continue
@@ -112,7 +112,7 @@ class asn(BaseReportModule):
             for i, source in enumerate(list(self.sources)):
                 get_asn_fn = getattr(self, f"get_asn_{source}")
                 res = await get_asn_fn(ip)
-                if res == False:
+                if res is False:
                     # demote the current source to lowest priority since it just failed
                     self.sources.append(self.sources.pop(i))
                     self.verbose(f"Failed to contact {source}, retrying")
@@ -125,7 +125,7 @@ class asn(BaseReportModule):
         url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}"
         response = await self.get_url(url, "ASN")
         asns = []
-        if response == False:
+        if response is False:
             return False
         data = response.get("data", {})
         if not data:
@@ -138,7 +138,7 @@ class asn(BaseReportModule):
             asn_numbers = []
         for number in asn_numbers:
             asn = await self.get_asn_metadata_ripe(number)
-            if asn == False:
+            if asn is False:
                 return False
             asn["subnet"] = prefix
             asns.append(asn)
@@ -155,7 +155,7 @@ class asn(BaseReportModule):
             }
             url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}"
             response = await self.get_url(url, "ASN Metadata", cache=True)
-            if response == False:
+            if response is False:
                 return False
             data = response.get("data", {})
             if not data:
@@ -187,7 +187,7 @@ class asn(BaseReportModule):
         data = await self.get_url(url, "ASN")
         asns = []
         asns_tried = set()
-        if data == False:
+        if data is False:
             return False
         data = data.get("data", {})
         prefixes = data.get("prefixes", [])
@@ -201,13 +201,13 @@ class asn(BaseReportModule):
             description = details.get("description") or prefix.get("description") or ""
             country = details.get("country_code") or prefix.get("country_code") or ""
             emails = []
-            if not asn in asns_tried:
+            if asn not in asns_tried:
                 emails = await self.get_emails_bgpview(asn)
-                if emails == False:
+                if emails is False:
                     return False
                 asns_tried.add(asn)
             asns.append(
-                dict(asn=asn, subnet=subnet, name=name, description=description, country=country, emails=emails)
+                {"asn": asn, "subnet": subnet, "name": name, "description": description, "country": country, "emails": emails}
             )
         if not asns:
             self.debug(f'No results for "{ip}"')
@@ -217,7 +217,7 @@ class asn(BaseReportModule):
         contacts = []
         url = f"https://api.bgpview.io/asn/{asn}"
         data = await self.get_url(url, "ASN metadata", cache=True)
-        if data == False:
+        if data is False:
             return False
         data = data.get("data", {})
         if not data:

bbot/modules/robots.py

@@ -33,14 +33,14 @@ class robots(BaseModule):
                 for l in lines:
                     if len(l) > 0:
                         split_l = l.split(": ")
-                        if (split_l[0].lower() == "allow" and self.config.get("include_allow") == True) or (
-                            split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True
+                        if (split_l[0].lower() == "allow" and self.config.get("include_allow") is True) or (
+                            split_l[0].lower() == "disallow" and self.config.get("include_disallow") is True
                         ):
                             unverified_url = f"{host}{split_l[1].lstrip('/')}".replace(
                                 "*", self.helpers.rand_string(4)
                             )
 
-                        elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") == True:
+                        elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") is True:
                             unverified_url = split_l[1]
                         else:
                             continue

bbot/modules/securitytrails.py

@@ -15,24 +15,21 @@ class securitytrails(subdomain_enum_apikey):
     options_desc = {"api_key": "SecurityTrails API key"}
 
     base_url = "https://api.securitytrails.com/v1"
+    ping_url = f"{base_url}/ping?apikey={{api_key}}"
 
     async def setup(self):
         self.limit = 100
         return await super().setup()
 
-    async def ping(self):
-        url = f"{self.base_url}/ping?apikey={self.api_key}"
-        r = await self.request_with_fail_count(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def request_url(self, query):
-        url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}"
-        response = await self.request_with_fail_count(url)
+        url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}"
+        response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for host in j.get("subdomains", []):
-                yield f"{host}.{query}"
+                results.add(f"{host}.{query}")
+        return results

bbot/modules/securitytxt.py

@@ -121,7 +121,7 @@ class securitytxt(BaseModule):
                         start, end = match.span()
                         found_url = v[start:end]
 
-                        if found_url != url and self._urls == True:
+                        if found_url != url and self._urls is True:
                             await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags)
 
 

bbot/modules/shodan_dns.py

@@ -16,13 +16,11 @@ class shodan_dns(shodan):
 
     base_url = "https://api.shodan.io"
 
-    async def request_url(self, query):
-        url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}"
-        response = await self.request_with_fail_count(url)
-        return response
+    async def handle_event(self, event):
+        await self.handle_event_paginated(event)
 
-    def parse_results(self, r, query):
-        json = r.json()
-        if json:
-            for hostname in json.get("subdomains", []):
-                yield f"{hostname}.{query}"
+    def make_url(self, query):
+        return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
+
+    async def parse_results(self, json, query):
+        return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

bbot/modules/sitedossier.py

@@ -52,5 +52,5 @@ class sitedossier(subdomain_enum):
                     results.add(hostname)
                     yield hostname
             if '<a href="/parentdomain/' not in response.text:
-                self.debug(f"Next page not found")
+                self.debug("Next page not found")
                 break

bbot/modules/skymem.py

@@ -24,7 +24,7 @@ class skymem(emailformat):
         _, query = self.helpers.split_domain(event.data)
         # get first page
         url = f"{self.base_url}/srch?q={self.helpers.quote(query)}"
-        r = await self.request_with_fail_count(url)
+        r = await self.api_request(url)
         if not r:
             return
         responses = [r]
@@ -34,7 +34,7 @@ class skymem(emailformat):
         if domain_ids:
             domain_id = domain_ids[0]
             for page in range(2, 22):
-                r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
+                r2 = await self.api_request(f"{self.base_url}/domain/{domain_id}?p={page}")
                 if not r2:
                     continue
                 responses.append(r2)

bbot/modules/social.py

@@ -25,6 +25,7 @@ class social(BaseModule):
         "discord": (r"discord.gg/([a-zA-Z0-9_-]+)", True),
         "docker": (r"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)", False),
         "huggingface": (r"huggingface.co/([a-zA-Z0-9_-]+)", False),
+        "postman": (r"www.postman.com/([a-zA-Z0-9_-]+)", False),
     }
 
     scope_distance_modifier = 1
@@ -44,7 +45,7 @@ class social(BaseModule):
                 url = f"https://{url}"
                 event_data = {"platform": platform, "url": url, "profile_name": profile_name}
                 # only emit if the same event isn't already in the parent chain
-                if not any([e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()]):
+                if not any(e.type == "SOCIAL" and e.data == event_data for e in event.get_parents()):
                     social_event = self.make_event(
                         event_data,
                         "SOCIAL",

bbot/modules/subdomaincenter.py

@@ -33,7 +33,7 @@ class subdomaincenter(subdomain_enum):
                 break
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

bbot/modules/subdomainradar.py

@@ -0,0 +1,160 @@
+import time
+import asyncio
+
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class SubdomainRadar(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query the Subdomain API for subdomains",
+        "created_date": "2022-07-08",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "group": "fast", "timeout": 120}
+    options_desc = {
+        "api_key": "SubDomainRadar.io API key",
+        "group": "The enumeration group to use. Choose from fast, medium, deep",
+        "timeout": "Timeout in seconds",
+    }
+
+    base_url = "https://api.subdomainradar.io"
+    ping_url = f"{base_url}/profile"
+    group_choices = ("fast", "medium", "deep")
+
+    # set this really high so the poll loop finishes as soon as possible
+    _qsize = 9999999
+
+    async def setup(self):
+        self.group = self.config.get("group", "fast").strip().lower()
+        self.timeout = self.config.get("timeout", 120)
+        if self.group not in self.group_choices:
+            return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}'
+        success, reason = await self.require_api_key()
+        if not success:
+            return success, reason
+        # convert groups to enumerators
+        enumerators = {}
+        response = await self.api_request(f"{self.base_url}/enumerators/groups")
+        status_code = getattr(response, "status_code", 0)
+        if status_code != 200:
+            return False, f"Failed to get enumerators: (HTTP status code: {status_code})"
+        else:
+            try:
+                j = response.json()
+            except Exception:
+                return False, "Failed to get enumerators: failed to parse response as JSON"
+            for group in j:
+                group_name = group.get("name", "").strip().lower()
+                if group_name:
+                    group_enumerators = []
+                    for enumerator in group.get("enumerators", []):
+                        enumerator_name = enumerator.get("display_name", "")
+                        if enumerator_name:
+                            group_enumerators.append(enumerator_name)
+                    if group_enumerators:
+                        enumerators[group_name] = group_enumerators
+
+        self.enumerators = enumerators.get(self.group, [])
+        if not self.enumerators:
+            return False, f'No enumerators found for group: "{self.group}" ({self.enumerators})'
+
+        self.enum_tasks = {}
+        self.poll_task = asyncio.create_task(self.task_poll_loop())
+
+        return True
+
+    def prepare_api_request(self, url, kwargs):
+        if self.api_key:
+            kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"}
+        return url, kwargs
+
+    async def handle_event(self, event):
+        query = self.make_query(event)
+        # start enumeration task
+        url = f"{self.base_url}/enumerate"
+        response = await self.api_request(
+            url, method="POST", json={"domains": [query], "enumerators": self.enumerators}
+        )
+        try:
+            j = response.json()
+        except Exception:
+            self.warning(f"Failed to parse response as JSON: {getattr(response, 'text', '')}")
+            return
+        task_id = j.get("tasks", {}).get(query, "")
+        if not task_id:
+            self.warning(f"Failed to start enumeration for {query}")
+            return
+        self.enum_tasks[query] = (task_id, time.time(), event)
+        self.debug(f"Started enumeration task for {query}; task id: {task_id}")
+
+    async def task_poll_loop(self):
+        # async with self._task_counter.count(f"{self.name}.task_poll_loop()"):
+        while 1:
+            for query, (task_id, start_time, event) in list(self.enum_tasks.items()):
+                url = f"{self.base_url}/tasks/{task_id}"
+                response = await self.api_request(url)
+                if getattr(response, "status_code", 0) == 200:
+                    finished = await self.parse_response(response, query, event)
+                    if finished:
+                        self.enum_tasks.pop(query)
+                        continue
+                # if scan is finishing, consider timeout
+                if self.scan.status == "FINISHING":
+                    if start_time + self.timeout < time.time():
+                        self.enum_tasks.pop(query)
+                        self.info(f"Enumeration task for {query} timed out")
+
+            if self.scan.status == "FINISHING" and not self.enum_tasks:
+                break
+            await self.helpers.sleep(5)
+
+    async def parse_response(self, response, query, event):
+        j = response.json()
+        status = j.get("status", "")
+        if status.lower() == "completed":
+            for subdomain in j.get("subdomains", []):
+                hostname = subdomain.get("subdomain", "")
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched SubDomainRadar.io API for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+            return True
+        return False
+
+    async def finish(self):
+        start_time = time.time()
+        while self.enum_tasks and not self.poll_task.done():
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= self.timeout:
+                self.warning(f"Timed out waiting for the following tasks to finish: {self.enum_tasks}")
+                for query, (task_id, _, _) in list(self.enum_tasks.items()):
+                    url = f"{self.base_url}/tasks/{task_id}"
+                    self.warning(f"    - {query} ({url})")
+                break
+
+            self.verbose(
+                f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)"
+            )
+
+            try:
+                # Wait for the task to complete or for 10 seconds, whichever comes first
+                await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10)
+            except asyncio.TimeoutError:
+                # This just means our 10-second check has elapsed, not that the task failed
+                pass
+
+        # Cancel the poll_task if it's still running
+        if not self.poll_task.done():
+            self.poll_task.cancel()
+            try:
+                await self.poll_task
+            except asyncio.CancelledError:
+                pass

bbot/modules/telerik.py

@@ -173,8 +173,8 @@ class telerik(BaseModule):
             webresource = "Telerik.Web.UI.WebResource.axd?type=rau"
             result, _ = await self.test_detector(event.data, webresource)
             if result:
-                if "RadAsyncUpload handler is registered succesfully" in result.text:
-                    self.debug(f"Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
+                if "RadAsyncUpload handler is registered successfully" in result.text:
+                    self.debug("Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
 
                     probe_data = {
                         "rauPostData": (
@@ -216,7 +216,7 @@ class telerik(BaseModule):
                         event,
                         context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik RAU AXD Handler",
                     )
-                    if self.config.get("exploit_RAU_crypto") == True:
+                    if self.config.get("exploit_RAU_crypto") is True:
                         hostname = urlparse(event.data).netloc
                         if hostname not in self.RAUConfirmed:
                             self.RAUConfirmed.append(hostname)
@@ -270,7 +270,7 @@ class telerik(BaseModule):
                 else:
                     if "Cannot deserialize dialog parameters" in response.text:
                         self.debug(f"Detected Telerik UI instance ({dh})")
-                        description = f"Telerik DialogHandler detected"
+                        description = "Telerik DialogHandler detected"
                         await self.emit_event(
                             {"host": str(event.host), "url": f"{event.data}{dh}", "description": description},
                             "FINDING",
@@ -289,8 +289,8 @@ class telerik(BaseModule):
                 self.debug(validate_result)
                 validate_status_code = getattr(validate_result, "status_code", 0)
                 if validate_status_code not in (0, 500):
-                    self.debug(f"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
-                    description = f"Telerik SpellCheckHandler detected"
+                    self.debug("Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)")
+                    description = "Telerik SpellCheckHandler detected"
                     await self.emit_event(
                         {
                             "host": str(event.host),
@@ -334,7 +334,7 @@ class telerik(BaseModule):
                         },
                         "FINDING",
                         event,
-                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik ChartImage AXD Handler",
+                        context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik ChartImage AXD Handler",
                     )
                 elif '"_serializedConfiguration":"' in resp_body:
                     await self.emit_event(
@@ -345,7 +345,7 @@ class telerik(BaseModule):
                         },
                         "FINDING",
                         event,
-                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik AsyncUpload",
+                        context="{module} searched HTTP_RESPONSE and identified {event.type}: Telerik AsyncUpload",
                     )
 
         # Check for RAD Controls in URL

bbot/modules/templates/bucket.py

@@ -159,7 +159,7 @@ class bucket_template(BaseModule):
         valid = self.cloud_helper.is_valid_bucket_name(bucket_name)
         if valid and not self.helpers.is_ip(bucket_name):
             bucket_hash = hash(bucket_name)
-            if not bucket_hash in self.buckets_tried:
+            if bucket_hash not in self.buckets_tried:
                 self.buckets_tried.add(bucket_hash)
                 return True
         return False

bbot/modules/templates/github.py

@@ -1,3 +1,5 @@
+import traceback
+
 from bbot.modules.base import BaseModule
 
 
@@ -9,30 +11,36 @@ class github(BaseModule):
 
     _qsize = 1
     base_url = "https://api.github.com"
+    ping_url = f"{base_url}/zen"
+
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["Authorization"] = f"token {self.api_key}"
+        return url, kwargs
 
     async def setup(self):
         await super().setup()
-        self.api_key = None
         self.headers = {}
-        for module_name in ("github", "github_codesearch", "github_org", "git_clone"):
-            module_config = self.scan.config.get("modules", {}).get(module_name, {})
+        api_keys = set()
+        modules_config = self.scan.config.get("modules", {})
+        git_modules = [m for m in modules_config if str(m).startswith("git")]
+        for module_name in git_modules:
+            module_config = modules_config.get(module_name, {})
             api_key = module_config.get("api_key", "")
-            if api_key:
-                self.api_key = api_key
-                self.headers = {"Authorization": f"token {self.api_key}"}
-                break
-        if not self.api_key:
+            if isinstance(api_key, str):
+                api_key = [api_key]
+            for key in api_key:
+                key = key.strip()
+                if key:
+                    api_keys.add(key)
+        if not api_keys:
             if self.auth_required:
                 return None, "No API key set"
+        self.api_key = api_keys
         try:
             await self.ping()
-            self.hugesuccess(f"API is ready")
+            self.hugesuccess("API is ready")
             return True
         except Exception as e:
+            self.trace(traceback.format_exc())
             return None, f"Error with API ({str(e).strip()})"
         return True
-
-    async def ping(self):
-        url = f"{self.base_url}/zen"
-        response = await self.helpers.request(url, headers=self.headers)
-        assert getattr(response, "status_code", 0) == 200, response.text

bbot/modules/templates/postman.py

@@ -0,0 +1,21 @@
+from bbot.modules.base import BaseModule
+
+
+class postman(BaseModule):
+    """
+    A template module for use of the GitHub API
+    Inherited by several other github modules.
+    """
+
+    base_url = "https://www.postman.com/_api"
+    api_url = "https://api.getpostman.com"
+    html_url = "https://www.postman.com"
+    ping_url = f"{api_url}/me"
+
+    headers = {
+        "Content-Type": "application/json",
+        "X-App-Version": "10.18.8-230926-0808",
+        "X-Entity-Team-Id": "0",
+        "Origin": "https://www.postman.com",
+        "Referer": "https://www.postman.com/search?q=&scope=public&type=all",
+    }

bbot/modules/templates/shodan.py

@@ -1,3 +1,5 @@
+import traceback
+
 from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
@@ -6,29 +8,28 @@ class shodan(subdomain_enum):
     options_desc = {"api_key": "Shodan API key"}
 
     base_url = "https://api.shodan.io"
+    ping_url = f"{base_url}/api-info?key={{api_key}}"
 
     async def setup(self):
         await super().setup()
-        self.api_key = None
+        api_keys = set()
         for module_name in ("shodan", "shodan_dns", "shodan_port"):
             module_config = self.scan.config.get("modules", {}).get(module_name, {})
             api_key = module_config.get("api_key", "")
-            if api_key:
-                self.api_key = api_key
-                break
-        if not self.api_key:
+            if isinstance(api_key, str):
+                api_key = [api_key]
+            for key in api_key:
+                key = key.strip()
+                if key:
+                    api_keys.add(key)
+        if not api_keys:
             if self.auth_required:
                 return None, "No API key set"
+        self.api_key = api_keys
         try:
             await self.ping()
-            self.hugesuccess(f"API is ready")
+            self.hugesuccess("API is ready")
             return True
         except Exception as e:
+            self.trace(traceback.format_exc())
             return None, f"Error with API ({str(e).strip()})"
-        return True
-
-    async def ping(self):
-        url = f"{self.base_url}/api-info?key={self.api_key}"
-        r = await self.request_with_fail_count(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content