bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/base.py

@@ -63,7 +63,7 @@ class BaseModule:
 
         batch_wait (int): Seconds to wait before force-submitting a batch. Default is 10.
 
-        failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `request_with_fail_count()` is used. Default is 5.
+        api_failure_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5.
 
         _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False.
 
@@ -103,7 +103,13 @@ class BaseModule:
     _module_threads = 1
     _batch_size = 1
     batch_wait = 10
-    failed_request_abort_threshold = 5
+
+    # API retries, etc.
+    _api_retries = 2
+    # disable the module after this many failed attempts in a row
+    _api_failure_abort_threshold = 3
+    # sleep for this many seconds after being rate limited
+    _429_sleep_interval = 30
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -148,8 +154,10 @@ class BaseModule:
         # string constant
         self._custom_filter_criteria_msg = "it did not meet custom filter criteria"
 
-        # track number of failures (for .request_with_fail_count())
-        self._request_failures = 0
+        self._api_keys = []
+
+        # track number of failures (for .api_request())
+        self._api_request_failures = 0
 
         self._tasks = []
         self._event_received = asyncio.Condition()
@@ -303,32 +311,76 @@ class BaseModule:
         if self.auth_secret:
             try:
                 await self.ping()
-                self.hugesuccess(f"API is ready")
-                return True
+                self.hugesuccess("API is ready")
+                return True, ""
             except Exception as e:
+                self.trace(traceback.format_exc())
                 return None, f"Error with API ({str(e).strip()})"
         else:
             return None, "No API key set"
 
-    async def ping(self):
+    @property
+    def api_key(self):
+        if self._api_keys:
+            return self._api_keys[0]
+
+    @api_key.setter
+    def api_key(self, api_keys):
+        if isinstance(api_keys, str):
+            api_keys = [api_keys]
+        self._api_keys = list(api_keys)
+
+    def cycle_api_key(self):
+        if len(self._api_keys) > 1:
+            self.verbose("Cycling API key")
+            self._api_keys.insert(0, self._api_keys.pop())
+        else:
+            self.debug("No extra API keys to cycle")
+
+    @property
+    def api_retries(self):
+        return max(self._api_retries + 1, len(self._api_keys))
+
+    @property
+    def api_failure_abort_threshold(self):
+        return (self.api_retries * self._api_failure_abort_threshold) + 1
+
+    async def ping(self, url=None):
         """Asynchronously checks the health of the configured API.
 
-        This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. This method should include an assert statement to validate the API's health, typically by making a test request to a known endpoint.
+        This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. It makes a test request to a known endpoint to validate the API's health.
 
-        Example Usage:
-            In your implementation, if the API has a "/ping" endpoint:
-            async def ping(self):
-                r = await self.request_with_fail_count(f"{self.base_url}/ping")
-                resp_content = getattr(r, "text", "")
-                assert getattr(r, "status_code", 0) == 200, resp_content
+        The method uses the `ping_url` attribute if defined, or falls back to a provided URL. If neither is available, no request is made.
+
+        Args:
+            url (str, optional): A specific URL to use for the ping request. If not provided, the method will use the `ping_url` attribute.
 
         Returns:
             None
 
         Raises:
-            AssertionError: If the API does not respond as expected.
-        """
-        return
+            ValueError: If the API response is not successful (status code != 200).
+
+        Example Usage:
+            To use this method, simply define the `ping_url` attribute in your module:
+
+            class MyModule(BaseModule):
+                ping_url = "https://api.example.com/ping"
+
+            Alternatively, you can override this method for more complex health checks:
+
+            async def ping(self):
+                r = await self.api_request(f"{self.base_url}/complex-health-check")
+                if r.status_code != 200 or r.json().get('status') != 'healthy':
+                    raise ValueError(f"API unhealthy: {r.text}")
+        """
+        if url is None:
+            url = getattr(self, "ping_url", "")
+        if url:
+            r = await self.api_request(url)
+            if getattr(r, "status_code", 0) != 200:
+                response_text = getattr(r, "text", "no response from server")
+                raise ValueError(response_text)
 
     @property
     def batch_size(self):
@@ -617,7 +669,7 @@ class BaseModule:
                             if self.incoming_event_queue is not False:
                                 event = await self.incoming_event_queue.get()
                             else:
-                                self.debug(f"Event queue is in bad state")
+                                self.debug("Event queue is in bad state")
                                 break
                         except asyncio.queues.QueueEmpty:
                             continue
@@ -648,7 +700,7 @@ class BaseModule:
                 else:
                     self.error(f"Critical failure in module {self.name}: {e}")
                     self.error(traceback.format_exc())
-        self.log.trace(f"Worker stopped")
+        self.log.trace("Worker stopped")
 
     @property
     def max_scope_distance(self):
@@ -691,7 +743,7 @@ class BaseModule:
         if event.type in ("FINISHED",):
             return True, "its type is FINISHED"
         if self.errored:
-            return False, f"module is in error state"
+            return False, "module is in error state"
         # exclude non-watched types
         if not any(t in self.get_watched_events() for t in ("*", event.type)):
             return False, "its type is not in watched_events"
@@ -718,7 +770,7 @@ class BaseModule:
             # check duplicates
             is_incoming_duplicate, reason = self.is_incoming_duplicate(event, add=True)
             if is_incoming_duplicate and not self.accept_dupes:
-                return False, f"module has already seen it" + (f" ({reason})" if reason else "")
+                return False, "module has already seen it" + (f" ({reason})" if reason else "")
 
         return acceptable, reason
 
@@ -811,7 +863,7 @@ class BaseModule:
         """
         async with self._task_counter.count("queue_event()", _log=False):
             if self.incoming_event_queue is False:
-                self.debug(f"Not in an acceptable state to queue incoming event")
+                self.debug("Not in an acceptable state to queue incoming event")
                 return
             acceptable, reason = self._event_precheck(event)
             if not acceptable:
@@ -827,7 +879,7 @@ class BaseModule:
                 if event.type != "FINISHED":
                     self.scan._new_activity = True
             except AttributeError:
-                self.debug(f"Not in an acceptable state to queue incoming event")
+                self.debug("Not in an acceptable state to queue incoming event")
 
     async def queue_outgoing_event(self, event, **kwargs):
         """
@@ -852,7 +904,7 @@ class BaseModule:
         try:
             await self.outgoing_event_queue.put((event, kwargs))
         except AttributeError:
-            self.debug(f"Not in an acceptable state to queue outgoing event")
+            self.debug("Not in an acceptable state to queue outgoing event")
 
     def set_error_state(self, message=None, clear_outgoing_queue=False, critical=False):
         """
@@ -887,7 +939,7 @@ class BaseModule:
             self.errored = True
             # clear incoming queue
             if self.incoming_event_queue is not False:
-                self.debug(f"Emptying event_queue")
+                self.debug("Emptying event_queue")
                 with suppress(asyncio.queues.QueueEmpty):
                     while 1:
                         self.incoming_event_queue.get_nowait()
@@ -932,8 +984,11 @@ class BaseModule:
     def _outgoing_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
+
+        We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules)
+        to inadvertently suppress a non-internal event.
         """
-        return hash((event, self.name))
+        return hash((event, self.name, event.internal, event.always_emit))
 
     def get_per_host_hash(self, event):
         """
@@ -1065,32 +1120,122 @@ class BaseModule:
         async for line in self.helpers.run_live(*args, **kwargs):
             yield line
 
-    async def request_with_fail_count(self, *args, **kwargs):
-        """Asynchronously perform an HTTP request while keeping track of consecutive failures.
+    def prepare_api_request(self, url, kwargs):
+        """
+        Prepare an API request by adding the necessary authentication - header, bearer token, etc.
+        """
+        if self.api_key:
+            url = url.format(api_key=self.api_key)
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            kwargs["headers"]["Authorization"] = f"Bearer {self.api_key}"
+        return url, kwargs
 
-        This function wraps the `self.helpers.request` method, incrementing a failure counter if
-        the request returns None. When the failure counter exceeds `self.failed_request_abort_threshold`,
-        the module is set to an error state.
+    async def api_request(self, *args, **kwargs):
+        """
+        Makes an HTTP request while automatically:
+            - avoiding rate limits (sleep/retry)
+            - cycling API keys
+            - cancelling after too many failed attempts
+        """
+        url = args[0] if args else kwargs.pop("url", "")
 
-        Args:
-            *args: Positional arguments to pass to `self.helpers.request`.
-            **kwargs: Keyword arguments to pass to `self.helpers.request`.
+        # loop until we have a successful request
+        for _ in range(self.api_retries):
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            new_url, kwargs = self.prepare_api_request(url, kwargs)
+            kwargs["url"] = new_url
 
-        Returns:
-            Any: The response object or None if the request failed.
+            r = await self.helpers.request(**kwargs)
+            success = False if r is None else r.is_success
+
+            if success:
+                self._api_request_failures = 0
+            else:
+                status_code = getattr(r, "status_code", 0)
+                response_text = getattr(r, "text", "")
+                self.trace(f"API response to {url} failed with status code {status_code}: {response_text}")
+                self._api_request_failures += 1
+                if self._api_request_failures >= self.api_failure_abort_threshold:
+                    self.set_error_state(
+                        f"Setting error state due to {self._api_request_failures:,} failed HTTP requests"
+                    )
+                else:
+                    # sleep for a bit if we're being rate limited
+                    if status_code == 429:
+                        self.verbose(
+                            f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)"
+                        )
+                        await asyncio.sleep(self._429_sleep_interval)
+                    elif self._api_keys:
+                        # if request failed, cycle API keys and try again
+                        self.cycle_api_key()
+                    continue
+            break
 
-        Raises:
-            None: Sets the module to an error state when the failure threshold is reached.
-        """
-        r = await self.helpers.request(*args, **kwargs)
-        if r is None:
-            self._request_failures += 1
-        else:
-            self._request_failures = 0
-        if self._request_failures >= self.failed_request_abort_threshold:
-            self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests")
         return r
 
+    async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
+        """
+        An asynchronous generator function for iterating through paginated API data.
+
+        This function continuously makes requests to a specified API URL, incrementing the page number
+        or applying a custom pagination function, and yields the received data one page at a time.
+        It is well-suited for APIs that provide paginated results.
+
+        Args:
+            url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.
+            page_size (int, optional): The number of items per page. Defaults to 100.
+            json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
+            next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+            **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
+
+        Yields:
+            dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.
+
+        Note:
+            The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.
+
+        Examples:
+            >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')
+            >>> try:
+            >>>     async for page in agen:
+            >>>         subdomains = page["subdomains"]
+            >>>         self.hugesuccess(subdomains)
+            >>>         if not subdomains:
+            >>>             break
+            >>> finally:
+            >>>     agen.aclose()
+        """
+        page = 1
+        offset = 0
+        result = None
+        while 1:
+            if result and callable(next_key):
+                try:
+                    new_url = next_key(result)
+                except Exception as e:
+                    self.debug(f"Failed to extract next page of results from {url}: {e}")
+                    self.debug(traceback.format_exc())
+            else:
+                new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
+            result = await self.api_request(new_url, **requests_kwargs)
+            if result is None:
+                self.verbose(f"api_page_iter() got no response for {url}")
+                break
+            try:
+                if json:
+                    result = result.json()
+                yield result
+            except Exception:
+                self.warning(f'Error in api_page_iter() for url: "{new_url}"')
+                self.trace(traceback.format_exc())
+                break
+            finally:
+                offset += page_size
+                page += 1
+
     @property
     def preset(self):
         return self.scan.preset
@@ -1417,7 +1562,7 @@ class BaseModule:
             self.trace()
 
 
-class InterceptModule(BaseModule):
+class BaseInterceptModule(BaseModule):
     """
     An Intercept Module is a special type of high-priority module that gets early access to events.
 
@@ -1429,7 +1574,6 @@ class InterceptModule(BaseModule):
     """
 
     accept_dupes = True
-    suppress_dupes = False
     _intercept = True
 
     async def _worker(self):
@@ -1445,7 +1589,7 @@ class InterceptModule(BaseModule):
                                 event = incoming
                                 kwargs = {}
                         else:
-                            self.debug(f"Event queue is in bad state")
+                            self.debug("Event queue is in bad state")
                             break
                     except asyncio.queues.QueueEmpty:
                         await asyncio.sleep(0.1)
@@ -1500,7 +1644,7 @@ class InterceptModule(BaseModule):
                 else:
                     self.critical(f"Critical failure in intercept module {self.name}: {e}")
                     self.critical(traceback.format_exc())
-        self.log.trace(f"Worker stopped")
+        self.log.trace("Worker stopped")
 
     async def get_incoming_event(self):
         """
@@ -1531,7 +1675,7 @@ class InterceptModule(BaseModule):
         try:
             self.incoming_event_queue.put_nowait((event, kwargs))
         except AttributeError:
-            self.debug(f"Not in an acceptable state to queue incoming event")
+            self.debug("Not in an acceptable state to queue incoming event")
 
     async def _event_postcheck(self, event):
         return await self._event_postcheck_inner(event)

bbot/modules/bevigil.py

@@ -22,12 +22,12 @@ class bevigil(subdomain_enum_apikey):
 
     async def setup(self):
         self.api_key = self.config.get("api_key", "")
-        self.headers = {"X-Access-Token": self.api_key}
         self.urls = self.config.get("urls", False)
         return await super().setup()
 
-    async def ping(self):
-        pass
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["X-Access-Token"] = self.api_key
+        return url, kwargs
 
     async def handle_event(self, event):
         query = self.make_query(event)
@@ -54,20 +54,20 @@ class bevigil(subdomain_enum_apikey):
 
     async def request_subdomains(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/"
-        return await self.request_with_fail_count(url, headers=self.headers)
+        return await self.api_request(url)
 
     async def request_urls(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}/urls/"
-        return await self.request_with_fail_count(url, headers=self.headers)
+        return await self.api_request(url)
 
-    def parse_subdomains(self, r, query=None):
+    async def parse_subdomains(self, r, query=None):
         results = set()
         subdomains = r.json().get("subdomains")
         if subdomains:
             results.update(subdomains)
         return results
 
-    def parse_urls(self, r, query=None):
+    async def parse_urls(self, r, query=None):
         results = set()
         urls = r.json().get("urls")
         if urls:

bbot/modules/binaryedge.py

@@ -21,19 +21,22 @@ class binaryedge(subdomain_enum_apikey):
 
     async def setup(self):
         self.max_records = self.config.get("max_records", 1000)
-        self.headers = {"X-Key": self.config.get("api_key", "")}
         return await super().setup()
 
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["X-Key"] = self.api_key
+        return url, kwargs
+
     async def ping(self):
         url = f"{self.base_url}/user/subscription"
-        j = (await self.request_with_fail_count(url, headers=self.headers)).json()
+        j = (await self.api_request(url)).json()
         assert j.get("requests_left", 0) > 0
 
     async def request_url(self, query):
         # todo: host query (certs + services)
         url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url, headers=self.headers)
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         return j.get("events", [])

bbot/modules/bufferoverrun.py

@@ -0,0 +1,47 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class BufferOverrun(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query BufferOverrun's TLS API for subdomains",
+        "created_date": "2024-10-23",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "commercial": False}
+    options_desc = {"api_key": "BufferOverrun API key", "commercial": "Use commercial API"}
+
+    base_url = "https://tls.bufferover.run/dns"
+    commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns"
+
+    async def setup(self):
+        self.commercial = self.config.get("commercial", False)
+        return await super().setup()
+
+    def prepare_api_request(self, url, kwargs):
+        if self.commercial:
+            kwargs["headers"]["x-rapidapi-host"] = "bufferover-run-tls.p.rapidapi.com"
+            kwargs["headers"]["x-rapidapi-key"] = self.api_key
+        else:
+            kwargs["headers"]["x-api-key"] = self.api_key
+        return url, kwargs
+
+    async def request_url(self, query):
+        url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
+        return await self.api_request(url)
+
+    async def parse_results(self, r, query):
+        j = r.json()
+        subdomains_set = set()
+        if isinstance(j, dict):
+            results = j.get("Results", [])
+            for result in results:
+                parts = result.split(",")
+                if len(parts) > 4:
+                    subdomain = parts[4].strip()
+                    if subdomain and subdomain.endswith(f".{query}"):
+                        subdomains_set.add(subdomain)
+        return subdomains_set

bbot/modules/builtwith.py

@@ -27,10 +27,6 @@ class builtwith(subdomain_enum_apikey):
     options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"}
     base_url = "https://api.builtwith.com"
 
-    async def ping(self):
-        # builtwith does not have a ping feature, so we skip it to save API credits
-        return
-
     async def handle_event(self, event):
         query = self.make_query(event)
         # domains
@@ -59,14 +55,14 @@ class builtwith(subdomain_enum_apikey):
                         )
 
     async def request_domains(self, query):
-        url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes"
-        return await self.request_with_fail_count(url)
+        url = f"{self.base_url}/v20/api.json?KEY={{api_key}}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes"
+        return await self.api_request(url)
 
     async def request_redirects(self, query):
-        url = f"{self.base_url}/redirect1/api.json?KEY={self.api_key}&LOOKUP={query}"
-        return await self.request_with_fail_count(url)
+        url = f"{self.base_url}/redirect1/api.json?KEY={{api_key}}&LOOKUP={query}"
+        return await self.api_request(url)
 
-    def parse_domains(self, r, query):
+    async def parse_domains(self, r, query):
         """
         This method returns a set of subdomains.
         Each subdomain is an "FQDN" that was reported in the "Detailed Technology Profile" page on builtwith.com
@@ -96,7 +92,7 @@ class builtwith(subdomain_enum_apikey):
                     self.verbose(f"No results for {query}: {error}")
         return results_set
 
-    def parse_redirects(self, r, query):
+    async def parse_redirects(self, r, query):
         """
         This method creates a set.
         Each entry in the set is either an Inbound or Outbound Redirect reported in the "Redirect Profile" page on builtwith.com

bbot/modules/bypass403.py

@@ -92,7 +92,7 @@ class bypass403(BaseModule):
                 return None
 
             sig = self.format_signature(sig, event)
-            if sig[2] != None:
+            if sig[2] is not None:
                 headers = dict(sig[2])
             else:
                 headers = None
@@ -106,13 +106,13 @@ class bypass403(BaseModule):
                 continue
 
             # In some cases WAFs will respond with a 200 code which causes a false positive
-            if subject_response != None:
+            if subject_response is not None:
                 for ws in waf_strings:
                     if ws in subject_response.text:
                         self.debug("Rejecting result based on presence of WAF string")
                         return
 
-            if match == False:
+            if match is False:
                 if str(subject_response.status_code)[0] != "4":
                     if sig[2]:
                         added_header_tuple = next(iter(sig[2].items()))
@@ -165,13 +165,13 @@ class bypass403(BaseModule):
         return False
 
     def format_signature(self, sig, event):
-        if sig[3] == True:
+        if sig[3] is True:
             cleaned_path = event.parsed_url.path.strip("/")
         else:
             cleaned_path = event.parsed_url.path.lstrip("/")
         kwargs = {"scheme": event.parsed_url.scheme, "netloc": event.parsed_url.netloc, "path": cleaned_path}
         formatted_url = sig[1].format(**kwargs)
-        if sig[2] != None:
+        if sig[2] is not None:
             formatted_headers = {k: v.format(**kwargs) for k, v in sig[2].items()}
         else:
             formatted_headers = None

bbot/modules/c99.py

@@ -15,17 +15,19 @@ class c99(subdomain_enum_apikey):
     options_desc = {"api_key": "c99.nl API key"}
 
     base_url = "https://api.c99.nl"
+    ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json"
 
     async def ping(self):
-        url = f"{self.base_url}/randomnumber?key={self.api_key}&between=1,100&json"
-        response = await self.request_with_fail_count(url)
-        assert response.json()["success"] == True
+        url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json"
+        response = await self.api_request(url)
+        assert response.json()["success"] is True, getattr(response, "text", "no response from server")
 
     async def request_url(self, query):
-        url = f"{self.base_url}/subdomainfinder?key={self.api_key}&domain={self.helpers.quote(query)}&json"
-        return await self.request_with_fail_count(url)
+        url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json"
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             subdomains = j.get("subdomains", [])
@@ -33,4 +35,5 @@ class c99(subdomain_enum_apikey):
                 for s in subdomains:
                     subdomain = s.get("subdomain", "")
                     if subdomain:
-                        yield subdomain
+                        results.add(subdomain)
+        return results