bbot 2.4.1.6095rc0__py3-none-any.whl → 2.4.1.6107rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.1.6095rc"
+__version__ = "v2.4.1.6107rc"
 
 from .scanner import Scanner, Preset
 

bbot/defaults.yml

@@ -92,10 +92,18 @@ web:
   # These are attached to all in-scope HTTP requests
   # Note that some modules (e.g. github) may end up sending these to out-of-scope resources
   http_headers: {}
-  # HTTP retries (for Python requests; API calls, etc.)
+  # How many times to retry API requests
+  # Note that this is a separate mechanism on top of HTTP retries
+  # which will retry API requests that don't return a successful status code
+  api_retries: 2
+  # HTTP retries - try again if the raw connection fails
   http_retries: 1
   # HTTP retries (for httpx)
   httpx_retries: 1
+  # Default sleep interval when rate limited by 429 (and retry-after isn't provided)
+  429_sleep_interval: 30
+  # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)
+  429_max_sleep_interval: 60
   # Enable/disable debug messages for web requests/responses
   debug: false
   # Maximum number of HTTP redirects to follow

bbot/modules/base.py

@@ -104,12 +104,8 @@ class BaseModule:
     _batch_size = 1
     batch_wait = 10
 
-    # API retries, etc.
-    _api_retries = 2
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
-    # sleep for this many seconds after being rate limited
-    _429_sleep_interval = 30
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -159,12 +155,18 @@ class BaseModule:
         # track number of failures (for .api_request())
         self._api_request_failures = 0
 
+        self._default_api_retries = self.scan.config.get("web", {}).get("api_retries", 2)
+
         self._tasks = []
         self._event_received = None
 
         # used for optional "per host" tracking
         self._per_host_tracker = set()
 
+        # 429 rate limit handling
+        self._429_sleep_interval = self.scan.web_config.get("429_sleep_interval", 30)
+        self._429_max_sleep_interval = self.scan.web_config.get("429_max_sleep_interval", 60)
+
     async def setup(self):
         """
         Performs one-time setup tasks for the module.
@@ -338,7 +340,7 @@ class BaseModule:
 
     @property
     def api_retries(self):
-        return max(self._api_retries + 1, len(self._api_keys))
+        return max(self._default_api_retries + 1, len(self._api_keys))
 
     @property
     def api_failure_abort_threshold(self):
@@ -1172,6 +1174,11 @@ class BaseModule:
                     retry_after = self._get_retry_after(r)
                     if retry_after or status_code == 429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
+                        if retry_after and retry_after > self._429_max_sleep_interval:
+                            self.verbose(
+                                f"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead"
+                            )
+                            sleep_interval = self._429_max_sleep_interval
                         self.verbose(
                             f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})"
                         )
@@ -1205,7 +1212,8 @@ class BaseModule:
         return url, requests_kwargs
 
     def _api_response_is_success(self, r):
-        return r.is_success
+        # 404s typically indicate no data rather than an actual error with the API, so we don't want to retry them
+        return getattr(r, "is_success", False) or getattr(r, "status_code", 0) == 404
 
     async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):
         """

bbot/modules/certspotter.py

@@ -15,7 +15,7 @@ class certspotter(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
-        return self.api_request(url, timeout=self.http_timeout + 30)
+        return self.api_request(url)
 
     async def parse_results(self, r, query):
         results = set()

bbot/modules/shodan_idb.py

@@ -1,4 +1,5 @@
 from bbot.modules.base import BaseModule
+import time
 
 
 class shodan_idb(BaseModule):
@@ -46,23 +47,48 @@ class shodan_idb(BaseModule):
         "created_date": "2023-12-22",
         "author": "@TheTechromancer",
     }
+    options = {"retries": None}
+    options_desc = {
+        "retries": "How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting."
+    }
 
-    # we get lots of 404s, that's normal
+    # we typically don't want to abort this module
     _api_failure_abort_threshold = 9999999999
 
-    # there aren't any rate limits to speak of, so our outgoing queue can be pretty big
-    _qsize = 500
+    # since there are rate limits, we set a lower qsize
+    # this way when our queue is full, we can give the API a break
+    _qsize = 100
 
     base_url = "https://internetdb.shodan.io"
 
+    async def setup(self):
+        await super().setup()
+        self.last_request_time = 0
+        return True
+
     def _incoming_dedup_hash(self, event):
         return hash(self.get_ip(event))
 
+    @property
+    def api_retries(self):
+        # allow the module to override global retry setting
+        return self.config.get("retries", None) or super().api_retries
+
     async def handle_event(self, event):
         ip = self.get_ip(event)
         if ip is None:
             return
         url = f"{self.base_url}/{ip}"
+
+        # Rate limiting: ensure at least 1 second between requests
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < 1:
+            await self.helpers.sleep(1 - time_since_last)
+
+        # Update the last request time
+        self.last_request_time = time.time()
+
         r = await self.api_request(url)
         if r is None:
             self.debug(f"No response for {event.data}")

bbot/modules/templates/webhook.py

@@ -16,10 +16,9 @@ class WebhookOutputModule(BaseOutputModule):
     # abort module after 10 failed requests (not including retries)
     _api_failure_abort_threshold = 10
     # retry each request up to 10 times, respecting the Retry-After header
-    _api_retries = 10
+    _default_api_retries = 10
 
     async def setup(self):
-        self._api_retries = self.config.get("retries", 10)
         self.webhook_url = self.config.get("webhook_url", "")
         self.min_severity = self.config.get("min_severity", "LOW").strip().upper()
         assert self.min_severity in self.vuln_severities, (
@@ -31,6 +30,10 @@ class WebhookOutputModule(BaseOutputModule):
             return False
         return await super().setup()
 
+    @property
+    def api_retries(self):
+        return self.config.get("retries", self._default_api_retries)
+
     async def handle_event(self, event):
         message = self.format_message(event)
         data = {self.content_key: message}

{bbot-2.4.1.6095rc0.dist-info → bbot-2.4.1.6107rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.1.6095rc0
+Version: 2.4.1.6107rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.1.6095rc0.dist-info → bbot-2.4.1.6107rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=IWliuFZfjWjwG33exEqeUFSip4K0QyS7Ah6FYizaqJI,163
+bbot/__init__.py,sha256=sam4KorFe9w2RyVwBP9vdpFRehUOzVwRYur0Gh67BqY,163
 bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -48,7 +48,7 @@ bbot/core/modules.py,sha256=U0Z2UoZAOPG9lLvR9Juc3UwdWCc_xbktF4t_NoiKPrY,31385
 bbot/core/multiprocess.py,sha256=ocQHanskJ09gHwe7RZmwNdZyCOQyeyUoIHCtLbtvXUk,1771
 bbot/core/shared_deps.py,sha256=mCMZeKSt46trzVqQDPGfXfEWg0Zw5YjiJx4BnsIRgHM,7640
 bbot/db/sql/models.py,sha256=SrUdDOBCICzXJBY29p0VvILhMQ1JCuh725bqvIYogX0,4884
-bbot/defaults.yml,sha256=XPaGfTKWFjKV-lLIJy-Qs7X0d-8X8EYM0QELEu8KjBw,6670
+bbot/defaults.yml,sha256=wIiOAEvsnJi_Q7qXC1Pn1TCSWiSh8CagpwVvVIGefLs,7106
 bbot/errors.py,sha256=xwQcD26nU9oc7-o0kv5jmEDTInmi8_W8eKAgQZZxdVM,953
 bbot/logger.py,sha256=wE-532v5FyKuSSoTdyW1xSfaOnLZB1axAJnB-uW2xrI,2745
 bbot/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -61,7 +61,7 @@ bbot/modules/baddns.py,sha256=ubO3KDfcIMJnMjyZX5FWZ4GWxLSekV_JQV7QvsPjtD0,6693
 bbot/modules/baddns_direct.py,sha256=hWThpkXP87nnCRTlUh5qBJ1t4eo4l9kUmKNNxVNJI8A,3819
 bbot/modules/baddns_zone.py,sha256=y1XaBUfFPnRbR2qaTqRyUsPgEL73722v2B8aS5YoGN4,1035
 bbot/modules/badsecrets.py,sha256=LG37p48Rlxsfc3BmACMpkypsbuFTVvXqNhlP1IEsx0k,5109
-bbot/modules/base.py,sha256=SCIeaLX1d5VMUl2nC3IyqPdzBF8mr_7__JB3855q5cc,74529
+bbot/modules/base.py,sha256=_RFTPNZ6X7mD7jBOTYZmjgzTSKWT9U_eYxL4gXIh8TE,75265
 bbot/modules/bevigil.py,sha256=0VLIxmeXRUI2-EoR6IzuHJMcX8KCHNNta-WYa3gVlDg,2862
 bbot/modules/binaryedge.py,sha256=5F9LnZwRM_rZnzTv29hLJLI2GEQdzOwSpahPFC1kJC0,1397
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
@@ -75,7 +75,7 @@ bbot/modules/builtwith.py,sha256=6ZQOc6vmSVc8LsdgsiuMWfDquGm5K0jxwsnL8MsKNWw,538
 bbot/modules/bypass403.py,sha256=HyONgOYlZUET61FZ0QWE7zPGG-N6n0x_j9KUGw8kVxQ,6855
 bbot/modules/c99.py,sha256=yHT9-eyqRODISV5eTi11uh-xwqX0JG7zey5AgcDYCdI,1448
 bbot/modules/censys.py,sha256=J7NhPnSeoCuG_9FkLjFBIg-tqHAB21HjvQUw_6OQNZo,3311
-bbot/modules/certspotter.py,sha256=AtL5BiOuDp4vu1-5fct4aQAGZM2qiODYsbgBsw0phoU,937
+bbot/modules/certspotter.py,sha256=qdRGCkGyP07_cP9h2o_AEZwoiQPXtrC-Bel3vgh24x8,905
 bbot/modules/chaos.py,sha256=JyuwytwE3IRmNbw-uyJ0gCaTnywhhsHzTiZ3OJ15PAw,1573
 bbot/modules/code_repository.py,sha256=x70Z45VnNNMF8BPkHfGWZXsZXw_fStGB3y0-8jbP1Ns,2078
 bbot/modules/credshed.py,sha256=HAF5wgRGKIIpdMAe4mIAtkZRLmFYjMFyXtjjst6RJ20,4203
@@ -177,7 +177,7 @@ bbot/modules/robots.py,sha256=LGG6ixsxrlaCk-mi4Lp6kB2RB1v-25NhTAQxdQEtH8s,2172
 bbot/modules/securitytrails.py,sha256=5Jk_HTQP8FRq6A30sN19FU79uLJt7aiOsI2dxNkLDcM,1148
 bbot/modules/securitytxt.py,sha256=nwaTOnRAl2NWcEc3i_I9agB56QjqK8dHqiKRHPPkCPE,4558
 bbot/modules/shodan_dns.py,sha256=ETOyUhCiAETlGUAQhvAP47oEEPYss7fm_F_CAeCQyoI,842
-bbot/modules/shodan_idb.py,sha256=kyPbGI68BrLgCab_T5emKXBoyD2OVn59K0J1V6CNR4s,5213
+bbot/modules/shodan_idb.py,sha256=PB5vplwpjPjbfoMeez333rWYqFY5RuMp40ZN9hNCJAg,6088
 bbot/modules/sitedossier.py,sha256=MR9fSkgE_3YGsUe7M-TyJ2GdpBtF3oLKDl9agAwzw5U,2284
 bbot/modules/skymem.py,sha256=ZrxWcePFTCiDkFeAc3YLegFG-Tgw4C9af_JHiVonk84,1930
 bbot/modules/smuggler.py,sha256=v8NCRgzd7wpEFZJUTAArG04bN8nNTGiHxYpGBapzi14,1580
@@ -192,7 +192,7 @@ bbot/modules/templates/postman.py,sha256=MIpz2q_r6LP0kIEgByo7oX5qHhMZLOhr7oKzJI9
 bbot/modules/templates/shodan.py,sha256=MXBvlmfw3jZFqT47v10UkqMSyQR-zBIxMJmK7PWw6uw,1174
 bbot/modules/templates/sql.py,sha256=o-CdyyoJvHJdJBKkj3CIGXYxUta4w2AB_2Vr-k7cDDU,3553
 bbot/modules/templates/subdomain_enum.py,sha256=epyKSly08jqaINV_AMMWbNafIeQjJqvd3aj63KD0Mck,8402
-bbot/modules/templates/webhook.py,sha256=0N7LorCxT0sVmT2XgrV81ubdcdPVW8s2X__BrKC2AcM,3313
+bbot/modules/templates/webhook.py,sha256=uGFmcJ81GzGN1UI2k2O7nQF_fyh4ehLDEg2NSXaPnhk,3373
 bbot/modules/trickest.py,sha256=MRgLW0YiDWzlWdAjyqfPPLFb-a51r-Ffn_dphiJI_gA,1550
 bbot/modules/trufflehog.py,sha256=KEObs5z0ce0Ck7gA3t3l66MJKUo6P_9TIfVOwFO28t4,8743
 bbot/modules/url_manipulation.py,sha256=4J3oFkqTSJPPmbKEKAHJg2Q2w4QNKtQhiN03ZJq5VtI,4326
@@ -428,8 +428,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.4.1.6095rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.4.1.6095rc0.dist-info/METADATA,sha256=s1SYf_crJYiFaXoX6ZM7VtItSyo5A6n2TY5XOp68Mdc,18218
-bbot-2.4.1.6095rc0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-bbot-2.4.1.6095rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.4.1.6095rc0.dist-info/RECORD,,
+bbot-2.4.1.6107rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.4.1.6107rc0.dist-info/METADATA,sha256=Tbmi58DKjLEf9JRM2BKNULln0TwoEqeiPdGOMn3kQdU,18218
+bbot-2.4.1.6107rc0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+bbot-2.4.1.6107rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.4.1.6107rc0.dist-info/RECORD,,