bbot 2.4.1.6100rc0__py3-none-any.whl → 2.4.2__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.1.6100rc"
+__version__ = "v2.4.2"
 
 from .scanner import Scanner, Preset
 

bbot/defaults.yml

@@ -92,7 +92,11 @@ web:
   # These are attached to all in-scope HTTP requests
   # Note that some modules (e.g. github) may end up sending these to out-of-scope resources
   http_headers: {}
-  # HTTP retries (for Python requests; API calls, etc.)
+  # How many times to retry API requests
+  # Note that this is a separate mechanism on top of HTTP retries
+  # which will retry API requests that don't return a successful status code
+  api_retries: 2
+  # HTTP retries - try again if the raw connection fails
   http_retries: 1
   # HTTP retries (for httpx)
   httpx_retries: 1

bbot/modules/base.py

@@ -104,8 +104,6 @@ class BaseModule:
     _batch_size = 1
     batch_wait = 10
 
-    # API retries, etc.
-    _api_retries = 2
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
 
@@ -157,6 +155,8 @@ class BaseModule:
         # track number of failures (for .api_request())
         self._api_request_failures = 0
 
+        self._default_api_retries = self.scan.config.get("web", {}).get("api_retries", 2)
+
         self._tasks = []
         self._event_received = None
 
@@ -340,7 +340,7 @@ class BaseModule:
 
     @property
     def api_retries(self):
-        return max(self._api_retries + 1, len(self._api_keys))
+        return max(self._default_api_retries + 1, len(self._api_keys))
 
     @property
     def api_failure_abort_threshold(self):
@@ -1212,7 +1212,8 @@ class BaseModule:
         return url, requests_kwargs
 
     def _api_response_is_success(self, r):
-        return r.is_success
+        # 404s typically indicate no data rather than an actual error with the API, so we don't want to retry them
+        return getattr(r, "is_success", False) or getattr(r, "status_code", 0) == 404
 
     async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):
         """

bbot/modules/shodan_idb.py

@@ -1,4 +1,5 @@
 from bbot.modules.base import BaseModule
+import time
 
 
 class shodan_idb(BaseModule):
@@ -46,23 +47,48 @@ class shodan_idb(BaseModule):
         "created_date": "2023-12-22",
         "author": "@TheTechromancer",
     }
+    options = {"retries": None}
+    options_desc = {
+        "retries": "How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting."
+    }
 
-    # we get lots of 404s, that's normal
+    # we typically don't want to abort this module
     _api_failure_abort_threshold = 9999999999
 
-    # there aren't any rate limits to speak of, so our outgoing queue can be pretty big
-    _qsize = 500
+    # since there are rate limits, we set a lower qsize
+    # this way when our queue is full, we can give the API a break
+    _qsize = 100
 
     base_url = "https://internetdb.shodan.io"
 
+    async def setup(self):
+        await super().setup()
+        self.last_request_time = 0
+        return True
+
     def _incoming_dedup_hash(self, event):
         return hash(self.get_ip(event))
 
+    @property
+    def api_retries(self):
+        # allow the module to override global retry setting
+        return self.config.get("retries", None) or super().api_retries
+
     async def handle_event(self, event):
         ip = self.get_ip(event)
         if ip is None:
             return
         url = f"{self.base_url}/{ip}"
+
+        # Rate limiting: ensure at least 1 second between requests
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < 1:
+            await self.helpers.sleep(1 - time_since_last)
+
+        # Update the last request time
+        self.last_request_time = time.time()
+
         r = await self.api_request(url)
         if r is None:
             self.debug(f"No response for {event.data}")

bbot/modules/templates/webhook.py

@@ -16,10 +16,9 @@ class WebhookOutputModule(BaseOutputModule):
     # abort module after 10 failed requests (not including retries)
     _api_failure_abort_threshold = 10
     # retry each request up to 10 times, respecting the Retry-After header
-    _api_retries = 10
+    _default_api_retries = 10
 
     async def setup(self):
-        self._api_retries = self.config.get("retries", 10)
         self.webhook_url = self.config.get("webhook_url", "")
         self.min_severity = self.config.get("min_severity", "LOW").strip().upper()
         assert self.min_severity in self.vuln_severities, (
@@ -31,6 +30,10 @@ class WebhookOutputModule(BaseOutputModule):
             return False
         return await super().setup()
 
+    @property
+    def api_retries(self):
+        return self.config.get("retries", self._default_api_retries)
+
     async def handle_event(self, event):
         message = self.format_message(event)
         data = {self.content_key: message}

{bbot-2.4.1.6100rc0.dist-info → bbot-2.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.1.6100rc0
+Version: 2.4.2
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.1.6100rc0.dist-info → bbot-2.4.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=5nN5mwrxsCWyzy0Rob1W3pW2f7A0ZLPueXgx-n5Lamo,163
+bbot/__init__.py,sha256=X4cKAUGMeCpzWhqGXpbsi6477sFR-gAsB5Ka6lEjahA,156
 bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -48,7 +48,7 @@ bbot/core/modules.py,sha256=U0Z2UoZAOPG9lLvR9Juc3UwdWCc_xbktF4t_NoiKPrY,31385
 bbot/core/multiprocess.py,sha256=ocQHanskJ09gHwe7RZmwNdZyCOQyeyUoIHCtLbtvXUk,1771
 bbot/core/shared_deps.py,sha256=mCMZeKSt46trzVqQDPGfXfEWg0Zw5YjiJx4BnsIRgHM,7640
 bbot/db/sql/models.py,sha256=SrUdDOBCICzXJBY29p0VvILhMQ1JCuh725bqvIYogX0,4884
-bbot/defaults.yml,sha256=r8nRJNwMlY59Ihigysa_pee6dm3rs69_U1ztpV6kbjs,6904
+bbot/defaults.yml,sha256=wIiOAEvsnJi_Q7qXC1Pn1TCSWiSh8CagpwVvVIGefLs,7106
 bbot/errors.py,sha256=xwQcD26nU9oc7-o0kv5jmEDTInmi8_W8eKAgQZZxdVM,953
 bbot/logger.py,sha256=wE-532v5FyKuSSoTdyW1xSfaOnLZB1axAJnB-uW2xrI,2745
 bbot/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -61,7 +61,7 @@ bbot/modules/baddns.py,sha256=ubO3KDfcIMJnMjyZX5FWZ4GWxLSekV_JQV7QvsPjtD0,6693
 bbot/modules/baddns_direct.py,sha256=hWThpkXP87nnCRTlUh5qBJ1t4eo4l9kUmKNNxVNJI8A,3819
 bbot/modules/baddns_zone.py,sha256=y1XaBUfFPnRbR2qaTqRyUsPgEL73722v2B8aS5YoGN4,1035
 bbot/modules/badsecrets.py,sha256=LG37p48Rlxsfc3BmACMpkypsbuFTVvXqNhlP1IEsx0k,5109
-bbot/modules/base.py,sha256=ehkGjo07bL0QKP-fW64eOLqES2GBSq9ZkM7VkSuOBhI,75038
+bbot/modules/base.py,sha256=_RFTPNZ6X7mD7jBOTYZmjgzTSKWT9U_eYxL4gXIh8TE,75265
 bbot/modules/bevigil.py,sha256=0VLIxmeXRUI2-EoR6IzuHJMcX8KCHNNta-WYa3gVlDg,2862
 bbot/modules/binaryedge.py,sha256=5F9LnZwRM_rZnzTv29hLJLI2GEQdzOwSpahPFC1kJC0,1397
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
@@ -177,7 +177,7 @@ bbot/modules/robots.py,sha256=LGG6ixsxrlaCk-mi4Lp6kB2RB1v-25NhTAQxdQEtH8s,2172
 bbot/modules/securitytrails.py,sha256=5Jk_HTQP8FRq6A30sN19FU79uLJt7aiOsI2dxNkLDcM,1148
 bbot/modules/securitytxt.py,sha256=nwaTOnRAl2NWcEc3i_I9agB56QjqK8dHqiKRHPPkCPE,4558
 bbot/modules/shodan_dns.py,sha256=ETOyUhCiAETlGUAQhvAP47oEEPYss7fm_F_CAeCQyoI,842
-bbot/modules/shodan_idb.py,sha256=kyPbGI68BrLgCab_T5emKXBoyD2OVn59K0J1V6CNR4s,5213
+bbot/modules/shodan_idb.py,sha256=PB5vplwpjPjbfoMeez333rWYqFY5RuMp40ZN9hNCJAg,6088
 bbot/modules/sitedossier.py,sha256=MR9fSkgE_3YGsUe7M-TyJ2GdpBtF3oLKDl9agAwzw5U,2284
 bbot/modules/skymem.py,sha256=ZrxWcePFTCiDkFeAc3YLegFG-Tgw4C9af_JHiVonk84,1930
 bbot/modules/smuggler.py,sha256=v8NCRgzd7wpEFZJUTAArG04bN8nNTGiHxYpGBapzi14,1580
@@ -192,7 +192,7 @@ bbot/modules/templates/postman.py,sha256=MIpz2q_r6LP0kIEgByo7oX5qHhMZLOhr7oKzJI9
 bbot/modules/templates/shodan.py,sha256=MXBvlmfw3jZFqT47v10UkqMSyQR-zBIxMJmK7PWw6uw,1174
 bbot/modules/templates/sql.py,sha256=o-CdyyoJvHJdJBKkj3CIGXYxUta4w2AB_2Vr-k7cDDU,3553
 bbot/modules/templates/subdomain_enum.py,sha256=epyKSly08jqaINV_AMMWbNafIeQjJqvd3aj63KD0Mck,8402
-bbot/modules/templates/webhook.py,sha256=0N7LorCxT0sVmT2XgrV81ubdcdPVW8s2X__BrKC2AcM,3313
+bbot/modules/templates/webhook.py,sha256=uGFmcJ81GzGN1UI2k2O7nQF_fyh4ehLDEg2NSXaPnhk,3373
 bbot/modules/trickest.py,sha256=MRgLW0YiDWzlWdAjyqfPPLFb-a51r-Ffn_dphiJI_gA,1550
 bbot/modules/trufflehog.py,sha256=KEObs5z0ce0Ck7gA3t3l66MJKUo6P_9TIfVOwFO28t4,8743
 bbot/modules/url_manipulation.py,sha256=4J3oFkqTSJPPmbKEKAHJg2Q2w4QNKtQhiN03ZJq5VtI,4326
@@ -428,8 +428,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.4.1.6100rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.4.1.6100rc0.dist-info/METADATA,sha256=ZjY7PWqXvYjN9vkB-WcSvyyIYvL26G4JTc-stO6VfFM,18218
-bbot-2.4.1.6100rc0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-bbot-2.4.1.6100rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.4.1.6100rc0.dist-info/RECORD,,
+bbot-2.4.2.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.4.2.dist-info/METADATA,sha256=ANe9OKQmUf_JvxIYa51HOhBw1THI3dGf2aM8g3uRsn4,18210
+bbot-2.4.2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+bbot-2.4.2.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.4.2.dist-info/RECORD,,