bbot 2.4.1.6095rc0__py3-none-any.whl → 2.4.1.6100rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.1.6095rc"
+__version__ = "v2.4.1.6100rc"
 
 from .scanner import Scanner, Preset
 

bbot/defaults.yml

@@ -96,6 +96,10 @@ web:
   http_retries: 1
   # HTTP retries (for httpx)
   httpx_retries: 1
+  # Default sleep interval when rate limited by 429 (and retry-after isn't provided)
+  429_sleep_interval: 30
+  # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)
+  429_max_sleep_interval: 60
   # Enable/disable debug messages for web requests/responses
   debug: false
   # Maximum number of HTTP redirects to follow

bbot/modules/base.py

@@ -108,8 +108,6 @@ class BaseModule:
     _api_retries = 2
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
-    # sleep for this many seconds after being rate limited
-    _429_sleep_interval = 30
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -165,6 +163,10 @@ class BaseModule:
         # used for optional "per host" tracking
         self._per_host_tracker = set()
 
+        # 429 rate limit handling
+        self._429_sleep_interval = self.scan.web_config.get("429_sleep_interval", 30)
+        self._429_max_sleep_interval = self.scan.web_config.get("429_max_sleep_interval", 60)
+
     async def setup(self):
         """
         Performs one-time setup tasks for the module.
@@ -1172,6 +1174,11 @@ class BaseModule:
                     retry_after = self._get_retry_after(r)
                     if retry_after or status_code == 429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
+                        if retry_after and retry_after > self._429_max_sleep_interval:
+                            self.verbose(
+                                f"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead"
+                            )
+                            sleep_interval = self._429_max_sleep_interval
                         self.verbose(
                             f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})"
                         )

bbot/modules/certspotter.py

@@ -15,7 +15,7 @@ class certspotter(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
-        return self.api_request(url, timeout=self.http_timeout + 30)
+        return self.api_request(url)
 
     async def parse_results(self, r, query):
         results = set()

{bbot-2.4.1.6095rc0.dist-info → bbot-2.4.1.6100rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.1.6095rc0
+Version: 2.4.1.6100rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.1.6095rc0.dist-info → bbot-2.4.1.6100rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=IWliuFZfjWjwG33exEqeUFSip4K0QyS7Ah6FYizaqJI,163
+bbot/__init__.py,sha256=5nN5mwrxsCWyzy0Rob1W3pW2f7A0ZLPueXgx-n5Lamo,163
 bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -48,7 +48,7 @@ bbot/core/modules.py,sha256=U0Z2UoZAOPG9lLvR9Juc3UwdWCc_xbktF4t_NoiKPrY,31385
 bbot/core/multiprocess.py,sha256=ocQHanskJ09gHwe7RZmwNdZyCOQyeyUoIHCtLbtvXUk,1771
 bbot/core/shared_deps.py,sha256=mCMZeKSt46trzVqQDPGfXfEWg0Zw5YjiJx4BnsIRgHM,7640
 bbot/db/sql/models.py,sha256=SrUdDOBCICzXJBY29p0VvILhMQ1JCuh725bqvIYogX0,4884
-bbot/defaults.yml,sha256=XPaGfTKWFjKV-lLIJy-Qs7X0d-8X8EYM0QELEu8KjBw,6670
+bbot/defaults.yml,sha256=r8nRJNwMlY59Ihigysa_pee6dm3rs69_U1ztpV6kbjs,6904
 bbot/errors.py,sha256=xwQcD26nU9oc7-o0kv5jmEDTInmi8_W8eKAgQZZxdVM,953
 bbot/logger.py,sha256=wE-532v5FyKuSSoTdyW1xSfaOnLZB1axAJnB-uW2xrI,2745
 bbot/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -61,7 +61,7 @@ bbot/modules/baddns.py,sha256=ubO3KDfcIMJnMjyZX5FWZ4GWxLSekV_JQV7QvsPjtD0,6693
 bbot/modules/baddns_direct.py,sha256=hWThpkXP87nnCRTlUh5qBJ1t4eo4l9kUmKNNxVNJI8A,3819
 bbot/modules/baddns_zone.py,sha256=y1XaBUfFPnRbR2qaTqRyUsPgEL73722v2B8aS5YoGN4,1035
 bbot/modules/badsecrets.py,sha256=LG37p48Rlxsfc3BmACMpkypsbuFTVvXqNhlP1IEsx0k,5109
-bbot/modules/base.py,sha256=SCIeaLX1d5VMUl2nC3IyqPdzBF8mr_7__JB3855q5cc,74529
+bbot/modules/base.py,sha256=ehkGjo07bL0QKP-fW64eOLqES2GBSq9ZkM7VkSuOBhI,75038
 bbot/modules/bevigil.py,sha256=0VLIxmeXRUI2-EoR6IzuHJMcX8KCHNNta-WYa3gVlDg,2862
 bbot/modules/binaryedge.py,sha256=5F9LnZwRM_rZnzTv29hLJLI2GEQdzOwSpahPFC1kJC0,1397
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
@@ -75,7 +75,7 @@ bbot/modules/builtwith.py,sha256=6ZQOc6vmSVc8LsdgsiuMWfDquGm5K0jxwsnL8MsKNWw,538
 bbot/modules/bypass403.py,sha256=HyONgOYlZUET61FZ0QWE7zPGG-N6n0x_j9KUGw8kVxQ,6855
 bbot/modules/c99.py,sha256=yHT9-eyqRODISV5eTi11uh-xwqX0JG7zey5AgcDYCdI,1448
 bbot/modules/censys.py,sha256=J7NhPnSeoCuG_9FkLjFBIg-tqHAB21HjvQUw_6OQNZo,3311
-bbot/modules/certspotter.py,sha256=AtL5BiOuDp4vu1-5fct4aQAGZM2qiODYsbgBsw0phoU,937
+bbot/modules/certspotter.py,sha256=qdRGCkGyP07_cP9h2o_AEZwoiQPXtrC-Bel3vgh24x8,905
 bbot/modules/chaos.py,sha256=JyuwytwE3IRmNbw-uyJ0gCaTnywhhsHzTiZ3OJ15PAw,1573
 bbot/modules/code_repository.py,sha256=x70Z45VnNNMF8BPkHfGWZXsZXw_fStGB3y0-8jbP1Ns,2078
 bbot/modules/credshed.py,sha256=HAF5wgRGKIIpdMAe4mIAtkZRLmFYjMFyXtjjst6RJ20,4203
@@ -428,8 +428,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.4.1.6095rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.4.1.6095rc0.dist-info/METADATA,sha256=s1SYf_crJYiFaXoX6ZM7VtItSyo5A6n2TY5XOp68Mdc,18218
-bbot-2.4.1.6095rc0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-bbot-2.4.1.6095rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.4.1.6095rc0.dist-info/RECORD,,
+bbot-2.4.1.6100rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.4.1.6100rc0.dist-info/METADATA,sha256=ZjY7PWqXvYjN9vkB-WcSvyyIYvL26G4JTc-stO6VfFM,18218
+bbot-2.4.1.6100rc0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+bbot-2.4.1.6100rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.4.1.6100rc0.dist-info/RECORD,,