bbot 2.3.2.5944rc0__py3-none-any.whl → 2.3.2.5958rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.2.5944rc"
+__version__ = "v2.3.2.5958rc"
 
 from .scanner import Scanner, Preset

bbot/modules/base.py

@@ -71,7 +71,7 @@ class BaseModule:
 
         _qsize (int): Outgoing queue size (0 for infinite). Default is 0.
 
-        _priority (int): Priority level of events raised by this module, 1-5. Default is 3.
+        _priority (int): Priority level of the module. Lower values are higher priority. Default is 3.
 
         _name (str): Module name, overridden automatically. Default is 'base'.
 

bbot/modules/crt.py

@@ -1,3 +1,6 @@
+import time
+import asyncpg
+
 from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
@@ -11,30 +14,53 @@ class crt(subdomain_enum):
         "author": "@TheTechromancer",
     }
 
-    base_url = "https://crt.sh"
+    deps_pip = ["asyncpg"]
+
+    db_host = "crt.sh"
+    db_port = 5432
+    db_user = "guest"
+    db_name = "certwatch"
     reject_wildcards = False
 
     async def setup(self):
-        self.cert_ids = set()
+        self.db_conn = None
         return await super().setup()
 
     async def request_url(self, query):
-        params = {"q": f"%.{query}", "output": "json"}
-        url = self.helpers.add_get_params(self.base_url, params).geturl()
-        return await self.api_request(url, timeout=self.http_timeout + 30)
-
-    async def parse_results(self, r, query):
-        results = set()
-        j = r.json()
-        for cert_info in j:
-            if not type(cert_info) == dict:
-                continue
-            cert_id = cert_info.get("id")
-            if cert_id:
-                if hash(cert_id) not in self.cert_ids:
-                    self.cert_ids.add(hash(cert_id))
-                    domain = cert_info.get("name_value")
-                    if domain:
-                        for d in domain.splitlines():
-                            results.add(d.lower())
+        if not self.db_conn:
+            self.db_conn = await asyncpg.connect(
+                host=self.db_host, port=self.db_port, user=self.db_user, database=self.db_name
+            )
+
+        sql = """
+        WITH ci AS (
+            SELECT array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES
+            FROM (
+                SELECT DISTINCT cai.CERTIFICATE, cai.NAME_VALUE
+                FROM certificate_and_identities cai
+                WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
+                    AND cai.NAME_VALUE ILIKE ('%.' || $1)
+                LIMIT 50000
+            ) sub
+            GROUP BY sub.CERTIFICATE
+        )
+        SELECT DISTINCT unnest(NAME_VALUES) as name_value FROM ci;
+        """
+        start = time.time()
+        results = await self.db_conn.fetch(sql, query)
+        end = time.time()
+        self.verbose(f"SQL query executed in: {end - start} seconds with {len(results):,} results")
         return results
+
+    async def parse_results(self, results, query):
+        domains = set()
+        for row in results:
+            domain = row["name_value"]
+            if domain:
+                for d in domain.splitlines():
+                    domains.add(d.lower())
+        return domains
+
+    async def cleanup(self):
+        if self.db_conn:
+            await self.db_conn.close()

bbot/modules/deadly/ffuf.py

@@ -18,6 +18,7 @@ class ffuf(BaseModule):
         "max_depth": 0,
         "extensions": "",
         "ignore_case": False,
+        "rate": 0,
     }
 
     options_desc = {
@@ -26,6 +27,7 @@ class ffuf(BaseModule):
         "max_depth": "the maximum directory depth to attempt to solve",
         "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
         "ignore_case": "Only put lowercase words into the wordlist",
+        "rate": "Rate of requests per second (default: 0)",
     }
 
     deps_common = ["ffuf"]
@@ -43,6 +45,7 @@ class ffuf(BaseModule):
         self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
+        self.rate = self.config.get("rate", 0)
         self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]")
         try:
             self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True)
@@ -245,6 +248,9 @@ class ffuf(BaseModule):
                 self.debug("invalid mode specified, aborting")
                 return
 
+            if self.rate > 0:
+                command += ["-rate", f"{self.rate}"]
+
             if self.proxy:
                 command += ["-x", self.proxy]
 

bbot/modules/ffuf_shortnames.py

@@ -26,6 +26,7 @@ class ffuf_shortnames(ffuf):
         "find_delimiters": True,
         "find_subwords": False,
         "max_predictions": 250,
+        "rate": 0,
     }
 
     options_desc = {
@@ -38,6 +39,7 @@ class ffuf_shortnames(ffuf):
         "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them",
         "find_subwords": "Attempt to detect subwords and make additional ffuf runs against them",
         "max_predictions": "The maximum number of predictions to generate per shortname prefix",
+        "rate": "Rate of requests per second (default: 0)",
     }
 
     deps_pip = ["numpy"]
@@ -96,6 +98,7 @@ class ffuf_shortnames(ffuf):
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")
+        self.rate = self.config.get("rate", 0)
 
         class MinimalWordPredictor:
             def __init__(self):

bbot/modules/portfilter.py

@@ -18,6 +18,8 @@ class portfilter(BaseInterceptModule):
         "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
 
+    _priority = 4
+
     async def setup(self):
         self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
         self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", "").strip()

bbot/scanner/scanner.py

@@ -289,6 +289,7 @@ class Scanner:
 
             # intercept modules get sewn together like human centipede
             self.intercept_modules = [m for m in self.modules.values() if m._intercept]
+            self.intercept_modules.sort(key=lambda x: x.priority)
             for i, intercept_module in enumerate(self.intercept_modules[1:]):
                 prev_intercept_module = self.intercept_modules[i]
                 self.debug(

bbot/test/test_step_2/module_tests/test_module_crt.py

@@ -3,12 +3,22 @@ from .base import ModuleTestBase
 
 class TestCRT(ModuleTestBase):
     async def setup_after_prep(self, module_test):
-        module_test.module.abort_if = lambda e: False
-        for t in self.targets:
-            module_test.httpx_mock.add_response(
-                url="https://crt.sh?q=%25.blacklanternsecurity.com&output=json",
-                json=[{"id": 1, "name_value": "asdf.blacklanternsecurity.com\nzzzz.blacklanternsecurity.com"}],
-            )
+        class AsyncMock:
+            async def fetch(self, *args, **kwargs):
+                print("mock_fetch", args, kwargs)
+                return [
+                    {"name_value": "asdf.blacklanternsecurity.com"},
+                    {"name_value": "zzzz.blacklanternsecurity.com"},
+                ]
+
+            async def close(self):
+                pass
+
+        async def mock_connect(*args, **kwargs):
+            print("mock_connect", args, kwargs)
+            return AsyncMock()
+
+        module_test.monkeypatch.setattr("asyncpg.connect", mock_connect)
 
     def check(self, module_test, events):
         assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"

{bbot-2.3.2.5944rc0.dist-info → bbot-2.3.2.5958rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.3.2.5944rc0
+Version: 2.3.2.5958rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.3.2.5944rc0.dist-info → bbot-2.3.2.5958rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=hBISVROF7aBils6i28vrQzWohknYZpMQBE0w2_JRRb4,130
+bbot/__init__.py,sha256=zs6Wq8dI1ItOD6H659-keQl5C_cvZ-SbzZvFI_4h338,130
 bbot/cli.py,sha256=H7L3yIoYuSpkrf1K1qqiBMQBTyt8UI9FfConpjqr7KI,11029
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -61,7 +61,7 @@ bbot/modules/baddns.py,sha256=y6zTuE668MvNdaQszmkwYSwWzrjqit5X3zf5kLJbQC8,6692
 bbot/modules/baddns_direct.py,sha256=yoOSqi4Z1ZwrycGbwklCukDBjWRWlz6JV67v_xVwHbE,3818
 bbot/modules/baddns_zone.py,sha256=IcewDBtA_-64NCNFojEFd9jt2YBek6ltB2mmqdDH6LE,1034
 bbot/modules/badsecrets.py,sha256=LG37p48Rlxsfc3BmACMpkypsbuFTVvXqNhlP1IEsx0k,5109
-bbot/modules/base.py,sha256=fm-Z1mhqgYkJI_PEZwslMS9ThRsKIBTDvQAc1CWU9HI,72775
+bbot/modules/base.py,sha256=ufqXGnY6W8-umqvaU5om6ghijv9sfJbqBNvnGQZzkM4,72786
 bbot/modules/bevigil.py,sha256=0VLIxmeXRUI2-EoR6IzuHJMcX8KCHNNta-WYa3gVlDg,2862
 bbot/modules/binaryedge.py,sha256=5F9LnZwRM_rZnzTv29hLJLI2GEQdzOwSpahPFC1kJC0,1397
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
@@ -79,9 +79,9 @@ bbot/modules/certspotter.py,sha256=AtL5BiOuDp4vu1-5fct4aQAGZM2qiODYsbgBsw0phoU,9
 bbot/modules/chaos.py,sha256=JyuwytwE3IRmNbw-uyJ0gCaTnywhhsHzTiZ3OJ15PAw,1573
 bbot/modules/code_repository.py,sha256=x70Z45VnNNMF8BPkHfGWZXsZXw_fStGB3y0-8jbP1Ns,2078
 bbot/modules/credshed.py,sha256=HAF5wgRGKIIpdMAe4mIAtkZRLmFYjMFyXtjjst6RJ20,4203
-bbot/modules/crt.py,sha256=6Zm90VKXwYYN6Sab0gwwhTARrtnQIqALJTVtFWMMTGk,1369
+bbot/modules/crt.py,sha256=-_ZTmQofCqhtGl-Xp4kWmDndivAKZx71I1JwHWjl4Rg,2048
 bbot/modules/deadly/dastardly.py,sha256=dxPkJUfAsuddDDuI_uVyTUxkJ5eps92nSrPtpBOTlQg,5315
-bbot/modules/deadly/ffuf.py,sha256=sCledEgIi1ZgAhb-XlwbLIH3wBD5dSELZAEun_pNLOk,14745
+bbot/modules/deadly/ffuf.py,sha256=VVehiHYEieVMuOjoAXD32NdgATW3aGg5PFm8NlQBjK0,14955
 bbot/modules/deadly/nuclei.py,sha256=-Ne3NMQwa55oMG_AkYWyxQMOA1j6Omkk7VHdSjh8isU,17808
 bbot/modules/deadly/vhost.py,sha256=m7RdR0w7Hs38IGVHUu_3Er-_5ABVdalRG_8znQepxD0,5456
 bbot/modules/dehashed.py,sha256=iyzWHmJs6zC7FsRhw9_AdkckQKCf_0oNnL9RwG409r0,5071
@@ -98,7 +98,7 @@ bbot/modules/dockerhub.py,sha256=JQkujjqvQRzQuvHjQ7JbFs_VlJj8dLRPRObAkBgUQhc,349
 bbot/modules/dotnetnuke.py,sha256=zipcHyNYr2FEecStb1Yrm938ps01RvHV8NnyqAvnGGc,10537
 bbot/modules/emailformat.py,sha256=RLPJW-xitYB-VT4Lp08qVzFkXx_kMyV_035JT_Yf4fM,1082
 bbot/modules/extractous.py,sha256=VSGKmHPAA_4r62jaN8Yqi3JcjehjxpI2lhe8i2j786s,4648
-bbot/modules/ffuf_shortnames.py,sha256=iJnm6bvFyQXfzOGnUioficE6v59QkaErY7mo4m6m7o0,18618
+bbot/modules/ffuf_shortnames.py,sha256=VxAxSrkU_XqRrOdv03YgwPLgW4OD6VcVwYpyQ7kt6_U,18744
 bbot/modules/filedownload.py,sha256=TOxftfxguaRSEKI8oG79XVRQqUGg1_IhYDYl_Jw9eYc,8694
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=zeehQb9akBSbHW9dF4icH8Vfd8LqoTrpIvnQEEMWes8,1311
@@ -165,7 +165,7 @@ bbot/modules/paramminer_getparams.py,sha256=_j6rgaqV5wGJoa8p5-KKbe2YsVGUtmWIanCV
 bbot/modules/paramminer_headers.py,sha256=CuiqT3nY3eIrZhT4CAEkRDfCbCYci8CmnXsippf3Qb4,10450
 bbot/modules/passivetotal.py,sha256=uGT6c_CUxBNInmClsTg8afIYA2ZykKYYCgjkyzujfHg,1653
 bbot/modules/pgp.py,sha256=Xu2M9WEIlwTm5-Lv29g7BblI05tD9Dl0XsYSeY6UURs,2065
-bbot/modules/portfilter.py,sha256=FB1plaBTZxiJnLJ8RiyPyc2Hrh7N7_DRSiyfeQEAfms,1723
+bbot/modules/portfilter.py,sha256=4ZJ6tE0opC07D3xNCEvEcRfhT_rHk-GCN8x46CkavrQ,1742
 bbot/modules/portscan.py,sha256=dM2p3tBGQKSdeZRMlCpOwqRpz0XJfIMwY2n3Ajaig_M,13441
 bbot/modules/postman.py,sha256=vo761Nzu3kPBzfCY3KJcvsGEsjImaa7iA2z-LyASBDc,4589
 bbot/modules/postman_download.py,sha256=LUB9cP-otkB1HaNACGS5YPwsxnwp1uSo28SpGvmQ60A,3467
@@ -234,7 +234,7 @@ bbot/scanner/preset/conditions.py,sha256=hFL9cSIWGEsv2TfM5UGurf0c91cyaM8egb5IngB
 bbot/scanner/preset/environ.py,sha256=9KbEOLWkUdoAf5Ez_2A1NNm6QduQElbnNnrPi6VDhZs,4731
 bbot/scanner/preset/path.py,sha256=Q29MO8cOEn690yW6bB08P72kbZ3C-H_TOEoXuwWnFM8,2274
 bbot/scanner/preset/preset.py,sha256=Xmh9mkO4c985sOG8BGRwujBUmO1jCpHRC_Sgq2fD_dY,40805
-bbot/scanner/scanner.py,sha256=goJbtDF0wiNZxjwmaAnA1t_F4ueIg30kjsf9S1giXf4,54551
+bbot/scanner/scanner.py,sha256=O9LPaOa33yjdtolri3pcmQOBj4_8fMpMlwoYICZYK0M,54617
 bbot/scanner/stats.py,sha256=re93sArKXZSiD0Owgqk2J3Kdvfm3RL4Y9Qy_VOcaVk8,3623
 bbot/scanner/target.py,sha256=EQwtFZLDeNlqt8JupyBEksqeQ_c_i3NARSWf3mQQC4k,12128
 bbot/scripts/docs.py,sha256=ZLY9-O6OeEElzOUvTglO5EMkRv1s4aEuxJb2CthCVsI,10782
@@ -305,7 +305,7 @@ bbot/test/test_step_2/module_tests/test_module_chaos.py,sha256=9JRgtDEnnJgmEMCTB
 bbot/test/test_step_2/module_tests/test_module_cloudcheck.py,sha256=9KjGREpzOVByDVjIEWoaWbS3RwPlYLN3mw-OnRvD7sg,4083
 bbot/test/test_step_2/module_tests/test_module_code_repository.py,sha256=i02Tgvr_F9_E4d6aEaXrfdk71NkoDvjzP4C98l2rNGg,2414
 bbot/test/test_step_2/module_tests/test_module_credshed.py,sha256=ipkCFL7YmZBLWWoGyGr98saL_yh3E99EnLtagHqdY1g,3360
-bbot/test/test_step_2/module_tests/test_module_crt.py,sha256=V15tE1jcXdXJEzEEdAJvSMRWhKBFtxBBUJ_eewvV3U4,717
+bbot/test/test_step_2/module_tests/test_module_crt.py,sha256=HOx8z9ZUWAvqhso3_xfgbUElL-KHw3yX_LgHTBGObPI,944
 bbot/test/test_step_2/module_tests/test_module_csv.py,sha256=UJqMqdiPjx-UjJw10OoVMAj378wu5mWIq0v04TCljTM,579
 bbot/test/test_step_2/module_tests/test_module_dastardly.py,sha256=jjpJD9mdCcbaJgnG63xE-J_Qqpjt9hpm_WvfHDLfTsc,2353
 bbot/test/test_step_2/module_tests/test_module_dehashed.py,sha256=BZ0LFO4xBwDsXzsqjZCYRMTbXdwwUEma2OeJh8YQIEs,3625
@@ -425,8 +425,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.3.2.5944rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.3.2.5944rc0.dist-info/METADATA,sha256=XEPcMePFQCzHGB4gk2cT6pmxBgORqxqBzqfy_6pqEjs,18218
-bbot-2.3.2.5944rc0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-bbot-2.3.2.5944rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.3.2.5944rc0.dist-info/RECORD,,
+bbot-2.3.2.5958rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.3.2.5958rc0.dist-info/METADATA,sha256=lep2AMvMwyENAV8jiAp_X5sSGF8fCbGHXPaRSUp8yFU,18218
+bbot-2.3.2.5958rc0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+bbot-2.3.2.5958rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.3.2.5958rc0.dist-info/RECORD,,