bbot 2.3.2.5950rc0__tar.gz → 2.3.2.5958rc0__tar.gz

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.3.2.5950rc0
+Version: 2.3.2.5958rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.2.5950rc"
+__version__ = "v2.3.2.5958rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/modules/base.py

@@ -71,7 +71,7 @@ class BaseModule:
 
         _qsize (int): Outgoing queue size (0 for infinite). Default is 0.
 
-        _priority (int): Priority level of events raised by this module, 1-5. Default is 3.
+        _priority (int): Priority level of the module. Lower values are higher priority. Default is 3.
 
         _name (str): Module name, overridden automatically. Default is 'base'.
 

bbot-2.3.2.5958rc0/bbot/modules/crt.py

@@ -0,0 +1,66 @@
+import time
+import asyncpg
+
+from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+
+class crt(subdomain_enum):
+    flags = ["subdomain-enum", "passive", "safe"]
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    meta = {
+        "description": "Query crt.sh (certificate transparency) for subdomains",
+        "created_date": "2022-05-13",
+        "author": "@TheTechromancer",
+    }
+
+    deps_pip = ["asyncpg"]
+
+    db_host = "crt.sh"
+    db_port = 5432
+    db_user = "guest"
+    db_name = "certwatch"
+    reject_wildcards = False
+
+    async def setup(self):
+        self.db_conn = None
+        return await super().setup()
+
+    async def request_url(self, query):
+        if not self.db_conn:
+            self.db_conn = await asyncpg.connect(
+                host=self.db_host, port=self.db_port, user=self.db_user, database=self.db_name
+            )
+
+        sql = """
+        WITH ci AS (
+            SELECT array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES
+            FROM (
+                SELECT DISTINCT cai.CERTIFICATE, cai.NAME_VALUE
+                FROM certificate_and_identities cai
+                WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
+                    AND cai.NAME_VALUE ILIKE ('%.' || $1)
+                LIMIT 50000
+            ) sub
+            GROUP BY sub.CERTIFICATE
+        )
+        SELECT DISTINCT unnest(NAME_VALUES) as name_value FROM ci;
+        """
+        start = time.time()
+        results = await self.db_conn.fetch(sql, query)
+        end = time.time()
+        self.verbose(f"SQL query executed in: {end - start} seconds with {len(results):,} results")
+        return results
+
+    async def parse_results(self, results, query):
+        domains = set()
+        for row in results:
+            domain = row["name_value"]
+            if domain:
+                for d in domain.splitlines():
+                    domains.add(d.lower())
+        return domains
+
+    async def cleanup(self):
+        if self.db_conn:
+            await self.db_conn.close()

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/modules/deadly/ffuf.py

@@ -18,6 +18,7 @@ class ffuf(BaseModule):
         "max_depth": 0,
         "extensions": "",
         "ignore_case": False,
+        "rate": 0,
     }
 
     options_desc = {
@@ -26,6 +27,7 @@ class ffuf(BaseModule):
         "max_depth": "the maximum directory depth to attempt to solve",
         "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
         "ignore_case": "Only put lowercase words into the wordlist",
+        "rate": "Rate of requests per second (default: 0)",
     }
 
     deps_common = ["ffuf"]
@@ -43,6 +45,7 @@ class ffuf(BaseModule):
         self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
+        self.rate = self.config.get("rate", 0)
         self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]")
         try:
             self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True)
@@ -245,6 +248,9 @@ class ffuf(BaseModule):
                 self.debug("invalid mode specified, aborting")
                 return
 
+            if self.rate > 0:
+                command += ["-rate", f"{self.rate}"]
+
             if self.proxy:
                 command += ["-x", self.proxy]
 

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/modules/ffuf_shortnames.py

@@ -26,6 +26,7 @@ class ffuf_shortnames(ffuf):
         "find_delimiters": True,
         "find_subwords": False,
         "max_predictions": 250,
+        "rate": 0,
     }
 
     options_desc = {
@@ -38,6 +39,7 @@ class ffuf_shortnames(ffuf):
         "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them",
         "find_subwords": "Attempt to detect subwords and make additional ffuf runs against them",
         "max_predictions": "The maximum number of predictions to generate per shortname prefix",
+        "rate": "Rate of requests per second (default: 0)",
     }
 
     deps_pip = ["numpy"]
@@ -96,6 +98,7 @@ class ffuf_shortnames(ffuf):
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")
+        self.rate = self.config.get("rate", 0)
 
         class MinimalWordPredictor:
             def __init__(self):

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/modules/portfilter.py

@@ -18,6 +18,8 @@ class portfilter(BaseInterceptModule):
         "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
 
+    _priority = 4
+
     async def setup(self):
         self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
         self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", "").strip()

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/bbot/scanner/scanner.py

@@ -289,6 +289,7 @@ class Scanner:
 
             # intercept modules get sewn together like human centipede
             self.intercept_modules = [m for m in self.modules.values() if m._intercept]
+            self.intercept_modules.sort(key=lambda x: x.priority)
             for i, intercept_module in enumerate(self.intercept_modules[1:]):
                 prev_intercept_module = self.intercept_modules[i]
                 self.debug(

bbot-2.3.2.5958rc0/bbot/test/test_step_2/module_tests/test_module_crt.py

@@ -0,0 +1,25 @@
+from .base import ModuleTestBase
+
+
+class TestCRT(ModuleTestBase):
+    async def setup_after_prep(self, module_test):
+        class AsyncMock:
+            async def fetch(self, *args, **kwargs):
+                print("mock_fetch", args, kwargs)
+                return [
+                    {"name_value": "asdf.blacklanternsecurity.com"},
+                    {"name_value": "zzzz.blacklanternsecurity.com"},
+                ]
+
+            async def close(self):
+                pass
+
+        async def mock_connect(*args, **kwargs):
+            print("mock_connect", args, kwargs)
+            return AsyncMock()
+
+        module_test.monkeypatch.setattr("asyncpg.connect", mock_connect)
+
+    def check(self, module_test, events):
+        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
+        assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"

{bbot-2.3.2.5950rc0 → bbot-2.3.2.5958rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.3.2.5950rc"
+version = "v2.3.2.5958rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",

bbot-2.3.2.5950rc0/bbot/modules/crt.py

@@ -1,40 +0,0 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
-
-
-class crt(subdomain_enum):
-    flags = ["subdomain-enum", "passive", "safe"]
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    meta = {
-        "description": "Query crt.sh (certificate transparency) for subdomains",
-        "created_date": "2022-05-13",
-        "author": "@TheTechromancer",
-    }
-
-    base_url = "https://crt.sh"
-    reject_wildcards = False
-
-    async def setup(self):
-        self.cert_ids = set()
-        return await super().setup()
-
-    async def request_url(self, query):
-        params = {"q": f"%.{query}", "output": "json"}
-        url = self.helpers.add_get_params(self.base_url, params).geturl()
-        return await self.api_request(url, timeout=self.http_timeout + 30)
-
-    async def parse_results(self, r, query):
-        results = set()
-        j = r.json()
-        for cert_info in j:
-            if not type(cert_info) == dict:
-                continue
-            cert_id = cert_info.get("id")
-            if cert_id:
-                if hash(cert_id) not in self.cert_ids:
-                    self.cert_ids.add(hash(cert_id))
-                    domain = cert_info.get("name_value")
-                    if domain:
-                        for d in domain.splitlines():
-                            results.add(d.lower())
-        return results

bbot-2.3.2.5950rc0/bbot/test/test_step_2/module_tests/test_module_crt.py

@@ -1,15 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestCRT(ModuleTestBase):
-    async def setup_after_prep(self, module_test):
-        module_test.module.abort_if = lambda e: False
-        for t in self.targets:
-            module_test.httpx_mock.add_response(
-                url="https://crt.sh?q=%25.blacklanternsecurity.com&output=json",
-                json=[{"id": 1, "name_value": "asdf.blacklanternsecurity.com\nzzzz.blacklanternsecurity.com"}],
-            )
-
-    def check(self, module_test, events):
-        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
-        assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"