bbot 2.3.0.5532rc0__py3-none-any.whl → 2.3.0.5546rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5532rc"
+__version__ = "v2.3.0.5546rc"
 
 from .scanner import Scanner, Preset

bbot/core/event/base.py

@@ -1232,11 +1232,25 @@ class URL_UNVERIFIED(BaseEvent):
         return data
 
     def add_tag(self, tag):
-        host_same_as_parent = self.parent and self.host == self.parent.host
-        if tag == "spider-danger" and host_same_as_parent and "spider-danger" not in self.tags:
-            # increment the web spider distance
-            if self.type == "URL_UNVERIFIED":
-                self.web_spider_distance += 1
+        self_url = getattr(self, "parsed_url", "")
+        self_host = getattr(self, "host", "")
+        # autoincrement web spider distance if the "spider-danger" tag is added
+        if tag == "spider-danger" and "spider-danger" not in self.tags and self_url and self_host:
+            parent_hosts_and_urls = set()
+            for p in self.get_parents():
+                # URL_UNVERIFIED events don't count because they haven't been visited yet
+                if p.type == "URL_UNVERIFIED":
+                    continue
+                url = getattr(p, "parsed_url", "")
+                parent_hosts_and_urls.add((p.host, url))
+            # if there's a URL anywhere in our parent chain that's different from ours but shares our host, we're in dAnGeR
+            dangerous_parent = any(
+                p_host == self.host and p_url != self_url for p_host, p_url in parent_hosts_and_urls
+            )
+            if dangerous_parent:
+                # increment the web spider distance
+                if self.type == "URL_UNVERIFIED":
+                    self.web_spider_distance += 1
                 if self.is_spider_max:
                     self.add_tag("spider-max")
         super().add_tag(tag)

bbot/test/test_step_1/test_events.py

@@ -808,6 +808,8 @@ async def test_event_discovery_context():
 async def test_event_web_spider_distance(bbot_scanner):
     # make sure web spider distance inheritance works as intended
     # and we don't have any runaway situations with SOCIAL events + URLs
+
+    # URL_UNVERIFIED events should not increment web spider distance
     scan = bbot_scanner(config={"web": {"spider_distance": 1}})
     url_event_1 = scan.make_event("http://www.evilcorp.com/test1", "URL_UNVERIFIED", parent=scan.root_event)
     assert url_event_1.web_spider_distance == 0
@@ -816,9 +818,24 @@ async def test_event_web_spider_distance(bbot_scanner):
     url_event_3 = scan.make_event(
         "http://www.evilcorp.com/test3", "URL_UNVERIFIED", parent=url_event_2, tags=["spider-danger"]
     )
+    assert url_event_3.web_spider_distance == 0
+    assert "spider-danger" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
+
+    # URL events should increment web spider distance
+    scan = bbot_scanner(config={"web": {"spider_distance": 1}})
+    url_event_1 = scan.make_event("http://www.evilcorp.com/test1", "URL", parent=scan.root_event, tags="status-200")
+    assert url_event_1.web_spider_distance == 0
+    url_event_2 = scan.make_event("http://www.evilcorp.com/test2", "URL", parent=url_event_1, tags="status-200")
+    assert url_event_2.web_spider_distance == 0
+    url_event_3 = scan.make_event(
+        "http://www.evilcorp.com/test3", "URL_UNVERIFIED", parent=url_event_2, tags=["spider-danger"]
+    )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
     assert "spider-max" not in url_event_3.tags
+
+    # SOCIAL events should inherit spider distance
     social_event = scan.make_event(
         {"platform": "github", "url": "http://www.evilcorp.com/test4"}, "SOCIAL", parent=url_event_3
     )
@@ -846,17 +863,17 @@ async def test_event_web_spider_distance(bbot_scanner):
     url_event_2 = scan.make_event(
         "http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event, tags="spider-danger"
     )
-    # spider distance shouldn't increment because it's not the same host
-    assert url_event_2.web_spider_distance == 0
-    assert "spider-danger" in url_event_2.tags
-    assert "spider-max" not in url_event_2.tags
+    url_event_2b = scan.make_event("http://www.evilcorp.com", "URL", parent=url_event_2, tags="status-200")
+    assert url_event_2b.web_spider_distance == 0
+    assert "spider-danger" in url_event_2b.tags
+    assert "spider-max" not in url_event_2b.tags
     url_event_3 = scan.make_event(
-        "http://www.evilcorp.com/3", "URL_UNVERIFIED", parent=url_event_2, tags="spider-danger"
+        "http://www.evilcorp.com/3", "URL_UNVERIFIED", parent=url_event_2b, tags="spider-danger"
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
     assert "spider-max" not in url_event_3.tags
-    url_event_4 = scan.make_event("http://evilcorp.com", "URL_UNVERIFIED", parent=url_event_3)
+    url_event_4 = scan.make_event("http://evilcorp.com", "URL", parent=url_event_3, tags="status-200")
     assert url_event_4.web_spider_distance == 0
     assert "spider-danger" not in url_event_4.tags
     assert "spider-max" not in url_event_4.tags

{bbot-2.3.0.5532rc0.dist-info → bbot-2.3.0.5546rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5532rc0
+Version: 2.3.0.5546rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.3.0.5532rc0.dist-info → bbot-2.3.0.5546rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=MaCgza8_F_R_eB6xdloaxyR1Icashwh_yUP9PF4gxIg,130
+bbot/__init__.py,sha256=b08asVCW_E943cmWsJyYVhuzPNX-HUP24Y4fFDcuRLw,130
 bbot/cli.py,sha256=SUEd4CcI-9QzFnqXpezza1sq_TNPcfDtJaSwL4MAl9g,10717
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -7,7 +7,7 @@ bbot/core/config/logger.py,sha256=FzQ7Myl0MVqBi7gpn9LOnbuL-UTXxSKpl11xuEGxS5I,10
 bbot/core/core.py,sha256=V0G3dKPN5xCbXOoFeBRkh-BZb6A3kSNA060De01LiTU,7065
 bbot/core/engine.py,sha256=uauGZgd7zAnJwNmVVPDicwHbUVCa8pV_pif49lgYeWk,29364
 bbot/core/event/__init__.py,sha256=8ut88ZUg0kbtWkOx2j3XzNr_3kTfgoM-3UdiWHFA_ag,56
-bbot/core/event/base.py,sha256=4QDGH4ZY1DsgJAxORnAomOa4sAMs-JPWVIjh0fgb9ZQ,63129
+bbot/core/event/base.py,sha256=sQZNx3Rxb_Y4B4DYbtgEX5U440C9ZL8k0Z1gTV4xbKU,63910
 bbot/core/event/helpers.py,sha256=PUN4Trq5_wpKVuhmwUQWAr40apgMXhJ9Gz-VfZ0j3lA,1554
 bbot/core/flags.py,sha256=Ltvm8Bc4D65I55HuU5bzyjO1R3yMDNpVmreGU83ZBXE,1266
 bbot/core/helpers/__init__.py,sha256=0UNwcZjNsX41hbHdo3yZPuARkYWch-okI68DScexve4,86
@@ -78,7 +78,6 @@ bbot/modules/censys.py,sha256=J9ntCZGpS-HPf4EfoS6NpkOv7RqGk7AZmYIpLI8uYIE,3309
 bbot/modules/certspotter.py,sha256=AtL5BiOuDp4vu1-5fct4aQAGZM2qiODYsbgBsw0phoU,937
 bbot/modules/chaos.py,sha256=JyuwytwE3IRmNbw-uyJ0gCaTnywhhsHzTiZ3OJ15PAw,1573
 bbot/modules/code_repository.py,sha256=x70Z45VnNNMF8BPkHfGWZXsZXw_fStGB3y0-8jbP1Ns,2078
-bbot/modules/columbus.py,sha256=BhorMAB9IG22hXXhKsrNnNlPEIeCDyZvuwoBi-1iIiE,815
 bbot/modules/credshed.py,sha256=HAF5wgRGKIIpdMAe4mIAtkZRLmFYjMFyXtjjst6RJ20,4203
 bbot/modules/crt.py,sha256=6Zm90VKXwYYN6Sab0gwwhTARrtnQIqALJTVtFWMMTGk,1369
 bbot/modules/deadly/dastardly.py,sha256=O3QKU9XxreKaYCeJ0KthafBhC8uWR6_dxFh8VSuRLCk,5315
@@ -249,7 +248,7 @@ bbot/test/test_step_1/test_depsinstaller.py,sha256=zr9f-wJDotD1ZvKXGEuDRWzFYMAYB
 bbot/test/test_step_1/test_dns.py,sha256=SjefP-8GGyx9q-PWotCWu4esF0dRhWJRrS-J5MGNi6w,32153
 bbot/test/test_step_1/test_docs.py,sha256=YWVGNRfzcrvDmFekX0Cq9gutQplsqvhKTpZ0XK4tWvo,82
 bbot/test/test_step_1/test_engine.py,sha256=3HkCPtYhUxiZzfA-BRHpLsyaRj9wIXKbb49BCk9dILM,4267
-bbot/test/test_step_1/test_events.py,sha256=fFlppfiCesP1DX1tgG0U8NlOpTQe0hWeaE43A_MNXyo,52858
+bbot/test/test_step_1/test_events.py,sha256=T500iG9s5lZxn16r06Dqhp6xHF8xDTFDGC8wKQMHdt8,53758
 bbot/test/test_step_1/test_files.py,sha256=5Q_3jPpMXULxDHsanSDUaj8zF8bXzKdiJZHOmoYpLhQ,699
 bbot/test/test_step_1/test_helpers.py,sha256=hZfnzjtegezYOqTuo5uAaXGI2GGsfuHhglbfaPPYV-U,39482
 bbot/test/test_step_1/test_manager_deduplication.py,sha256=hZQpDXzg6zvzxFolVOcJuY-ME8NXjZUsqS70BRNXp8A,15594
@@ -295,7 +294,6 @@ bbot/test/test_step_2/module_tests/test_module_certspotter.py,sha256=60jCOeK1yaU
 bbot/test/test_step_2/module_tests/test_module_chaos.py,sha256=9JRgtDEnnJgmEMCTB2bqRJRkBavLys-6ypHPxrM_hXk,956
 bbot/test/test_step_2/module_tests/test_module_cloudcheck.py,sha256=9DVhJfXaM42JXz577T0LdrrmArvhFXNdhICZP8aLLFU,4081
 bbot/test/test_step_2/module_tests/test_module_code_repository.py,sha256=i02Tgvr_F9_E4d6aEaXrfdk71NkoDvjzP4C98l2rNGg,2414
-bbot/test/test_step_2/module_tests/test_module_columbus.py,sha256=6CpIJCsykUfsn0CqxQWciIiNdM3Z73-HTnpBnVtS-L8,563
 bbot/test/test_step_2/module_tests/test_module_credshed.py,sha256=ipkCFL7YmZBLWWoGyGr98saL_yh3E99EnLtagHqdY1g,3360
 bbot/test/test_step_2/module_tests/test_module_crt.py,sha256=V15tE1jcXdXJEzEEdAJvSMRWhKBFtxBBUJ_eewvV3U4,717
 bbot/test/test_step_2/module_tests/test_module_csv.py,sha256=UJqMqdiPjx-UjJw10OoVMAj378wu5mWIq0v04TCljTM,579
@@ -414,8 +412,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.3.0.5532rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.3.0.5532rc0.dist-info/METADATA,sha256=kmnYb6OiZWZbIas--ps5g4u1G6TmekxioY86bxnXkQo,18213
-bbot-2.3.0.5532rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-bbot-2.3.0.5532rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.3.0.5532rc0.dist-info/RECORD,,
+bbot-2.3.0.5546rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.3.0.5546rc0.dist-info/METADATA,sha256=cnsUy8H__x4px_DYgtiImR_fhtjMwwx4kpOaFlCXoD8,18213
+bbot-2.3.0.5546rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+bbot-2.3.0.5546rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.3.0.5546rc0.dist-info/RECORD,,

bbot/modules/columbus.py

@@ -1,25 +0,0 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
-
-
-class columbus(subdomain_enum):
-    flags = ["subdomain-enum", "passive", "safe"]
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    meta = {
-        "description": "Query the Columbus Project API for subdomains",
-        "created_date": "2023-06-01",
-        "author": "@TheTechromancer",
-    }
-
-    base_url = "https://columbus.elmasy.com/api/lookup"
-
-    async def request_url(self, query):
-        url = f"{self.base_url}/{self.helpers.quote(query)}?days=365"
-        return await self.api_request(url)
-
-    async def parse_results(self, r, query):
-        results = set()
-        json = r.json()
-        if json and isinstance(json, list):
-            return {f"{s.lower()}.{query}" for s in json}
-        return results

bbot/test/test_step_2/module_tests/test_module_columbus.py

@@ -1,13 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestColumbus(ModuleTestBase):
-    async def setup_after_prep(self, module_test):
-        module_test.httpx_mock.add_response(
-            url="https://columbus.elmasy.com/api/lookup/blacklanternsecurity.com?days=365",
-            json=["asdf", "zzzz"],
-        )
-
-    def check(self, module_test, events):
-        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
-        assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"