bbot 2.2.0.5279rc0__py3-none-any.whl → 2.2.0.5311rc0__py3-none-any.whl

bbot/test/test_step_1/test_target.py

@@ -3,39 +3,31 @@ from ..bbot_fixtures import *  # noqa: F401
 
 @pytest.mark.asyncio
 async def test_target(bbot_scanner):
-    import random
+    from radixtarget import RadixTarget
     from ipaddress import ip_address, ip_network
-    from bbot.scanner.target import Target, BBOTTarget
+    from bbot.scanner.target import BBOTTarget, ScanSeeds
 
     scan1 = bbot_scanner("api.publicapis.org", "8.8.8.8/30", "2001:4860:4860::8888/126")
     scan2 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125")
     scan3 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125")
     scan4 = bbot_scanner("8.8.8.8/29")
     scan5 = bbot_scanner()
-    assert not scan5.target
-    assert len(scan1.target) == 9
-    assert len(scan4.target) == 8
-    assert "8.8.8.9" in scan1.target
-    assert "8.8.8.12" not in scan1.target
-    assert "8.8.8.8/31" in scan1.target
-    assert "8.8.8.8/30" in scan1.target
-    assert "8.8.8.8/29" not in scan1.target
-    assert "2001:4860:4860::8889" in scan1.target
-    assert "2001:4860:4860::888c" not in scan1.target
-    assert "www.api.publicapis.org" in scan1.target
-    assert "api.publicapis.org" in scan1.target
-    assert "publicapis.org" not in scan1.target
-    assert "bob@www.api.publicapis.org" in scan1.target
-    assert "https://www.api.publicapis.org" in scan1.target
-    assert "www.api.publicapis.org:80" in scan1.target
-    assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target
-    assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target
-    assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target
-    assert scan1.target in scan2.target
-    assert scan2.target not in scan1.target
-    assert scan3.target in scan2.target
-    assert scan2.target == scan3.target
-    assert scan4.target != scan1.target
+
+    # test different types of inputs
+    target = BBOTTarget("evilcorp.com", "1.2.3.4/8")
+    assert "www.evilcorp.com" in target.seeds
+    assert "www.evilcorp.com:80" in target.seeds
+    assert "http://www.evilcorp.com:80" in target.seeds
+    assert "1.2.3.4" in target.seeds
+    assert "1.2.3.4/24" in target.seeds
+    assert ip_address("1.2.3.4") in target.seeds
+    assert ip_network("1.2.3.4/24", strict=False) in target.seeds
+    event = scan1.make_event("https://www.evilcorp.com:80", dummy=True)
+    assert event in target.seeds
+    with pytest.raises(ValueError):
+        ["asdf"] in target.seeds
+    with pytest.raises(ValueError):
+        target.seeds.get(["asdf"])
 
     assert not scan5.target.seeds
     assert len(scan1.target.seeds) == 9
@@ -56,6 +48,36 @@ async def test_target(bbot_scanner):
     assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.seeds
     assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.seeds
     assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.seeds
+    assert scan1.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds not in scan1.target.seeds
+    assert scan3.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds == scan3.target.seeds
+    assert scan4.target.seeds != scan1.target.seeds
+
+    assert not scan5.target.whitelist
+    assert len(scan1.target.whitelist) == 9
+    assert len(scan4.target.whitelist) == 8
+    assert "8.8.8.9" in scan1.target.whitelist
+    assert "8.8.8.12" not in scan1.target.whitelist
+    assert "8.8.8.8/31" in scan1.target.whitelist
+    assert "8.8.8.8/30" in scan1.target.whitelist
+    assert "8.8.8.8/29" not in scan1.target.whitelist
+    assert "2001:4860:4860::8889" in scan1.target.whitelist
+    assert "2001:4860:4860::888c" not in scan1.target.whitelist
+    assert "www.api.publicapis.org" in scan1.target.whitelist
+    assert "api.publicapis.org" in scan1.target.whitelist
+    assert "publicapis.org" not in scan1.target.whitelist
+    assert "bob@www.api.publicapis.org" in scan1.target.whitelist
+    assert "https://www.api.publicapis.org" in scan1.target.whitelist
+    assert "www.api.publicapis.org:80" in scan1.target.whitelist
+    assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.whitelist
+    assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.whitelist
+    assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.whitelist
+    assert scan1.target.whitelist in scan2.target.whitelist
+    assert scan2.target.whitelist not in scan1.target.whitelist
+    assert scan3.target.whitelist in scan2.target.whitelist
+    assert scan2.target.whitelist == scan3.target.whitelist
+    assert scan4.target.whitelist != scan1.target.whitelist
 
     assert scan1.whitelisted("https://[2001:4860:4860::8888]:80")
     assert scan1.whitelisted("[2001:4860:4860::8888]:80")
@@ -70,28 +92,34 @@ async def test_target(bbot_scanner):
     assert scan2.target.seeds == scan3.target.seeds
     assert scan4.target.seeds != scan1.target.seeds
 
-    assert str(scan1.target.get("8.8.8.9").host) == "8.8.8.8/30"
-    assert scan1.target.get("8.8.8.12") is None
-    assert str(scan1.target.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
-    assert scan1.target.get("2001:4860:4860::888c") is None
-    assert str(scan1.target.get("www.api.publicapis.org").host) == "api.publicapis.org"
-    assert scan1.target.get("publicapis.org") is None
-
-    target = Target("evilcorp.com")
+    assert str(scan1.target.seeds.get("8.8.8.9").host) == "8.8.8.8/30"
+    assert str(scan1.target.whitelist.get("8.8.8.9").host) == "8.8.8.8/30"
+    assert scan1.target.seeds.get("8.8.8.12") is None
+    assert scan1.target.whitelist.get("8.8.8.12") is None
+    assert str(scan1.target.seeds.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
+    assert str(scan1.target.whitelist.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
+    assert scan1.target.seeds.get("2001:4860:4860::888c") is None
+    assert scan1.target.whitelist.get("2001:4860:4860::888c") is None
+    assert str(scan1.target.seeds.get("www.api.publicapis.org").host) == "api.publicapis.org"
+    assert str(scan1.target.whitelist.get("www.api.publicapis.org").host) == "api.publicapis.org"
+    assert scan1.target.seeds.get("publicapis.org") is None
+    assert scan1.target.whitelist.get("publicapis.org") is None
+
+    target = RadixTarget("evilcorp.com")
     assert not "com" in target
     assert "evilcorp.com" in target
     assert "www.evilcorp.com" in target
-    strict_target = Target("evilcorp.com", strict_scope=True)
+    strict_target = RadixTarget("evilcorp.com", strict_dns_scope=True)
     assert not "com" in strict_target
     assert "evilcorp.com" in strict_target
     assert not "www.evilcorp.com" in strict_target
 
-    target = Target()
+    target = RadixTarget()
     target.add("evilcorp.com")
     assert not "com" in target
     assert "evilcorp.com" in target
     assert "www.evilcorp.com" in target
-    strict_target = Target(strict_scope=True)
+    strict_target = RadixTarget(strict_dns_scope=True)
     strict_target.add("evilcorp.com")
     assert not "com" in strict_target
     assert "evilcorp.com" in strict_target
@@ -99,16 +127,23 @@ async def test_target(bbot_scanner):
 
     # test target hashing
 
-    target1 = Target()
-    target1.add("evilcorp.com")
-    target1.add("1.2.3.4/24")
-    target1.add("https://evilcorp.net:8080")
-
-    target2 = Target()
-    target2.add("bob@evilcorp.org")
-    target2.add("evilcorp.com")
-    target2.add("1.2.3.4/24")
-    target2.add("https://evilcorp.net:8080")
+    target1 = BBOTTarget()
+    target1.whitelist.add("evilcorp.com")
+    target1.whitelist.add("1.2.3.4/24")
+    target1.whitelist.add("https://evilcorp.net:8080")
+    target1.seeds.add("evilcorp.com")
+    target1.seeds.add("1.2.3.4/24")
+    target1.seeds.add("https://evilcorp.net:8080")
+
+    target2 = BBOTTarget()
+    target2.whitelist.add("bob@evilcorp.org")
+    target2.whitelist.add("evilcorp.com")
+    target2.whitelist.add("1.2.3.4/24")
+    target2.whitelist.add("https://evilcorp.net:8080")
+    target2.seeds.add("bob@evilcorp.org")
+    target2.seeds.add("evilcorp.com")
+    target2.seeds.add("1.2.3.4/24")
+    target2.seeds.add("https://evilcorp.net:8080")
 
     # make sure it's a sha1 hash
     assert isinstance(target1.hash, bytes)
@@ -116,11 +151,22 @@ async def test_target(bbot_scanner):
 
     # hashes shouldn't match yet
     assert target1.hash != target2.hash
+    assert target1.scope_hash != target2.scope_hash
     # add missing email
-    target1.add("bob@evilcorp.org")
+    target1.whitelist.add("bob@evilcorp.org")
+    assert target1.hash != target2.hash
+    assert target1.scope_hash == target2.scope_hash
+    target1.seeds.add("bob@evilcorp.org")
     # now they should match
     assert target1.hash == target2.hash
 
+    # test default whitelist
+    bbottarget = BBOTTarget("http://1.2.3.4:8443", "bob@evilcorp.com")
+    assert bbottarget.seeds.hosts == {ip_network("1.2.3.4"), "evilcorp.com"}
+    assert bbottarget.whitelist.hosts == {ip_network("1.2.3.4"), "evilcorp.com"}
+    assert set([e.data for e in bbottarget.seeds.events]) == {"http://1.2.3.4:8443/", "bob@evilcorp.com"}
+    assert set([e.data for e in bbottarget.whitelist.events]) == {"1.2.3.4", "evilcorp.com"}
+
     bbottarget1 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"])
     bbottarget2 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"])
     bbottarget3 = BBOTTarget("evilcorp.com", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"])
@@ -137,14 +183,23 @@ async def test_target(bbot_scanner):
 
     assert bbottarget1 == bbottarget2
     assert bbottarget2 == bbottarget1
+    # 1 and 3 have different seeds
     assert bbottarget1 != bbottarget3
     assert bbottarget3 != bbottarget1
-    bbottarget3.add("evilcorp.net")
+    # until we make them the same
+    bbottarget3.seeds.add("evilcorp.net")
     assert bbottarget1 == bbottarget3
     assert bbottarget3 == bbottarget1
 
-    bbottarget1.add("http://evilcorp.co.nz")
-    bbottarget2.add("evilcorp.co.nz")
+    # adding different events (but with same host) to whitelist should not change hash (since only hosts matter)
+    bbottarget1.whitelist.add("http://evilcorp.co.nz")
+    bbottarget2.whitelist.add("evilcorp.co.nz")
+    assert bbottarget1 == bbottarget2
+    assert bbottarget2 == bbottarget1
+
+    # but seeds should change hash
+    bbottarget1.seeds.add("http://evilcorp.co.nz")
+    bbottarget2.seeds.add("evilcorp.co.nz")
     assert bbottarget1 != bbottarget2
     assert bbottarget2 != bbottarget1
 
@@ -156,15 +211,11 @@ async def test_target(bbot_scanner):
     assert bbottarget8 != bbottarget9
     assert bbottarget9 != bbottarget8
 
-    bbottarget10 = bbottarget9.copy()
-    assert bbottarget10 == bbottarget9
-    assert bbottarget9 == bbottarget10
-
     # make sure duplicate events don't change hash
-    target1 = Target("https://evilcorp.com")
-    target2 = Target("https://evilcorp.com")
+    target1 = BBOTTarget("https://evilcorp.com")
+    target2 = BBOTTarget("https://evilcorp.com")
     assert target1 == target2
-    target1.add("https://evilcorp.com:443")
+    target1.seeds.add("https://evilcorp.com:443")
     assert target1 == target2
 
     # make sure hosts are collapsed in whitelist and blacklist
@@ -173,10 +224,12 @@ async def test_target(bbot_scanner):
         whitelist=["evilcorp.net:443", "http://evilcorp.net:8080"],
         blacklist=["http://evilcorp.org:8080", "evilcorp.org:443"],
     )
-    assert list(bbottarget) == ["http://evilcorp.com:8080"]
+    # base class is not iterable
+    with pytest.raises(TypeError):
+        assert list(bbottarget) == ["http://evilcorp.com:8080"]
     assert list(bbottarget.seeds) == ["http://evilcorp.com:8080"]
-    assert list(bbottarget.whitelist) == ["evilcorp.net"]
-    assert list(bbottarget.blacklist) == ["evilcorp.org"]
+    assert set([e.data for e in bbottarget.whitelist]) == {"evilcorp.net:443", "http://evilcorp.net:8080/"}
+    assert set([e.data for e in bbottarget.blacklist]) == {"http://evilcorp.org:8080/", "evilcorp.org:443"}
 
     # test org stub as target
     for org_target in ("ORG:evilcorp", "ORG_STUB:evilcorp"):
@@ -205,16 +258,25 @@ async def test_target(bbot_scanner):
         "http://www.evilcorp.net/",
         "bob@fdsa.evilcorp.net",
     }
-    assert set([e.data for e in bbottarget.whitelist.events]) == {"evilcorp.com", "evilcorp.net"}
-    assert set([e.data for e in bbottarget.blacklist.events]) == {"1.2.3.4", "4.3.2.0/24", "asdf.evilcorp.net"}
+    assert set([e.data for e in bbottarget.whitelist.events]) == {
+        "evilcorp.com",
+        "evilcorp.net",
+        "bob@www.evilcorp.com",
+    }
+    assert set([e.data for e in bbottarget.blacklist.events]) == {
+        "1.2.3.4",
+        "4.3.2.0/24",
+        "http://1.2.3.4/",
+        "bob@asdf.evilcorp.net",
+    }
     assert set(bbottarget.seeds.hosts) == {ip_network("1.2.3.0/24"), "www.evilcorp.net", "fdsa.evilcorp.net"}
     assert set(bbottarget.whitelist.hosts) == {"evilcorp.com", "evilcorp.net"}
-    assert set(bbottarget.blacklist.hosts) == {ip_address("1.2.3.4"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"}
-    assert bbottarget.hash == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba"
-    assert bbottarget.scope_hash == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw"
-    assert bbottarget.seeds.hash == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5"
-    assert bbottarget.whitelist.hash == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC"
-    assert bbottarget.blacklist.hash == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/"
+    assert set(bbottarget.blacklist.hosts) == {ip_network("1.2.3.4/32"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"}
+    assert bbottarget.hash == b"\xb3iU\xa8#\x8aq\x84/\xc5\xf2;\x11\x11\x0c&\xea\x07\xd4Q"
+    assert bbottarget.scope_hash == b"f\xe1\x01c^3\xf5\xd24B\x87P\xa0Glq0p3J"
+    assert bbottarget.seeds.hash == b"V\n\xf5\x1d\x1f=i\xbc\\\x15o\xc2p\xb2\x84\x97\xfeR\xde\xc1"
+    assert bbottarget.whitelist.hash == b"\x8e\xd0\xa76\x8em4c\x0e\x1c\xfdA\x9d*sv}\xeb\xc4\xc4"
+    assert bbottarget.blacklist.hash == b'\xf7\xaf\xa1\xda4"C:\x13\xf42\xc3,\xc3\xa9\x9f\x15\x15n\\'
 
     scan = bbot_scanner(
         "http://www.evilcorp.net",
@@ -227,72 +289,35 @@ async def test_target(bbot_scanner):
     scan_events = [e for e in events if e.type == "SCAN"]
     assert len(scan_events) == 2
     target_dict = scan_events[0].data["target"]
+
+    assert target_dict["seeds"] == ["1.2.3.0/24", "bob@fdsa.evilcorp.net", "http://www.evilcorp.net/"]
+    assert target_dict["whitelist"] == ["bob@www.evilcorp.com", "evilcorp.com", "evilcorp.net"]
+    assert target_dict["blacklist"] == ["1.2.3.4", "4.3.2.0/24", "bob@asdf.evilcorp.net", "http://1.2.3.4/"]
     assert target_dict["strict_scope"] == False
-    assert target_dict["hash"] == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba".hex()
-    assert target_dict["scope_hash"] == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw".hex()
-    assert target_dict["seed_hash"] == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5".hex()
-    assert target_dict["whitelist_hash"] == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC".hex()
-    assert target_dict["blacklist_hash"] == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/".hex()
-    assert target_dict["hash"] == "0b9038e3ef0a3d1364df003bac6b0cbcd2cc27ba"
-    assert target_dict["scope_hash"] == "00f556fb2eeb23cbf071f9e965b71fe2542bdb77"
-    assert target_dict["seed_hash"] == "af2e8683a143adb4e7605894e2a001c2e33a4ac5"
-    assert target_dict["whitelist_hash"] == "a04166076e10d9b60aa7544fb037cd57c4764c43"
-    assert target_dict["blacklist_hash"] == "af0e8ae94a5a86beeea9a9db30af27238420552f"
-
-    # test target sorting
-    big_subnet = scan.make_event("1.2.3.4/24", dummy=True)
-    medium_subnet = scan.make_event("1.2.3.4/28", dummy=True)
-    small_subnet = scan.make_event("1.2.3.4/30", dummy=True)
-    ip_event = scan.make_event("1.2.3.4", dummy=True)
-    parent_domain = scan.make_event("evilcorp.com", dummy=True)
-    grandparent_domain = scan.make_event("www.evilcorp.com", dummy=True)
-    greatgrandparent_domain = scan.make_event("api.www.evilcorp.com", dummy=True)
-    target = Target()
-    assert big_subnet._host_size == -256
-    assert medium_subnet._host_size == -16
-    assert small_subnet._host_size == -4
-    assert ip_event._host_size == 1
-    assert parent_domain._host_size == 12
-    assert grandparent_domain._host_size == 16
-    assert greatgrandparent_domain._host_size == 20
-    events = [
-        big_subnet,
-        medium_subnet,
-        small_subnet,
-        ip_event,
-        parent_domain,
-        grandparent_domain,
-        greatgrandparent_domain,
-    ]
-    random.shuffle(events)
-    assert target._sort_events(events) == [
-        big_subnet,
-        medium_subnet,
-        small_subnet,
-        ip_event,
-        parent_domain,
-        grandparent_domain,
-        greatgrandparent_domain,
-    ]
+    assert target_dict["hash"] == "b36955a8238a71842fc5f23b11110c26ea07d451"
+    assert target_dict["seed_hash"] == "560af51d1f3d69bc5c156fc270b28497fe52dec1"
+    assert target_dict["whitelist_hash"] == "8ed0a7368e6d34630e1cfd419d2a73767debc4c4"
+    assert target_dict["blacklist_hash"] == "f7afa1da3422433a13f432c32cc3a99f15156e5c"
+    assert target_dict["scope_hash"] == "66e101635e33f5d234428750a0476c713070334a"
 
     # make sure child subnets/IPs don't get added to whitelist/blacklist
-    target = Target("1.2.3.4/24", "1.2.3.4/28", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/24"}
-    target = Target("1.2.3.4/28", "1.2.3.4/24", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/24"}
-    target = Target("1.2.3.4/28", "1.2.3.4", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/28"}
-    target = Target("1.2.3.4", "1.2.3.4/28", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/28"}
+    target = RadixTarget("1.2.3.4/24", "1.2.3.4/28", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/24")}
+    target = RadixTarget("1.2.3.4/28", "1.2.3.4/24", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/24")}
+    target = RadixTarget("1.2.3.4/28", "1.2.3.4", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/28")}
+    target = RadixTarget("1.2.3.4", "1.2.3.4/28", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/28")}
 
     # same but for domains
-    target = Target("evilcorp.com", "www.evilcorp.com", acl_mode=True)
-    assert set(e.data for e in target) == {"evilcorp.com"}
-    target = Target("www.evilcorp.com", "evilcorp.com", acl_mode=True)
-    assert set(e.data for e in target) == {"evilcorp.com"}
+    target = RadixTarget("evilcorp.com", "www.evilcorp.com", acl_mode=True)
+    assert set(target) == {"evilcorp.com"}
+    target = RadixTarget("www.evilcorp.com", "evilcorp.com", acl_mode=True)
+    assert set(target) == {"evilcorp.com"}
 
     # make sure strict_scope doesn't mess us up
-    target = Target("evilcorp.co.uk", "www.evilcorp.co.uk", acl_mode=True, strict_scope=True)
+    target = RadixTarget("evilcorp.co.uk", "www.evilcorp.co.uk", acl_mode=True, strict_dns_scope=True)
     assert set(target.hosts) == {"evilcorp.co.uk", "www.evilcorp.co.uk"}
     assert "evilcorp.co.uk" in target
     assert "www.evilcorp.co.uk" in target
@@ -300,10 +325,83 @@ async def test_target(bbot_scanner):
     assert not "api.www.evilcorp.co.uk" in target
 
     # test 'single' boolean argument
-    target = Target("http://evilcorp.com", "evilcorp.com:443")
+    target = ScanSeeds("http://evilcorp.com", "evilcorp.com:443")
     assert "www.evilcorp.com" in target
+    assert "bob@evilcorp.com" in target
     event = target.get("www.evilcorp.com")
     assert event.host == "evilcorp.com"
     events = target.get("www.evilcorp.com", single=False)
     assert len(events) == 2
     assert set([e.data for e in events]) == {"http://evilcorp.com/", "evilcorp.com:443"}
+
+
+@pytest.mark.asyncio
+async def test_blacklist_regex(bbot_scanner, bbot_httpserver):
+
+    from bbot.scanner.target import ScanBlacklist
+
+    blacklist = ScanBlacklist("evilcorp.com")
+    assert blacklist.inputs == {"evilcorp.com"}
+    assert "www.evilcorp.com" in blacklist
+    assert "http://www.evilcorp.com" in blacklist
+    blacklist.add("RE:test")
+    assert "RE:test" in blacklist.inputs
+    assert set(blacklist.inputs) == {"evilcorp.com", "RE:test"}
+    assert blacklist.blacklist_regexes
+    assert next(iter(blacklist.blacklist_regexes)).pattern == "test"
+    result1 = blacklist.get("test.com")
+    assert result1.type == "DNS_NAME"
+    assert result1.data == "test.com"
+    result2 = blacklist.get("www.evilcorp.com")
+    assert result2.type == "DNS_NAME"
+    assert result2.data == "evilcorp.com"
+    result2 = blacklist.get("www.evil.com")
+    assert result2 is None
+    with pytest.raises(KeyError):
+        blacklist.get("www.evil.com", raise_error=True)
+    assert "test.com" in blacklist
+    assert "http://evilcorp.com/test.aspx" in blacklist
+    assert not "http://tes.com" in blacklist
+
+    blacklist = ScanBlacklist("evilcorp.com", r"RE:[0-9]{6}\.aspx$")
+    assert "http://evilcorp.com" in blacklist
+    assert not "http://test.com/123456" in blacklist
+    assert not "http://test.com/12345.aspx?a=asdf" in blacklist
+    assert not "http://test.com/asdf/123456.aspx/asdf" in blacklist
+    assert "http://test.com/asdf/123456.aspx?a=asdf" in blacklist
+    assert "http://test.com/asdf/123456.aspx" in blacklist
+
+    bbot_httpserver.expect_request(uri="/").respond_with_data(
+        """
+        <a href='http://127.0.0.1:8888/asdfevil333asdf'/>
+        <a href='http://127.0.0.1:8888/logout.aspx'/>
+    """
+    )
+    bbot_httpserver.expect_request(uri="/asdfevilasdf").respond_with_data("")
+    bbot_httpserver.expect_request(uri="/logout.aspx").respond_with_data("")
+
+    # make sure URL is detected normally
+    scan = bbot_scanner("http://127.0.0.1:8888/", presets=["spider"], config={"excavate": True}, debug=True)
+    assert set([r.pattern for r in scan.target.blacklist.blacklist_regexes]) == {r"/.*(sign|log)[_-]?out"}
+    events = [e async for e in scan.async_start()]
+    urls = [e.data for e in events if e.type == "URL"]
+    assert len(urls) == 2
+    assert set(urls) == {"http://127.0.0.1:8888/", "http://127.0.0.1:8888/asdfevil333asdf"}
+
+    # same scan again but with blacklist regex
+    scan = bbot_scanner(
+        "http://127.0.0.1:8888/",
+        blacklist=[r"RE:evil[0-9]{3}"],
+        presets=["spider"],
+        config={"excavate": True},
+        debug=True,
+    )
+    assert scan.target.blacklist.blacklist_regexes
+    assert set([r.pattern for r in scan.target.blacklist.blacklist_regexes]) == {
+        r"evil[0-9]{3}",
+        r"/.*(sign|log)[_-]?out",
+    }
+    events = [e async for e in scan.async_start()]
+    urls = [e.data for e in events if e.type == "URL"]
+    assert len(urls) == 1
+    assert set(urls) == {"http://127.0.0.1:8888/"}

bbot/test/test_step_2/module_tests/test_module_dastardly.py

@@ -44,7 +44,7 @@ class TestDastardly(ModuleTestBase):
 
         # get docker IP
         docker_ip = await self.get_docker_ip(module_test)
-        module_test.scan.target.add(docker_ip)
+        module_test.scan.target.seeds.add(docker_ip)
 
         # replace 127.0.0.1 with docker host IP to allow dastardly access to local http server
         old_filter_event = module_test.module.filter_event

bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py

@@ -142,7 +142,7 @@ class TestFFUFShortnames(ModuleTestBase):
                 tags=["shortname-file"],
             )
         )
-        module_test.scan.target.seeds._events = set(seed_events)
+        module_test.scan.target.seeds.events = set(seed_events)
 
         expect_args = {"method": "GET", "uri": "/administrator.aspx"}
         respond_args = {"response_data": "alive"}

{bbot-2.2.0.5279rc0.dist-info → bbot-2.2.0.5311rc0.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.2.0.5279rc0
+Version: 2.2.0.5311rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
-Keywords: python,cli,automation,osint,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
+Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
 Author: TheTechromancer
 Requires-Python: >=3.9,<4.0
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -20,7 +20,7 @@ Requires-Dist: ansible (>=7.3,<9.0)
 Requires-Dist: ansible-runner (>=2.3.2,<3.0.0)
 Requires-Dist: beautifulsoup4 (>=4.12.2,<5.0.0)
 Requires-Dist: cachetools (>=5.3.2,<6.0.0)
-Requires-Dist: cloudcheck (>=5.0.0.350,<6.0.0.0)
+Requires-Dist: cloudcheck (>=6.0.0.602,<7.0.0.0)
 Requires-Dist: deepdiff (>=6.2.3,<8.0.0)
 Requires-Dist: dnspython (>=2.4.2,<3.0.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0)
@@ -35,7 +35,7 @@ Requires-Dist: pycryptodome (>=3.17,<4.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pyjwt (>=2.7.0,<3.0.0)
 Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
-Requires-Dist: radixtarget (>=1.0.0.15,<2.0.0.0)
+Requires-Dist: radixtarget (>=2.0.0.50,<3.0.0.0)
 Requires-Dist: regex (>=2024.4.16,<2025.0.0)
 Requires-Dist: setproctitle (>=1.3.3,<2.0.0)
 Requires-Dist: socksio (>=1.0.0,<2.0.0)