bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/test/test_step_1/test_scan.py

@@ -1,3 +1,5 @@
+from ipaddress import ip_network
+
 from ..bbot_fixtures import *
 
 
@@ -12,6 +14,7 @@ async def test_scan(
         "1.1.1.0",
         "1.1.1.1/31",
         "evilcorp.com",
+        "test.evilcorp.com",
         blacklist=["1.1.1.1/28", "www.evilcorp.com"],
         modules=["ipneighbor"],
     )
@@ -31,8 +34,11 @@ async def test_scan(
     assert not scan0.in_scope("test.www.evilcorp.com")
     assert not scan0.in_scope("www.evilcorp.co.uk")
     j = scan0.json
-    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com"}
-    assert set(j["target"]["whitelist"]) == {"1.1.1.0/31", "evilcorp.com"}
+    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # we preserve the original whitelist inputs
+    assert set(j["target"]["whitelist"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # but in the background they are collapsed
+    assert scan0.target.whitelist.hosts == {ip_network("1.1.1.0/31"), "evilcorp.com"}
     assert set(j["target"]["blacklist"]) == {"1.1.1.0/28", "www.evilcorp.com"}
     assert "ipneighbor" in j["preset"]["modules"]
 
@@ -78,6 +84,9 @@ async def test_scan(
     for scan in (scan0, scan1, scan2, scan4, scan5):
         await scan._cleanup()
 
+    scan6 = bbot_scanner("a.foobar.io", "b.foobar.io", "c.foobar.io", "foobar.io")
+    assert len(scan6.dns_strings) == 1
+
 
 @pytest.mark.asyncio
 async def test_url_extension_handling(bbot_scanner):
@@ -91,13 +100,13 @@ async def test_url_extension_handling(bbot_scanner):
     assert "blacklisted" not in bad_event.tags
     assert "httpx-only" not in httpx_event.tags
     result = await scan.ingress_module.handle_event(good_event)
-    assert result == None
+    assert result is None
     result, reason = await scan.ingress_module.handle_event(bad_event)
-    assert result == False
+    assert result is False
     assert reason == "event is blacklisted"
     assert "blacklisted" in bad_event.tags
     result = await scan.ingress_module.handle_event(httpx_event)
-    assert result == None
+    assert result is None
     assert "httpx-only" in httpx_event.tags
 
     await scan._cleanup()
@@ -115,3 +124,28 @@ async def test_speed_counter():
         await asyncio.sleep(0.2)
     # only 5 should show
     assert 4 <= counter.speed <= 5
+
+
+@pytest.mark.asyncio
+async def test_python_output_matches_json(bbot_scanner):
+    import json
+
+    scan = bbot_scanner(
+        "blacklanternsecurity.com",
+        config={"speculate": True, "dns": {"minimal": False}, "scope": {"report_distance": 10}},
+    )
+    await scan.helpers.dns._mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.1"]}})
+    events = [e.json() async for e in scan.async_start()]
+    output_json = scan.home / "output.json"
+    json_events = []
+    for line in open(output_json):
+        json_events.append(json.loads(line))
+
+    assert len(events) == 5
+    scan_events = [e for e in events if e["type"] == "SCAN"]
+    assert len(scan_events) == 2
+    assert all(isinstance(e["data"]["status"], str) for e in scan_events)
+    assert len([e for e in events if e["type"] == "DNS_NAME"]) == 1
+    assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1
+    assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1
+    assert events == json_events

bbot/test/test_step_1/test_scope.py

@@ -12,7 +12,8 @@ class TestScopeBaseline(ModuleTestBase):
         module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
 
     def check(self, module_test, events):
-        assert len(events) == 6
+        assert len(events) == 7
+        assert 2 == len([e for e in events if e.type == "SCAN"])
         assert 1 == len(
             [
                 e
@@ -62,7 +63,7 @@ class TestScopeBlacklist(TestScopeBaseline):
         module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
 
     def check(self, module_test, events):
-        assert len(events) == 1
+        assert len(events) == 2
         assert not any(e.type == "URL" for e in events)
         assert not any(str(e.host) == "127.0.0.1" for e in events)
 
@@ -72,7 +73,7 @@ class TestScopeWhitelist(TestScopeBlacklist):
     whitelist = ["255.255.255.255"]
 
     def check(self, module_test, events):
-        assert len(events) == 3
+        assert len(events) == 4
         assert not any(e.type == "URL" for e in events)
         assert 1 == len(
             [

bbot/test/test_step_1/test_target.py

@@ -3,39 +3,31 @@ from ..bbot_fixtures import *  # noqa: F401
 
 @pytest.mark.asyncio
 async def test_target(bbot_scanner):
-    import random
+    from radixtarget import RadixTarget
     from ipaddress import ip_address, ip_network
-    from bbot.scanner.target import Target, BBOTTarget
+    from bbot.scanner.target import BBOTTarget, ScanSeeds
 
     scan1 = bbot_scanner("api.publicapis.org", "8.8.8.8/30", "2001:4860:4860::8888/126")
     scan2 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125")
     scan3 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125")
     scan4 = bbot_scanner("8.8.8.8/29")
     scan5 = bbot_scanner()
-    assert not scan5.target
-    assert len(scan1.target) == 9
-    assert len(scan4.target) == 8
-    assert "8.8.8.9" in scan1.target
-    assert "8.8.8.12" not in scan1.target
-    assert "8.8.8.8/31" in scan1.target
-    assert "8.8.8.8/30" in scan1.target
-    assert "8.8.8.8/29" not in scan1.target
-    assert "2001:4860:4860::8889" in scan1.target
-    assert "2001:4860:4860::888c" not in scan1.target
-    assert "www.api.publicapis.org" in scan1.target
-    assert "api.publicapis.org" in scan1.target
-    assert "publicapis.org" not in scan1.target
-    assert "bob@www.api.publicapis.org" in scan1.target
-    assert "https://www.api.publicapis.org" in scan1.target
-    assert "www.api.publicapis.org:80" in scan1.target
-    assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target
-    assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target
-    assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target
-    assert scan1.target in scan2.target
-    assert scan2.target not in scan1.target
-    assert scan3.target in scan2.target
-    assert scan2.target == scan3.target
-    assert scan4.target != scan1.target
+
+    # test different types of inputs
+    target = BBOTTarget("evilcorp.com", "1.2.3.4/8")
+    assert "www.evilcorp.com" in target.seeds
+    assert "www.evilcorp.com:80" in target.seeds
+    assert "http://www.evilcorp.com:80" in target.seeds
+    assert "1.2.3.4" in target.seeds
+    assert "1.2.3.4/24" in target.seeds
+    assert ip_address("1.2.3.4") in target.seeds
+    assert ip_network("1.2.3.4/24", strict=False) in target.seeds
+    event = scan1.make_event("https://www.evilcorp.com:80", dummy=True)
+    assert event in target.seeds
+    with pytest.raises(ValueError):
+        ["asdf"] in target.seeds
+    with pytest.raises(ValueError):
+        target.seeds.get(["asdf"])
 
     assert not scan5.target.seeds
     assert len(scan1.target.seeds) == 9
@@ -56,6 +48,36 @@ async def test_target(bbot_scanner):
     assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.seeds
     assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.seeds
     assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.seeds
+    assert scan1.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds not in scan1.target.seeds
+    assert scan3.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds == scan3.target.seeds
+    assert scan4.target.seeds != scan1.target.seeds
+
+    assert not scan5.target.whitelist
+    assert len(scan1.target.whitelist) == 9
+    assert len(scan4.target.whitelist) == 8
+    assert "8.8.8.9" in scan1.target.whitelist
+    assert "8.8.8.12" not in scan1.target.whitelist
+    assert "8.8.8.8/31" in scan1.target.whitelist
+    assert "8.8.8.8/30" in scan1.target.whitelist
+    assert "8.8.8.8/29" not in scan1.target.whitelist
+    assert "2001:4860:4860::8889" in scan1.target.whitelist
+    assert "2001:4860:4860::888c" not in scan1.target.whitelist
+    assert "www.api.publicapis.org" in scan1.target.whitelist
+    assert "api.publicapis.org" in scan1.target.whitelist
+    assert "publicapis.org" not in scan1.target.whitelist
+    assert "bob@www.api.publicapis.org" in scan1.target.whitelist
+    assert "https://www.api.publicapis.org" in scan1.target.whitelist
+    assert "www.api.publicapis.org:80" in scan1.target.whitelist
+    assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.whitelist
+    assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.whitelist
+    assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.whitelist
+    assert scan1.target.whitelist in scan2.target.whitelist
+    assert scan2.target.whitelist not in scan1.target.whitelist
+    assert scan3.target.whitelist in scan2.target.whitelist
+    assert scan2.target.whitelist == scan3.target.whitelist
+    assert scan4.target.whitelist != scan1.target.whitelist
 
     assert scan1.whitelisted("https://[2001:4860:4860::8888]:80")
     assert scan1.whitelisted("[2001:4860:4860::8888]:80")
@@ -70,45 +92,58 @@ async def test_target(bbot_scanner):
     assert scan2.target.seeds == scan3.target.seeds
     assert scan4.target.seeds != scan1.target.seeds
 
-    assert str(scan1.target.get("8.8.8.9").host) == "8.8.8.8/30"
-    assert scan1.target.get("8.8.8.12") is None
-    assert str(scan1.target.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
-    assert scan1.target.get("2001:4860:4860::888c") is None
-    assert str(scan1.target.get("www.api.publicapis.org").host) == "api.publicapis.org"
-    assert scan1.target.get("publicapis.org") is None
-
-    target = Target("evilcorp.com")
-    assert not "com" in target
+    assert str(scan1.target.seeds.get("8.8.8.9").host) == "8.8.8.8/30"
+    assert str(scan1.target.whitelist.get("8.8.8.9").host) == "8.8.8.8/30"
+    assert scan1.target.seeds.get("8.8.8.12") is None
+    assert scan1.target.whitelist.get("8.8.8.12") is None
+    assert str(scan1.target.seeds.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
+    assert str(scan1.target.whitelist.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"
+    assert scan1.target.seeds.get("2001:4860:4860::888c") is None
+    assert scan1.target.whitelist.get("2001:4860:4860::888c") is None
+    assert str(scan1.target.seeds.get("www.api.publicapis.org").host) == "api.publicapis.org"
+    assert str(scan1.target.whitelist.get("www.api.publicapis.org").host) == "api.publicapis.org"
+    assert scan1.target.seeds.get("publicapis.org") is None
+    assert scan1.target.whitelist.get("publicapis.org") is None
+
+    target = RadixTarget("evilcorp.com")
+    assert "com" not in target
     assert "evilcorp.com" in target
     assert "www.evilcorp.com" in target
-    strict_target = Target("evilcorp.com", strict_scope=True)
-    assert not "com" in strict_target
+    strict_target = RadixTarget("evilcorp.com", strict_dns_scope=True)
+    assert "com" not in strict_target
     assert "evilcorp.com" in strict_target
-    assert not "www.evilcorp.com" in strict_target
+    assert "www.evilcorp.com" not in strict_target
 
-    target = Target()
+    target = RadixTarget()
     target.add("evilcorp.com")
-    assert not "com" in target
+    assert "com" not in target
     assert "evilcorp.com" in target
     assert "www.evilcorp.com" in target
-    strict_target = Target(strict_scope=True)
+    strict_target = RadixTarget(strict_dns_scope=True)
     strict_target.add("evilcorp.com")
-    assert not "com" in strict_target
+    assert "com" not in strict_target
     assert "evilcorp.com" in strict_target
-    assert not "www.evilcorp.com" in strict_target
+    assert "www.evilcorp.com" not in strict_target
 
     # test target hashing
 
-    target1 = Target()
-    target1.add("evilcorp.com")
-    target1.add("1.2.3.4/24")
-    target1.add("https://evilcorp.net:8080")
-
-    target2 = Target()
-    target2.add("bob@evilcorp.org")
-    target2.add("evilcorp.com")
-    target2.add("1.2.3.4/24")
-    target2.add("https://evilcorp.net:8080")
+    target1 = BBOTTarget()
+    target1.whitelist.add("evilcorp.com")
+    target1.whitelist.add("1.2.3.4/24")
+    target1.whitelist.add("https://evilcorp.net:8080")
+    target1.seeds.add("evilcorp.com")
+    target1.seeds.add("1.2.3.4/24")
+    target1.seeds.add("https://evilcorp.net:8080")
+
+    target2 = BBOTTarget()
+    target2.whitelist.add("bob@evilcorp.org")
+    target2.whitelist.add("evilcorp.com")
+    target2.whitelist.add("1.2.3.4/24")
+    target2.whitelist.add("https://evilcorp.net:8080")
+    target2.seeds.add("bob@evilcorp.org")
+    target2.seeds.add("evilcorp.com")
+    target2.seeds.add("1.2.3.4/24")
+    target2.seeds.add("https://evilcorp.net:8080")
 
     # make sure it's a sha1 hash
     assert isinstance(target1.hash, bytes)
@@ -116,11 +151,22 @@ async def test_target(bbot_scanner):
 
     # hashes shouldn't match yet
     assert target1.hash != target2.hash
+    assert target1.scope_hash != target2.scope_hash
     # add missing email
-    target1.add("bob@evilcorp.org")
+    target1.whitelist.add("bob@evilcorp.org")
+    assert target1.hash != target2.hash
+    assert target1.scope_hash == target2.scope_hash
+    target1.seeds.add("bob@evilcorp.org")
     # now they should match
     assert target1.hash == target2.hash
 
+    # test default whitelist
+    bbottarget = BBOTTarget("http://1.2.3.4:8443", "bob@evilcorp.com")
+    assert bbottarget.seeds.hosts == {ip_network("1.2.3.4"), "evilcorp.com"}
+    assert bbottarget.whitelist.hosts == {ip_network("1.2.3.4"), "evilcorp.com"}
+    assert {e.data for e in bbottarget.seeds.events} == {"http://1.2.3.4:8443/", "bob@evilcorp.com"}
+    assert {e.data for e in bbottarget.whitelist.events} == {"1.2.3.4", "evilcorp.com"}
+
     bbottarget1 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"])
     bbottarget2 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"])
     bbottarget3 = BBOTTarget("evilcorp.com", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"])
@@ -137,14 +183,23 @@ async def test_target(bbot_scanner):
 
     assert bbottarget1 == bbottarget2
     assert bbottarget2 == bbottarget1
+    # 1 and 3 have different seeds
     assert bbottarget1 != bbottarget3
     assert bbottarget3 != bbottarget1
-    bbottarget3.add("evilcorp.net")
+    # until we make them the same
+    bbottarget3.seeds.add("evilcorp.net")
     assert bbottarget1 == bbottarget3
     assert bbottarget3 == bbottarget1
 
-    bbottarget1.add("http://evilcorp.co.nz")
-    bbottarget2.add("evilcorp.co.nz")
+    # adding different events (but with same host) to whitelist should not change hash (since only hosts matter)
+    bbottarget1.whitelist.add("http://evilcorp.co.nz")
+    bbottarget2.whitelist.add("evilcorp.co.nz")
+    assert bbottarget1 == bbottarget2
+    assert bbottarget2 == bbottarget1
+
+    # but seeds should change hash
+    bbottarget1.seeds.add("http://evilcorp.co.nz")
+    bbottarget2.seeds.add("evilcorp.co.nz")
     assert bbottarget1 != bbottarget2
     assert bbottarget2 != bbottarget1
 
@@ -156,15 +211,11 @@ async def test_target(bbot_scanner):
     assert bbottarget8 != bbottarget9
     assert bbottarget9 != bbottarget8
 
-    bbottarget10 = bbottarget9.copy()
-    assert bbottarget10 == bbottarget9
-    assert bbottarget9 == bbottarget10
-
     # make sure duplicate events don't change hash
-    target1 = Target("https://evilcorp.com")
-    target2 = Target("https://evilcorp.com")
+    target1 = BBOTTarget("https://evilcorp.com")
+    target2 = BBOTTarget("https://evilcorp.com")
     assert target1 == target2
-    target1.add("https://evilcorp.com:443")
+    target1.seeds.add("https://evilcorp.com:443")
     assert target1 == target2
 
     # make sure hosts are collapsed in whitelist and blacklist
@@ -173,24 +224,26 @@ async def test_target(bbot_scanner):
         whitelist=["evilcorp.net:443", "http://evilcorp.net:8080"],
         blacklist=["http://evilcorp.org:8080", "evilcorp.org:443"],
     )
-    assert list(bbottarget) == ["http://evilcorp.com:8080"]
+    # base class is not iterable
+    with pytest.raises(TypeError):
+        assert list(bbottarget) == ["http://evilcorp.com:8080"]
     assert list(bbottarget.seeds) == ["http://evilcorp.com:8080"]
-    assert list(bbottarget.whitelist) == ["evilcorp.net"]
-    assert list(bbottarget.blacklist) == ["evilcorp.org"]
+    assert {e.data for e in bbottarget.whitelist} == {"evilcorp.net:443", "http://evilcorp.net:8080/"}
+    assert {e.data for e in bbottarget.blacklist} == {"http://evilcorp.org:8080/", "evilcorp.org:443"}
 
     # test org stub as target
     for org_target in ("ORG:evilcorp", "ORG_STUB:evilcorp"):
         scan = bbot_scanner(org_target)
         events = [e async for e in scan.async_start()]
-        assert len(events) == 2
-        assert set([e.type for e in events]) == {"SCAN", "ORG_STUB"}
+        assert len(events) == 3
+        assert {e.type for e in events} == {"SCAN", "ORG_STUB"}
 
     # test username as target
     for user_target in ("USER:vancerefrigeration", "USERNAME:vancerefrigeration"):
         scan = bbot_scanner(user_target)
         events = [e async for e in scan.async_start()]
-        assert len(events) == 2
-        assert set([e.type for e in events]) == {"SCAN", "USERNAME"}
+        assert len(events) == 3
+        assert {e.type for e in events} == {"SCAN", "USERNAME"}
 
     # verify hash values
     bbottarget = BBOTTarget(
@@ -200,21 +253,30 @@ async def test_target(bbot_scanner):
         whitelist=["evilcorp.com", "bob@www.evilcorp.com", "evilcorp.net"],
         blacklist=["1.2.3.4", "4.3.2.1/24", "http://1.2.3.4", "bob@asdf.evilcorp.net"],
     )
-    assert set([e.data for e in bbottarget.seeds.events]) == {
+    assert {e.data for e in bbottarget.seeds.events} == {
         "1.2.3.0/24",
         "http://www.evilcorp.net/",
         "bob@fdsa.evilcorp.net",
     }
-    assert set([e.data for e in bbottarget.whitelist.events]) == {"evilcorp.com", "evilcorp.net"}
-    assert set([e.data for e in bbottarget.blacklist.events]) == {"1.2.3.4", "4.3.2.0/24", "asdf.evilcorp.net"}
+    assert {e.data for e in bbottarget.whitelist.events} == {
+        "evilcorp.com",
+        "evilcorp.net",
+        "bob@www.evilcorp.com",
+    }
+    assert {e.data for e in bbottarget.blacklist.events} == {
+        "1.2.3.4",
+        "4.3.2.0/24",
+        "http://1.2.3.4/",
+        "bob@asdf.evilcorp.net",
+    }
     assert set(bbottarget.seeds.hosts) == {ip_network("1.2.3.0/24"), "www.evilcorp.net", "fdsa.evilcorp.net"}
     assert set(bbottarget.whitelist.hosts) == {"evilcorp.com", "evilcorp.net"}
-    assert set(bbottarget.blacklist.hosts) == {ip_address("1.2.3.4"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"}
-    assert bbottarget.hash == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba"
-    assert bbottarget.scope_hash == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw"
-    assert bbottarget.seeds.hash == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5"
-    assert bbottarget.whitelist.hash == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC"
-    assert bbottarget.blacklist.hash == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/"
+    assert set(bbottarget.blacklist.hosts) == {ip_network("1.2.3.4/32"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"}
+    assert bbottarget.hash == b"\xb3iU\xa8#\x8aq\x84/\xc5\xf2;\x11\x11\x0c&\xea\x07\xd4Q"
+    assert bbottarget.scope_hash == b"f\xe1\x01c^3\xf5\xd24B\x87P\xa0Glq0p3J"
+    assert bbottarget.seeds.hash == b"V\n\xf5\x1d\x1f=i\xbc\\\x15o\xc2p\xb2\x84\x97\xfeR\xde\xc1"
+    assert bbottarget.whitelist.hash == b"\x8e\xd0\xa76\x8em4c\x0e\x1c\xfdA\x9d*sv}\xeb\xc4\xc4"
+    assert bbottarget.blacklist.hash == b'\xf7\xaf\xa1\xda4"C:\x13\xf42\xc3,\xc3\xa9\x9f\x15\x15n\\'
 
     scan = bbot_scanner(
         "http://www.evilcorp.net",
@@ -225,85 +287,120 @@ async def test_target(bbot_scanner):
     )
     events = [e async for e in scan.async_start()]
     scan_events = [e for e in events if e.type == "SCAN"]
-    assert len(scan_events) == 1
+    assert len(scan_events) == 2
     target_dict = scan_events[0].data["target"]
-    assert target_dict["strict_scope"] == False
-    assert target_dict["hash"] == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba".hex()
-    assert target_dict["scope_hash"] == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw".hex()
-    assert target_dict["seed_hash"] == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5".hex()
-    assert target_dict["whitelist_hash"] == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC".hex()
-    assert target_dict["blacklist_hash"] == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/".hex()
-    assert target_dict["hash"] == "0b9038e3ef0a3d1364df003bac6b0cbcd2cc27ba"
-    assert target_dict["scope_hash"] == "00f556fb2eeb23cbf071f9e965b71fe2542bdb77"
-    assert target_dict["seed_hash"] == "af2e8683a143adb4e7605894e2a001c2e33a4ac5"
-    assert target_dict["whitelist_hash"] == "a04166076e10d9b60aa7544fb037cd57c4764c43"
-    assert target_dict["blacklist_hash"] == "af0e8ae94a5a86beeea9a9db30af27238420552f"
-
-    # test target sorting
-    big_subnet = scan.make_event("1.2.3.4/24", dummy=True)
-    medium_subnet = scan.make_event("1.2.3.4/28", dummy=True)
-    small_subnet = scan.make_event("1.2.3.4/30", dummy=True)
-    ip_event = scan.make_event("1.2.3.4", dummy=True)
-    parent_domain = scan.make_event("evilcorp.com", dummy=True)
-    grandparent_domain = scan.make_event("www.evilcorp.com", dummy=True)
-    greatgrandparent_domain = scan.make_event("api.www.evilcorp.com", dummy=True)
-    target = Target()
-    assert big_subnet._host_size == -256
-    assert medium_subnet._host_size == -16
-    assert small_subnet._host_size == -4
-    assert ip_event._host_size == 1
-    assert parent_domain._host_size == 12
-    assert grandparent_domain._host_size == 16
-    assert greatgrandparent_domain._host_size == 20
-    events = [
-        big_subnet,
-        medium_subnet,
-        small_subnet,
-        ip_event,
-        parent_domain,
-        grandparent_domain,
-        greatgrandparent_domain,
-    ]
-    random.shuffle(events)
-    assert target._sort_events(events) == [
-        big_subnet,
-        medium_subnet,
-        small_subnet,
-        ip_event,
-        parent_domain,
-        grandparent_domain,
-        greatgrandparent_domain,
-    ]
+
+    assert target_dict["seeds"] == ["1.2.3.0/24", "bob@fdsa.evilcorp.net", "http://www.evilcorp.net/"]
+    assert target_dict["whitelist"] == ["bob@www.evilcorp.com", "evilcorp.com", "evilcorp.net"]
+    assert target_dict["blacklist"] == ["1.2.3.4", "4.3.2.0/24", "bob@asdf.evilcorp.net", "http://1.2.3.4/"]
+    assert target_dict["strict_scope"] is False
+    assert target_dict["hash"] == "b36955a8238a71842fc5f23b11110c26ea07d451"
+    assert target_dict["seed_hash"] == "560af51d1f3d69bc5c156fc270b28497fe52dec1"
+    assert target_dict["whitelist_hash"] == "8ed0a7368e6d34630e1cfd419d2a73767debc4c4"
+    assert target_dict["blacklist_hash"] == "f7afa1da3422433a13f432c32cc3a99f15156e5c"
+    assert target_dict["scope_hash"] == "66e101635e33f5d234428750a0476c713070334a"
 
     # make sure child subnets/IPs don't get added to whitelist/blacklist
-    target = Target("1.2.3.4/24", "1.2.3.4/28", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/24"}
-    target = Target("1.2.3.4/28", "1.2.3.4/24", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/24"}
-    target = Target("1.2.3.4/28", "1.2.3.4", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/28"}
-    target = Target("1.2.3.4", "1.2.3.4/28", acl_mode=True)
-    assert set(e.data for e in target) == {"1.2.3.0/28"}
+    target = RadixTarget("1.2.3.4/24", "1.2.3.4/28", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/24")}
+    target = RadixTarget("1.2.3.4/28", "1.2.3.4/24", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/24")}
+    target = RadixTarget("1.2.3.4/28", "1.2.3.4", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/28")}
+    target = RadixTarget("1.2.3.4", "1.2.3.4/28", acl_mode=True)
+    assert set(target) == {ip_network("1.2.3.0/28")}
 
     # same but for domains
-    target = Target("evilcorp.com", "www.evilcorp.com", acl_mode=True)
-    assert set(e.data for e in target) == {"evilcorp.com"}
-    target = Target("www.evilcorp.com", "evilcorp.com", acl_mode=True)
-    assert set(e.data for e in target) == {"evilcorp.com"}
+    target = RadixTarget("evilcorp.com", "www.evilcorp.com", acl_mode=True)
+    assert set(target) == {"evilcorp.com"}
+    target = RadixTarget("www.evilcorp.com", "evilcorp.com", acl_mode=True)
+    assert set(target) == {"evilcorp.com"}
 
     # make sure strict_scope doesn't mess us up
-    target = Target("evilcorp.co.uk", "www.evilcorp.co.uk", acl_mode=True, strict_scope=True)
+    target = RadixTarget("evilcorp.co.uk", "www.evilcorp.co.uk", acl_mode=True, strict_dns_scope=True)
     assert set(target.hosts) == {"evilcorp.co.uk", "www.evilcorp.co.uk"}
     assert "evilcorp.co.uk" in target
     assert "www.evilcorp.co.uk" in target
-    assert not "api.evilcorp.co.uk" in target
-    assert not "api.www.evilcorp.co.uk" in target
+    assert "api.evilcorp.co.uk" not in target
+    assert "api.www.evilcorp.co.uk" not in target
 
     # test 'single' boolean argument
-    target = Target("http://evilcorp.com", "evilcorp.com:443")
+    target = ScanSeeds("http://evilcorp.com", "evilcorp.com:443")
     assert "www.evilcorp.com" in target
+    assert "bob@evilcorp.com" in target
     event = target.get("www.evilcorp.com")
     assert event.host == "evilcorp.com"
     events = target.get("www.evilcorp.com", single=False)
     assert len(events) == 2
-    assert set([e.data for e in events]) == {"http://evilcorp.com/", "evilcorp.com:443"}
+    assert {e.data for e in events} == {"http://evilcorp.com/", "evilcorp.com:443"}
+
+
+@pytest.mark.asyncio
+async def test_blacklist_regex(bbot_scanner, bbot_httpserver):
+    from bbot.scanner.target import ScanBlacklist
+
+    blacklist = ScanBlacklist("evilcorp.com")
+    assert blacklist.inputs == {"evilcorp.com"}
+    assert "www.evilcorp.com" in blacklist
+    assert "http://www.evilcorp.com" in blacklist
+    blacklist.add("RE:test")
+    assert "RE:test" in blacklist.inputs
+    assert set(blacklist.inputs) == {"evilcorp.com", "RE:test"}
+    assert blacklist.blacklist_regexes
+    assert next(iter(blacklist.blacklist_regexes)).pattern == "test"
+    result1 = blacklist.get("test.com")
+    assert result1.type == "DNS_NAME"
+    assert result1.data == "test.com"
+    result2 = blacklist.get("www.evilcorp.com")
+    assert result2.type == "DNS_NAME"
+    assert result2.data == "evilcorp.com"
+    result2 = blacklist.get("www.evil.com")
+    assert result2 is None
+    with pytest.raises(KeyError):
+        blacklist.get("www.evil.com", raise_error=True)
+    assert "test.com" in blacklist
+    assert "http://evilcorp.com/test.aspx" in blacklist
+    assert "http://tes.com" not in blacklist
+
+    blacklist = ScanBlacklist("evilcorp.com", r"RE:[0-9]{6}\.aspx$")
+    assert "http://evilcorp.com" in blacklist
+    assert "http://test.com/123456" not in blacklist
+    assert "http://test.com/12345.aspx?a=asdf" not in blacklist
+    assert "http://test.com/asdf/123456.aspx/asdf" not in blacklist
+    assert "http://test.com/asdf/123456.aspx?a=asdf" in blacklist
+    assert "http://test.com/asdf/123456.aspx" in blacklist
+
+    bbot_httpserver.expect_request(uri="/").respond_with_data(
+        """
+        <a href='http://127.0.0.1:8888/asdfevil333asdf'/>
+        <a href='http://127.0.0.1:8888/logout.aspx'/>
+    """
+    )
+    bbot_httpserver.expect_request(uri="/asdfevilasdf").respond_with_data("")
+    bbot_httpserver.expect_request(uri="/logout.aspx").respond_with_data("")
+
+    # make sure URL is detected normally
+    scan = bbot_scanner("http://127.0.0.1:8888/", presets=["spider"], config={"excavate": True}, debug=True)
+    assert {r.pattern for r in scan.target.blacklist.blacklist_regexes} == {r"/.*(sign|log)[_-]?out"}
+    events = [e async for e in scan.async_start()]
+    urls = [e.data for e in events if e.type == "URL"]
+    assert len(urls) == 2
+    assert set(urls) == {"http://127.0.0.1:8888/", "http://127.0.0.1:8888/asdfevil333asdf"}
+
+    # same scan again but with blacklist regex
+    scan = bbot_scanner(
+        "http://127.0.0.1:8888/",
+        blacklist=[r"RE:evil[0-9]{3}"],
+        presets=["spider"],
+        config={"excavate": True},
+        debug=True,
+    )
+    assert scan.target.blacklist.blacklist_regexes
+    assert {r.pattern for r in scan.target.blacklist.blacklist_regexes} == {
+        r"evil[0-9]{3}",
+        r"/.*(sign|log)[_-]?out",
+    }
+    events = [e async for e in scan.async_start()]
+    urls = [e.data for e in events if e.type == "URL"]
+    assert len(urls) == 1
+    assert set(urls) == {"http://127.0.0.1:8888/"}