bbot 2.2.0.5279rc0__py3-none-any.whl → 2.2.0.5311rc0__py3-none-any.whl

bbot/test/test_step_1/test_bloom_filter.py

@@ -66,4 +66,6 @@ async def test_bloom_filter():
     # ensure false positives are less than .02 percent
     assert false_positive_percent < 0.02
 
+    bloom_filter.close()
+
     await scan._cleanup()

bbot/test/test_step_1/test_cli.py

@@ -535,6 +535,13 @@ def test_cli_module_validation(monkeypatch, caplog):
         ]
     )
 
+    # bad target
+    caplog.clear()
+    assert not caplog.text
+    monkeypatch.setattr("sys.argv", ["bbot", "-t", "asdf:::sdf"])
+    cli.main()
+    assert 'Unable to autodetect event type from "asdf:::sdf"' in caplog.text
+
     # incorrect flag
     caplog.clear()
     assert not caplog.text

bbot/test/test_step_1/test_dns.py

@@ -106,7 +106,8 @@ async def test_dns_resolution(bbot_scanner):
     assert "2606:4700:4700::1111" in await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert "one.one.one.one" in await dnsengine.resolve("1.1.1.1")
     for rdtype in ("NS", "SOA", "MX", "TXT"):
-        assert len(await dnsengine.resolve("google.com", type=rdtype)) > 0
+        results = await dnsengine.resolve("google.com", type=rdtype)
+        assert len(results) > 0
 
     # batch resolution
     batch_results = [r async for r in dnsengine.resolve_batch(["1.1.1.1", "one.one.one.one"])]

bbot/test/test_step_1/test_events.py

@@ -42,6 +42,7 @@ async def test_events(events, helpers):
     # ip tests
     assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True)
     assert "8.8.8.8" in events.ipv4
+    assert events.ipv4.host_filterable == "8.8.8.8"
     assert "8.8.8.8" == events.ipv4
     assert "8.8.8.8" in events.netv4
     assert "8.8.8.9" not in events.ipv4
@@ -59,11 +60,19 @@ async def test_events(events, helpers):
     assert events.emoji not in events.ipv4
     assert events.emoji not in events.netv6
     assert events.netv6 not in events.emoji
-    assert "dead::c0de" == scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    assert "dead::c0de" == ipv6_event
+    assert ipv6_event.host_filterable == "dead::c0de"
+    range_to_ip = scan.make_event("1.2.3.4/32", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
+    range_to_ip = scan.make_event("dead::beef/128", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
 
     # hostname tests
     assert events.domain.host == "publicapis.org"
+    assert events.domain.host_filterable == "publicapis.org"
     assert events.subdomain.host == "api.publicapis.org"
+    assert events.subdomain.host_filterable == "api.publicapis.org"
     assert events.domain.host_stem == "publicapis"
     assert events.subdomain.host_stem == "api.publicapis"
     assert "api.publicapis.org" in events.domain
@@ -86,7 +95,11 @@ async def test_events(events, helpers):
         assert "port" not in e.json()
 
     # url tests
-    assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
+    url_no_trailing_slash = scan.make_event("http://evilcorp.com", dummy=True)
+    url_trailing_slash = scan.make_event("http://evilcorp.com/", dummy=True)
+    assert url_no_trailing_slash == url_trailing_slash
+    assert url_no_trailing_slash.host_filterable == "http://evilcorp.com/"
+    assert url_trailing_slash.host_filterable == "http://evilcorp.com/"
     assert events.url_unverified.host == "api.publicapis.org"
     assert events.url_unverified in events.domain
     assert events.url_unverified in events.subdomain
@@ -129,6 +142,7 @@ async def test_events(events, helpers):
     assert events.http_response.port == 80
     assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
+    assert events.http_response.host_filterable == "http://example.com/"
 
     http_response = scan.make_event(
         {

bbot/test/test_step_1/test_helpers.py

@@ -93,8 +93,23 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         ipaddress.ip_network("0.0.0.0/0"),
     ]
     assert helpers.is_ip("127.0.0.1")
+    assert helpers.is_ip("127.0.0.1", include_network=True)
+    assert helpers.is_ip("127.0.0.1", version=4)
+    assert not helpers.is_ip("127.0.0.1", version=6)
     assert not helpers.is_ip("127.0.0.0.1")
 
+    assert helpers.is_ip("dead::beef")
+    assert helpers.is_ip("dead::beef", include_network=True)
+    assert not helpers.is_ip("dead::beef", version=4)
+    assert helpers.is_ip("dead::beef", version=6)
+    assert not helpers.is_ip("dead:::beef")
+
+    assert not helpers.is_ip("1.2.3.4/24")
+    assert helpers.is_ip("1.2.3.4/24", include_network=True)
+    assert not helpers.is_ip("1.2.3.4/24", version=4)
+    assert helpers.is_ip("1.2.3.4/24", include_network=True, version=4)
+    assert not helpers.is_ip("1.2.3.4/24", include_network=True, version=6)
+
     assert not helpers.is_ip_type("127.0.0.1")
     assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"))
     assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True)
@@ -104,6 +119,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False)
 
     assert helpers.is_dns_name("evilcorp.com")
+    assert not helpers.is_dns_name("evilcorp.com:80")
+    assert not helpers.is_dns_name("http://evilcorp.com:80")
     assert helpers.is_dns_name("evilcorp")
     assert not helpers.is_dns_name("evilcorp", include_local=False)
     assert helpers.is_dns_name("ドメイン.テスト")

bbot/test/test_step_1/test_presets.py

@@ -88,9 +88,13 @@ def test_preset_yaml(clean_default_config):
         config={"preset_test_asdf": 1},
     )
     preset1 = preset1.bake()
-    assert "evilcorp.com" in preset1.target
+    assert "evilcorp.com" in preset1.target.seeds
+    assert "evilcorp.ce" not in preset1.target.seeds
+    assert "asdf.www.evilcorp.ce" in preset1.target.seeds
     assert "evilcorp.ce" in preset1.whitelist
+    assert "asdf.evilcorp.ce" in preset1.whitelist
     assert "test.www.evilcorp.ce" in preset1.blacklist
+    assert "asdf.test.www.evilcorp.ce" in preset1.blacklist
     assert "sslcert" in preset1.scan_modules
     assert preset1.whitelisted("evilcorp.ce")
     assert preset1.whitelisted("www.evilcorp.ce")
@@ -170,12 +174,14 @@ def test_preset_scope():
 
     # test target merging
     scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
-    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4", "evilcorp.com"}
-    assert set([e.data for e in scan.target]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4/32", "evilcorp.com"}
+    assert set([e.data for e in scan.target.seeds]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([e.data for e in scan.target.whitelist]) == {"1.2.3.4", "evilcorp.com"}
 
     blank_preset = Preset()
     blank_preset = blank_preset.bake()
-    assert not blank_preset.target
+    assert not blank_preset.target.seeds
+    assert not blank_preset.target.whitelist
     assert blank_preset.strict_scope == False
 
     preset1 = Preset(
@@ -187,10 +193,11 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # make sure target logic works as expected
-    assert "evilcorp.com" in preset1_baked.target
-    assert "asdf.evilcorp.com" in preset1_baked.target
-    assert "asdf.www.evilcorp.ce" in preset1_baked.target
-    assert not "evilcorp.ce" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert not "evilcorp.com" in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.com" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.com" in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "test.www.evilcorp.ce" in preset1_baked.blacklist
     assert not "evilcorp.ce" in preset1_baked.blacklist
@@ -217,17 +224,21 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # targets should be merged
-    assert "evilcorp.com" in preset1_baked.target
-    assert "www.evilcorp.ce" in preset1_baked.target
-    assert "evilcorp.org" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert "www.evilcorp.ce" in preset1_baked.target.seeds
+    assert "evilcorp.org" in preset1_baked.target.seeds
     # strict scope is enabled
-    assert not "asdf.evilcorp.com" in preset1_baked.target
-    assert not "asdf.www.evilcorp.ce" in preset1_baked.target
+    assert not "asdf.www.evilcorp.ce" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.org" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.com" in preset1_baked.target.seeds
+    assert not "asdf.www.evilcorp.ce" in preset1_baked.target.seeds
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.de" in preset1_baked.whitelist
     assert not "asdf.evilcorp.de" in preset1_baked.whitelist
     assert not "asdf.evilcorp.ce" in preset1_baked.whitelist
     # blacklist should be merged, strict scope does not apply
+    assert "test.www.evilcorp.ce" in preset1_baked.blacklist
+    assert "test.www.evilcorp.de" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.ce" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.de" in preset1_baked.blacklist
     assert not "asdf.test.www.evilcorp.org" in preset1_baked.blacklist
@@ -263,14 +274,14 @@ def test_preset_scope():
     }
     assert preset_whitelist_baked.to_dict(include_target=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}},
     }
     assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"otherthing": "asdf"}}},
     }
 
@@ -278,7 +289,8 @@ def test_preset_scope():
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.de")
     assert not preset_nowhitelist_baked.in_scope("1.2.3.4/24")
 
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
+    assert "www.evilcorp.org" in preset_whitelist_baked.target.seeds
+    assert not "www.evilcorp.org" in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.de")
@@ -291,17 +303,17 @@ def test_preset_scope():
     assert preset_whitelist_baked.whitelisted("1.2.3.4/28")
     assert preset_whitelist_baked.whitelisted("1.2.3.4/24")
 
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist_baked.seeds]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
+    assert set([e.data for e in preset_whitelist_baked.seeds]) == {"evilcorp.org"}
+    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24", "http://evilcorp.net/"}
 
     preset_nowhitelist.merge(preset_whitelist)
     preset_nowhitelist_baked = preset_nowhitelist.bake()
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
-    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
-    assert "www.evilcorp.org" in preset_nowhitelist_baked.target
-    assert "www.evilcorp.com" in preset_nowhitelist_baked.target
+    assert set([e.data for e in preset_nowhitelist_baked.seeds]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"1.2.3.0/24", "http://evilcorp.net/"}
+    assert "www.evilcorp.org" in preset_nowhitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_nowhitelist_baked.seeds
     assert "1.2.3.4" in preset_nowhitelist_baked.whitelist
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.com")
@@ -313,10 +325,12 @@ def test_preset_scope():
     preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
     preset_whitelist.merge(preset_nowhitelist)
     preset_whitelist_baked = preset_whitelist.bake()
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_whitelist_baked.seeds]) == {"evilcorp.com", "evilcorp.org"}
     assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24"}
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
-    assert "www.evilcorp.com" in preset_whitelist_baked.target
+    assert "www.evilcorp.org" in preset_whitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_whitelist_baked.seeds
+    assert not "www.evilcorp.org" in preset_whitelist_baked.target.whitelist
+    assert not "www.evilcorp.com" in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.com")
@@ -328,18 +342,18 @@ def test_preset_scope():
     preset_nowhitelist2 = Preset("evilcorp.de")
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
     preset_nowhitelist1.merge(preset_nowhitelist2)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com", "evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
-    assert "www.evilcorp.com" in preset_nowhitelist1_baked.target
-    assert "www.evilcorp.de" in preset_nowhitelist1_baked.target
+    assert "www.evilcorp.com" in preset_nowhitelist1_baked.seeds
+    assert "www.evilcorp.de" in preset_nowhitelist1_baked.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.de" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.whitelist
@@ -356,8 +370,8 @@ def test_preset_scope():
     preset_nowhitelist2.merge(preset_nowhitelist1)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.com", "evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
 

bbot/test/test_step_1/test_python_api.py

@@ -84,6 +84,10 @@ def test_python_api_sync():
 def test_python_api_validation():
     from bbot.scanner import Scanner, Preset
 
+    # invalid target
+    with pytest.raises(ValidationError) as error:
+        Scanner("asdf:::asdf")
+    assert str(error.value) == 'Unable to autodetect event type from "asdf:::asdf"'
     # invalid module
     with pytest.raises(ValidationError) as error:
         Scanner(modules=["asdf"])

bbot/test/test_step_1/test_scan.py

@@ -1,3 +1,5 @@
+from ipaddress import ip_network
+
 from ..bbot_fixtures import *
 
 
@@ -12,6 +14,7 @@ async def test_scan(
         "1.1.1.0",
         "1.1.1.1/31",
         "evilcorp.com",
+        "test.evilcorp.com",
         blacklist=["1.1.1.1/28", "www.evilcorp.com"],
         modules=["ipneighbor"],
     )
@@ -31,8 +34,11 @@ async def test_scan(
     assert not scan0.in_scope("test.www.evilcorp.com")
     assert not scan0.in_scope("www.evilcorp.co.uk")
     j = scan0.json
-    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com"}
-    assert set(j["target"]["whitelist"]) == {"1.1.1.0/31", "evilcorp.com"}
+    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # we preserve the original whitelist inputs
+    assert set(j["target"]["whitelist"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # but in the background they are collapsed
+    assert scan0.target.whitelist.hosts == {ip_network("1.1.1.0/31"), "evilcorp.com"}
     assert set(j["target"]["blacklist"]) == {"1.1.1.0/28", "www.evilcorp.com"}
     assert "ipneighbor" in j["preset"]["modules"]