bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/test/test_step_1/test_events.py

@@ -4,19 +4,28 @@ import ipaddress
 
 from ..bbot_fixtures import *
 from bbot.scanner import Scanner
+from bbot.core.helpers.regexes import event_uuid_regex
 
 
 @pytest.mark.asyncio
 async def test_events(events, helpers):
-
     scan = Scanner()
     await scan._prep()
 
     assert events.ipv4.type == "IP_ADDRESS"
+    assert events.ipv4.netloc == "8.8.8.8"
+    assert events.ipv4.port is None
     assert events.ipv6.type == "IP_ADDRESS"
+    assert events.ipv6.netloc == "[2001:4860:4860::8888]"
+    assert events.ipv6.port is None
+    assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443"
     assert events.netv4.type == "IP_RANGE"
+    assert events.netv4.netloc is None
+    assert "netloc" not in events.netv4.json()
     assert events.netv6.type == "IP_RANGE"
     assert events.domain.type == "DNS_NAME"
+    assert events.domain.netloc == "publicapis.org"
+    assert events.domain.port is None
     assert "domain" in events.domain.tags
     assert events.subdomain.type == "DNS_NAME"
     assert "subdomain" in events.subdomain.tags
@@ -32,6 +41,7 @@ async def test_events(events, helpers):
     # ip tests
     assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True)
     assert "8.8.8.8" in events.ipv4
+    assert events.ipv4.host_filterable == "8.8.8.8"
     assert "8.8.8.8" == events.ipv4
     assert "8.8.8.8" in events.netv4
     assert "8.8.8.9" not in events.ipv4
@@ -49,11 +59,19 @@ async def test_events(events, helpers):
     assert events.emoji not in events.ipv4
     assert events.emoji not in events.netv6
     assert events.netv6 not in events.emoji
-    assert "dead::c0de" == scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    assert "dead::c0de" == ipv6_event
+    assert ipv6_event.host_filterable == "dead::c0de"
+    range_to_ip = scan.make_event("1.2.3.4/32", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
+    range_to_ip = scan.make_event("dead::beef/128", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
 
     # hostname tests
     assert events.domain.host == "publicapis.org"
+    assert events.domain.host_filterable == "publicapis.org"
     assert events.subdomain.host == "api.publicapis.org"
+    assert events.subdomain.host_filterable == "api.publicapis.org"
     assert events.domain.host_stem == "publicapis"
     assert events.subdomain.host_stem == "api.publicapis"
     assert "api.publicapis.org" in events.domain
@@ -62,23 +80,39 @@ async def test_events(events, helpers):
     assert "fsocie.ty" not in events.subdomain
     assert events.subdomain in events.domain
     assert events.domain not in events.subdomain
-    assert not events.ipv4 in events.domain
-    assert not events.netv6 in events.domain
+    assert events.ipv4 not in events.domain
+    assert events.netv6 not in events.domain
     assert events.emoji not in events.domain
     assert events.domain not in events.emoji
-    assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
-    assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
+    dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    for e in (open_port_event, dns_event):
+        assert "evilcorp.com" == e
+        assert e.netloc == "evilcorp.com"
+        assert e.json()["netloc"] == "evilcorp.com"
+        assert e.port is None
+        assert "port" not in e.json()
 
     # url tests
-    assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
+    url_no_trailing_slash = scan.make_event("http://evilcorp.com", dummy=True)
+    url_trailing_slash = scan.make_event("http://evilcorp.com/", dummy=True)
+    assert url_no_trailing_slash == url_trailing_slash
+    assert url_no_trailing_slash.host_filterable == "http://evilcorp.com/"
+    assert url_trailing_slash.host_filterable == "http://evilcorp.com/"
     assert events.url_unverified.host == "api.publicapis.org"
     assert events.url_unverified in events.domain
     assert events.url_unverified in events.subdomain
     assert "api.publicapis.org:443" in events.url_unverified
     assert "publicapis.org" not in events.url_unverified
     assert events.ipv4_url_unverified in events.ipv4
+    assert events.ipv4_url_unverified.netloc == "8.8.8.8:443"
+    assert events.ipv4_url_unverified.port == 443
+    assert events.ipv4_url_unverified.json()["port"] == 443
     assert events.ipv4_url_unverified in events.netv4
     assert events.ipv6_url_unverified in events.ipv6
+    assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443"
+    assert events.ipv6_url_unverified.port == 443
+    assert events.ipv6_url_unverified.json()["port"] == 443
     assert events.ipv6_url_unverified in events.netv6
     assert events.emoji not in events.url_unverified
     assert events.emoji not in events.ipv6_url_unverified
@@ -107,6 +141,7 @@ async def test_events(events, helpers):
     assert events.http_response.port == 80
     assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
+    assert events.http_response.host_filterable == "http://example.com/"
 
     http_response = scan.make_event(
         {
@@ -171,7 +206,7 @@ async def test_events(events, helpers):
 
     # scope distance
     event1 = scan.make_event("1.2.3.4", dummy=True)
-    assert event1._scope_distance == None
+    assert event1._scope_distance is None
     event1.scope_distance = 0
     assert event1._scope_distance == 0
     event2 = scan.make_event("2.3.4.5", parent=event1)
@@ -192,6 +227,8 @@ async def test_events(events, helpers):
 
     org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
+    assert org_stub_1.netloc is None
+    assert "netloc" not in org_stub_1.json()
     org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 
@@ -199,7 +236,7 @@ async def test_events(events, helpers):
     root_event = scan.make_event("0.0.0.0", dummy=True)
     root_event.scope_distance = 0
     internal_event1 = scan.make_event("1.2.3.4", parent=root_event, internal=True)
-    assert internal_event1._internal == True
+    assert internal_event1._internal is True
     assert "internal" in internal_event1.tags
 
     # tag inheritance
@@ -231,8 +268,8 @@ async def test_events(events, helpers):
     # updating module
     event3 = scan.make_event("127.0.0.1", parent=scan.root_event)
     updated_event = scan.make_event(event3, internal=True)
-    assert event3.internal == False
-    assert updated_event.internal == True
+    assert event3.internal is False
+    assert updated_event.internal is True
 
     # event sorting
     parent1 = scan.make_event("127.0.0.1", parent=scan.root_event)
@@ -412,17 +449,55 @@ async def test_events(events, helpers):
         == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ"
     )
 
+    # test event uuid
+    import uuid
+
+    parent_event1 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+    parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+
+    event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context")
+    assert hasattr(event1, "_uuid")
+    assert hasattr(event1, "uuid")
+    assert isinstance(event1._uuid, uuid.UUID)
+    assert isinstance(event1.uuid, str)
+    assert event1.uuid == f"{event1.type}:{event1._uuid}"
+    event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context")
+    assert hasattr(event2, "_uuid")
+    assert hasattr(event2, "uuid")
+    assert isinstance(event2._uuid, uuid.UUID)
+    assert isinstance(event2.uuid, str)
+    assert event2.uuid == f"{event2.type}:{event2._uuid}"
+    # ids should match because the event type + data is the same
+    assert event1.id == event2.id
+    # but uuids should be unique!
+    assert event1.uuid != event2.uuid
+    # parent ids should match
+    assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id
+    # uuids should not
+    assert event1.parent_uuid == parent_event1.uuid
+    assert event2.parent_uuid == parent_event2.uuid
+    assert event1.parent_uuid != event2.parent_uuid
+
     # test event serialization
     from bbot.core.event import event_from_json
 
     db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context")
+    assert db_event.parent == scan.root_event
+    assert db_event.parent is scan.root_event
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     assert db_event.discovery_context == "test context"
     assert db_event.discovery_path == ["test context"]
-    assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert len(db_event.parent_chain) == 1
+    assert all(event_uuid_regex.match(u) for u in db_event.parent_chain)
+    assert db_event.parent_chain[0] == str(db_event.uuid)
+    assert db_event.parent.uuid == scan.root_event.uuid
+    assert db_event.parent_uuid == scan.root_event.uuid
     timestamp = db_event.timestamp.isoformat()
     json_event = db_event.json()
+    assert isinstance(json_event["uuid"], str)
+    assert json_event["uuid"] == str(db_event.uuid)
+    assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"
     assert json_event["type"] == "OPEN_TCP_PORT"
@@ -430,8 +505,14 @@ async def test_events(events, helpers):
     assert json_event["timestamp"] == timestamp
     assert json_event["discovery_context"] == "test context"
     assert json_event["discovery_path"] == ["test context"]
-    assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert json_event["parent_chain"] == db_event.parent_chain
+    assert json_event["parent_chain"][0] == str(db_event.uuid)
     reconstituted_event = event_from_json(json_event)
+    assert isinstance(reconstituted_event._uuid, uuid.UUID)
+    assert str(reconstituted_event.uuid) == json_event["uuid"]
+    assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"]
+    assert reconstituted_event.uuid == db_event.uuid
+    assert reconstituted_event.parent_uuid == scan.root_event.uuid
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.isoformat() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
@@ -439,13 +520,13 @@ async def test_events(events, helpers):
     assert reconstituted_event.host == "evilcorp.com"
     assert reconstituted_event.discovery_context == "test context"
     assert reconstituted_event.discovery_path == ["test context"]
-    assert reconstituted_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert reconstituted_event.parent_chain == db_event.parent_chain
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
     assert hostless_event_json["type"] == "ASDF"
     assert hostless_event_json["data"] == "asdf"
-    assert not "host" in hostless_event_json
+    assert "host" not in hostless_event_json
 
     # SIEM-friendly serialize/deserialize
     json_event_siemfriendly = db_event.json(siem_friendly=True)
@@ -535,7 +616,6 @@ async def test_events(events, helpers):
 
 @pytest.mark.asyncio
 async def test_event_discovery_context():
-
     from bbot.modules.base import BaseModule
 
     scan = Scanner("evilcorp.com")
@@ -616,7 +696,7 @@ async def test_event_discovery_context():
     )
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 6
+    assert len(events) == 7
 
     assert 1 == len(
         [
@@ -723,7 +803,7 @@ async def test_event_web_spider_distance(bbot_scanner):
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     social_event = scan.make_event(
         {"platform": "github", "url": "http://www.evilcorp.com/test4"}, "SOCIAL", parent=url_event_3
     )
@@ -746,42 +826,42 @@ async def test_event_web_spider_distance(bbot_scanner):
 
     url_event = scan.make_event("http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event)
     assert url_event.web_spider_distance == 0
-    assert not "spider-danger" in url_event.tags
-    assert not "spider-max" in url_event.tags
+    assert "spider-danger" not in url_event.tags
+    assert "spider-max" not in url_event.tags
     url_event_2 = scan.make_event(
         "http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event, tags="spider-danger"
     )
     # spider distance shouldn't increment because it's not the same host
     assert url_event_2.web_spider_distance == 0
     assert "spider-danger" in url_event_2.tags
-    assert not "spider-max" in url_event_2.tags
+    assert "spider-max" not in url_event_2.tags
     url_event_3 = scan.make_event(
         "http://www.evilcorp.com/3", "URL_UNVERIFIED", parent=url_event_2, tags="spider-danger"
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     url_event_4 = scan.make_event("http://evilcorp.com", "URL_UNVERIFIED", parent=url_event_3)
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.add_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
     assert "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.remove_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_5 = scan.make_event("http://evilcorp.com/5", "URL_UNVERIFIED", parent=url_event_4)
     assert url_event_5.web_spider_distance == 0
-    assert not "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-danger" not in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
     url_event_5.add_tag("spider-danger")
     # if host is the same as parent, web spider distance should auto-increment after adding spider-danger tag
     assert url_event_5.web_spider_distance == 1
     assert "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
 
 
 def test_event_confidence():
@@ -856,3 +936,76 @@ def test_event_closest_host():
         vuln = scan.make_event(
             {"path": "/tmp/asdf.txt", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
         )
+
+
+def test_event_magic():
+    from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+    import base64
+
+    zip_base64 = "UEsDBAoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIAAAAYXNkZi50eHRhc2RmClBLAQI/AwoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIACQAAAAAAAAAIICkgQAAAABhc2RmLnR4dAoAIAAAAAAAAQAYAICi2B77MNsBgKLYHvsw2wGAotge+zDbAVBLBQYAAAAAAQABAFoAAAArAAAAAAA="
+    zip_bytes = base64.b64decode(zip_base64)
+    zip_file = Path("/tmp/.bbottestzipasdkfjalsdf.zip")
+    with open(zip_file, "wb") as f:
+        f.write(zip_bytes)
+
+    # test magic helpers
+    extension, mime_type, description, confidence = get_magic_info(zip_file)
+    assert extension == ".zip"
+    assert mime_type == "application/zip"
+    assert description == "PKZIP Archive file"
+    assert confidence > 0
+    assert get_compression(mime_type) == "zip"
+
+    # test filesystem event - file
+    scan = Scanner()
+    event = scan.make_event({"path": zip_file}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {
+        "path": "/tmp/.bbottestzipasdkfjalsdf.zip",
+        "magic_extension": ".zip",
+        "magic_mime_type": "application/zip",
+        "magic_description": "PKZIP Archive file",
+        "magic_confidence": 0.9,
+        "compression": "zip",
+    }
+    assert event.tags == {"file", "zip-archive", "compressed"}
+
+    # test filesystem event - folder
+    scan = Scanner()
+    event = scan.make_event({"path": "/tmp"}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {"path": "/tmp"}
+    assert event.tags == {"folder"}
+
+    zip_file.unlink()
+
+
+def test_event_hashing():
+    scan = Scanner("example.com")
+    url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
+    host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
+    host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
+    finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
+    finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
+    finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+    finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+
+    assert finding1.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "www.example.com",
+    }
+    assert finding2.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding3.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding1.id != finding2.id
+    assert finding2.id == finding3.id
+    assert finding1.data_id != finding2.data_id
+    assert finding2.data_id == finding3.data_id
+    assert finding1.data_hash != finding2.data_hash
+    assert finding2.data_hash == finding3.data_hash
+    assert hash(finding1) != hash(finding2)
+    assert hash(finding2) == hash(finding3)

bbot/test/test_step_1/test_helpers.py

@@ -49,6 +49,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.url_depth("http://evilcorp.com/") == 0
     assert helpers.url_depth("http://evilcorp.com") == 0
 
+    assert helpers.parent_url("http://evilcorp.com/subdir1/subdir2?foo=bar") == "http://evilcorp.com/subdir1"
+
     ### MISC ###
     assert helpers.is_domain("evilcorp.co.uk")
     assert not helpers.is_domain("www.evilcorp.co.uk")
@@ -62,8 +64,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert not helpers.is_subdomain("notreal")
     assert helpers.is_url("http://evilcorp.co.uk/asdf?a=b&c=d#asdf")
     assert helpers.is_url("https://evilcorp.co.uk/asdf?a=b&c=d#asdf")
-    assert helpers.is_uri("ftp://evilcorp.co.uk") == True
-    assert helpers.is_uri("http://evilcorp.co.uk") == True
+    assert helpers.is_uri("ftp://evilcorp.co.uk") is True
+    assert helpers.is_uri("http://evilcorp.co.uk") is True
     assert helpers.is_uri("evilcorp.co.uk", return_scheme=True) == ""
     assert helpers.is_uri("ftp://evilcorp.co.uk", return_scheme=True) == "ftp"
     assert helpers.is_uri("FTP://evilcorp.co.uk", return_scheme=True) == "ftp"
@@ -91,8 +93,34 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         ipaddress.ip_network("0.0.0.0/0"),
     ]
     assert helpers.is_ip("127.0.0.1")
+    assert helpers.is_ip("127.0.0.1", include_network=True)
+    assert helpers.is_ip("127.0.0.1", version=4)
+    assert not helpers.is_ip("127.0.0.1", version=6)
     assert not helpers.is_ip("127.0.0.0.1")
+
+    assert helpers.is_ip("dead::beef")
+    assert helpers.is_ip("dead::beef", include_network=True)
+    assert not helpers.is_ip("dead::beef", version=4)
+    assert helpers.is_ip("dead::beef", version=6)
+    assert not helpers.is_ip("dead:::beef")
+
+    assert not helpers.is_ip("1.2.3.4/24")
+    assert helpers.is_ip("1.2.3.4/24", include_network=True)
+    assert not helpers.is_ip("1.2.3.4/24", version=4)
+    assert helpers.is_ip("1.2.3.4/24", include_network=True, version=4)
+    assert not helpers.is_ip("1.2.3.4/24", include_network=True, version=6)
+
+    assert not helpers.is_ip_type("127.0.0.1")
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"))
+    assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True)
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=False)
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"))
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=True)
+    assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False)
+
     assert helpers.is_dns_name("evilcorp.com")
+    assert not helpers.is_dns_name("evilcorp.com:80")
+    assert not helpers.is_dns_name("http://evilcorp.com:80")
     assert helpers.is_dns_name("evilcorp")
     assert not helpers.is_dns_name("evilcorp", include_local=False)
     assert helpers.is_dns_name("ドメイン.テスト")
@@ -210,8 +238,12 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
 
     ipv4_netloc = helpers.make_netloc("192.168.1.1", 80)
     assert ipv4_netloc == "192.168.1.1:80"
-    ipv6_netloc = helpers.make_netloc("dead::beef", "443")
-    assert ipv6_netloc == "[dead::beef]:443"
+    assert helpers.make_netloc("192.168.1.1") == "192.168.1.1"
+    assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1"), None) == "192.168.1.1"
+    assert helpers.make_netloc("dead::beef", "443") == "[dead::beef]:443"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), 443) == "[dead::beef]:443"
+    assert helpers.make_netloc("dead::beef", None) == "[dead::beef]"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), None) == "[dead::beef]"
 
     assert helpers.get_file_extension("https://evilcorp.com/evilcorp.com/test/asdf.TXT") == "txt"
     assert helpers.get_file_extension("/etc/conf/test.tar.gz") == "gz"
@@ -251,7 +283,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     replaced = helpers.search_format_dict(
         {"asdf": [{"wat": {"here": "#{replaceme}!"}}, {500: True}]}, replaceme="asdf"
     )
-    assert replaced["asdf"][1][500] == True
+    assert replaced["asdf"][1][500] is True
     assert replaced["asdf"][0]["wat"]["here"] == "asdf!"
 
     filtered_dict = helpers.filter_dict(
@@ -283,7 +315,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         fuzzy=True,
         exclude_keys="modules",
     )
-    assert not "secrets_db" in filtered_dict4["modules"]
+    assert "secrets_db" not in filtered_dict4["modules"]
     assert "ipneighbor" in filtered_dict4["modules"]
     assert "secret" in filtered_dict4["modules"]["ipneighbor"]
     assert "asdf" not in filtered_dict4["modules"]["ipneighbor"]
@@ -376,15 +408,15 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.validators.validate_host("LOCALHOST ") == "localhost"
     assert helpers.validators.validate_host(" 192.168.1.1") == "192.168.1.1"
     assert helpers.validators.validate_host(" Dead::c0dE ") == "dead::c0de"
-    assert helpers.validators.soft_validate(" evilCorp.COM", "host") == True
-    assert helpers.validators.soft_validate("!@#$", "host") == False
+    assert helpers.validators.soft_validate(" evilCorp.COM", "host") is True
+    assert helpers.validators.soft_validate("!@#$", "host") is False
     with pytest.raises(ValueError):
         assert helpers.validators.validate_host("!@#$")
     # ports
     assert helpers.validators.validate_port(666) == 666
     assert helpers.validators.validate_port(666666) == 65535
-    assert helpers.validators.soft_validate(666, "port") == True
-    assert helpers.validators.soft_validate("!@#$", "port") == False
+    assert helpers.validators.soft_validate(666, "port") is True
+    assert helpers.validators.soft_validate("!@#$", "port") is False
     with pytest.raises(ValueError):
         helpers.validators.validate_port("asdf")
     # top tcp ports
@@ -396,7 +428,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert top_tcp_ports[-10:] == [65526, 65527, 65528, 65529, 65530, 65531, 65532, 65533, 65534, 65535]
     assert len(top_tcp_ports) == 65535
     assert len(set(top_tcp_ports)) == 65535
-    assert all([isinstance(i, int) for i in top_tcp_ports])
+    assert all(isinstance(i, int) for i in top_tcp_ports)
     top_tcp_ports = helpers.top_tcp_ports(10, as_string=True)
     assert top_tcp_ports == "80,23,443,21,22,25,3389,110,445,139"
     # urls
@@ -405,25 +437,29 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         helpers.validators.validate_url_parsed(" httP://evilcorP.com/asdf?a=b&c=d#e").geturl()
         == "http://evilcorp.com/asdf"
     )
-    assert helpers.validators.soft_validate(" httP://evilcorP.com/asdf?a=b&c=d#e", "url") == True
-    assert helpers.validators.soft_validate("!@#$", "url") == False
+    assert helpers.validators.soft_validate(" httP://evilcorP.com/asdf?a=b&c=d#e", "url") is True
+    assert helpers.validators.soft_validate("!@#$", "url") is False
     with pytest.raises(ValueError):
         helpers.validators.validate_url("!@#$")
     # severities
     assert helpers.validators.validate_severity(" iNfo") == "INFO"
-    assert helpers.validators.soft_validate(" iNfo", "severity") == True
-    assert helpers.validators.soft_validate("NOPE", "severity") == False
+    assert helpers.validators.soft_validate(" iNfo", "severity") is True
+    assert helpers.validators.soft_validate("NOPE", "severity") is False
     with pytest.raises(ValueError):
         helpers.validators.validate_severity("NOPE")
     # emails
     assert helpers.validators.validate_email(" bOb@eViLcorp.COM") == "bob@evilcorp.com"
-    assert helpers.validators.soft_validate(" bOb@eViLcorp.COM", "email") == True
-    assert helpers.validators.soft_validate("!@#$", "email") == False
+    assert helpers.validators.soft_validate(" bOb@eViLcorp.COM", "email") is True
+    assert helpers.validators.soft_validate("!@#$", "email") is False
     with pytest.raises(ValueError):
         helpers.validators.validate_email("!@#$")
 
     assert type(helpers.make_date()) == str
 
+    # string formatter
+    s = "asdf {unused} {used}"
+    assert helpers.safe_format(s, used="fdsa") == "asdf {unused} fdsa"
+
     # punycode
     assert helpers.smart_encode_punycode("ドメイン.テスト") == "xn--eckwd4c7c.xn--zckzah"
     assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah") == "ドメイン.テスト"
@@ -497,9 +533,9 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     truncated_filename.unlink()
 
     # misc DNS helpers
-    assert helpers.is_ptr("wsc-11-22-33-44-wat.evilcorp.com") == True
-    assert helpers.is_ptr("wsc-11-22-33-wat.evilcorp.com") == False
-    assert helpers.is_ptr("11wat.evilcorp.com") == False
+    assert helpers.is_ptr("wsc-11-22-33-44-wat.evilcorp.com") is True
+    assert helpers.is_ptr("wsc-11-22-33-wat.evilcorp.com") is False
+    assert helpers.is_ptr("11wat.evilcorp.com") is False
 
     ## NTLM
     testheader = "TlRMTVNTUAACAAAAHgAeADgAAAAVgorilwL+bvnVipUAAAAAAAAAAJgAmABWAAAACgBjRQAAAA9XAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAACAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgAAQAeAFcASQBOAC0AUwA0ADIATgBPAEIARABWAFQASwA4AAQAHgBXAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAADAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgABwAIAHUwOZlfoNgBAAAAAA=="
@@ -577,8 +613,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         assert len(helpers.get_exception_chain(e)) == 2
         assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, KeyboardInterrupt)]) == 1
         assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1
-        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) == True
-        assert helpers.in_exception_chain(e, (TypeError, OSError)) == False
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) is True
+        assert helpers.in_exception_chain(e, (TypeError, OSError)) is False
         test_ran = True
     assert test_ran
     test_ran = False
@@ -591,9 +627,9 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         assert len(helpers.get_exception_chain(e)) == 2
         assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, AttributeError)]) == 1
         assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1
-        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) == False
-        assert helpers.in_exception_chain(e, (KeyboardInterrupt, AttributeError)) == True
-        assert helpers.in_exception_chain(e, (AttributeError,)) == True
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) is False
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, AttributeError)) is True
+        assert helpers.in_exception_chain(e, (AttributeError,)) is True
         test_ran = True
     assert test_ran
 
@@ -821,7 +857,6 @@ def test_liststring_invalidfnchars(helpers):
 # test parameter validation
 @pytest.mark.asyncio
 async def test_parameter_validation(helpers):
-
     getparam_valid_params = {
         "name",
         "age",
@@ -850,7 +885,7 @@ async def test_parameter_validation(helpers):
         if helpers.validate_parameter(p, "getparam"):
             assert p in getparam_valid_params and p not in getparam_invalid_params
         else:
-            assert p in getparam_invalid_params and not p in getparam_valid_params
+            assert p in getparam_invalid_params and p not in getparam_valid_params
 
     header_valid_params = {
         "name",
@@ -881,7 +916,7 @@ async def test_parameter_validation(helpers):
         if helpers.validate_parameter(p, "header"):
             assert p in header_valid_params and p not in header_invalid_params
         else:
-            assert p in header_invalid_params and not p in header_valid_params
+            assert p in header_invalid_params and p not in header_valid_params
 
     cookie_valid_params = {
         "name",
@@ -911,4 +946,4 @@ async def test_parameter_validation(helpers):
         if helpers.validate_parameter(p, "cookie"):
             assert p in cookie_valid_params and p not in cookie_invalid_params
         else:
-            assert p in cookie_invalid_params and not p in cookie_valid_params
+            assert p in cookie_invalid_params and p not in cookie_valid_params

bbot/test/test_step_1/test_manager_deduplication.py

@@ -91,7 +91,7 @@ async def test_manager_deduplication(bbot_scanner):
         _dns_mock=dns_mock_chain,
     )
 
-    assert len(events) == 21
+    assert len(events) == 22
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])