bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/test/test_step_1/test_dns.py

@@ -23,7 +23,7 @@ async def test_dns_engine(bbot_scanner):
     )
     result = await scan.helpers.resolve("one.one.one.one")
     assert "1.1.1.1" in result
-    assert not "2606:4700:4700::1111" in result
+    assert "2606:4700:4700::1111" not in result
 
     results = [_ async for _ in scan.helpers.resolve_batch(("one.one.one.one", "1.1.1.1"))]
     pass_1 = False
@@ -85,12 +85,12 @@ async def test_dns_resolution(bbot_scanner):
     for answer in answers:
         responses += list(extract_targets(answer))
     assert ("A", "1.1.1.1") in responses
-    assert not ("AAAA", "2606:4700:4700::1111") in responses
+    assert ("AAAA", "2606:4700:4700::1111") not in responses
     answers, errors = await dnsengine.resolve_raw("one.one.one.one", type="AAAA")
     responses = []
     for answer in answers:
         responses += list(extract_targets(answer))
-    assert not ("A", "1.1.1.1") in responses
+    assert ("A", "1.1.1.1") not in responses
     assert ("AAAA", "2606:4700:4700::1111") in responses
     answers, errors = await dnsengine.resolve_raw("1.1.1.1")
     responses = []
@@ -106,13 +106,14 @@ async def test_dns_resolution(bbot_scanner):
     assert "2606:4700:4700::1111" in await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert "one.one.one.one" in await dnsengine.resolve("1.1.1.1")
     for rdtype in ("NS", "SOA", "MX", "TXT"):
-        assert len(await dnsengine.resolve("google.com", type=rdtype)) > 0
+        results = await dnsengine.resolve("google.com", type=rdtype)
+        assert len(results) > 0
 
     # batch resolution
     batch_results = [r async for r in dnsengine.resolve_batch(["1.1.1.1", "one.one.one.one"])]
     assert len(batch_results) == 2
     batch_results = dict(batch_results)
-    assert any([x in batch_results["one.one.one.one"] for x in ("1.1.1.1", "1.0.0.1")])
+    assert any(x in batch_results["one.one.one.one"] for x in ("1.1.1.1", "1.0.0.1"))
     assert "one.one.one.one" in batch_results["1.1.1.1"]
 
     # custom batch resolution
@@ -140,11 +141,11 @@ async def test_dns_resolution(bbot_scanner):
     assert hash(("1.1.1.1", "PTR")) in dnsengine._dns_cache
     await dnsengine.resolve("one.one.one.one", type="A")
     assert hash(("one.one.one.one", "A")) in dnsengine._dns_cache
-    assert not hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache
     dnsengine._dns_cache.clear()
     await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
-    assert not hash(("one.one.one.one", "A")) in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache
 
     await dnsengine._shutdown()
 
@@ -164,7 +165,7 @@ async def test_dns_resolution(bbot_scanner):
     assert "A" in resolved_hosts_event1.raw_dns_records
     assert "AAAA" in resolved_hosts_event1.raw_dns_records
     assert "a-record" in resolved_hosts_event1.tags
-    assert not "a-record" in resolved_hosts_event2.tags
+    assert "a-record" not in resolved_hosts_event2.tags
 
     scan2 = bbot_scanner("evilcorp.com", config={"dns": {"minimal": False}})
     await scan2.helpers.dns._mock_dns(
@@ -197,7 +198,7 @@ async def test_wildcards(bbot_scanner):
     assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2)
     for rdtype in all_rdtypes:
         assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache
-        if not rdtype in ("A", "AAAA"):
+        if rdtype not in ("A", "AAAA"):
             assert hash(("asdf.github.io", rdtype)) in dnsengine._wildcard_cache
     assert "github.io" in wildcard_domains
     assert "A" in wildcard_domains["github.io"]
@@ -263,7 +264,7 @@ def custom_lookup(query, rdtype):
     await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 11
+    assert len(events) == 12
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -320,7 +321,7 @@ def custom_lookup(query, rdtype):
     await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 11
+    assert len(events) == 12
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -418,7 +419,7 @@ def custom_lookup(query, rdtype):
 
     events = [e async for e in scan.async_start()]
 
-    assert len(events) == 10
+    assert len(events) == 11
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -546,8 +547,8 @@ def custom_lookup(query, rdtype):
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
-    assert len(events) == 3
-    assert 1 == len([e for e in events if e.type == "SCAN"])
+    assert len(events) == 4
+    assert 2 == len([e for e in events if e.type == "SCAN"])
     unmodified_wildcard_events = [
         e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io"
     ]
@@ -592,8 +593,8 @@ def custom_lookup(query, rdtype):
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
-    assert len(events) == 3
-    assert 1 == len([e for e in events if e.type == "SCAN"])
+    assert len(events) == 4
+    assert 2 == len([e for e in events if e.type == "SCAN"])
     unmodified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and "_wildcard" not in e.data]
     assert len(unmodified_wildcard_events) == 2
     assert 1 == len(
@@ -631,6 +632,42 @@ def custom_lookup(query, rdtype):
     assert len(modified_wildcard_events) == 0
 
 
+@pytest.mark.asyncio
+async def test_wildcard_deduplication(bbot_scanner):
+
+    custom_lookup = """
+def custom_lookup(query, rdtype):
+    if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"):
+        return {""}
+"""
+
+    mock_data = {
+        "evilcorp.com": {"A": ["127.0.0.1"]},
+    }
+
+    from bbot.modules.base import BaseModule
+
+    class DummyModule(BaseModule):
+        watched_events = ["DNS_NAME"]
+        per_domain_only = True
+
+        async def handle_event(self, event):
+            for i in range(30):
+                await self.emit_event(f"www{i}.evilcorp.com", "DNS_NAME", parent=event)
+
+    # scan without omitted event type
+    scan = bbot_scanner(
+        "evilcorp.com", config={"dns": {"minimal": False, "wildcard_ignore": []}, "omit_event_types": []}
+    )
+    await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
+    dummy_module = DummyModule(scan)
+    scan.modules["dummy_module"] = dummy_module
+    events = [e async for e in scan.async_start()]
+    dns_name_events = [e for e in events if e.type == "DNS_NAME"]
+    assert len(dns_name_events) == 2
+    assert 1 == len([e for e in dns_name_events if e.data == "_wildcard.evilcorp.com"])
+
+
 @pytest.mark.asyncio
 async def test_dns_raw_records(bbot_scanner):
 
@@ -696,7 +733,7 @@ async def test_dns_raw_records(bbot_scanner):
     dummy_module = DummyModule(scan)
     scan.modules["dummy_module"] = dummy_module
     events = [e async for e in scan.async_start()]
-    # no raw records should be ouptut
+    # no raw records should be output
     assert 0 == len([e for e in events if e.type == "RAW_DNS_RECORD"])
     # but they should still make it to the module
     assert 1 == len(
@@ -729,7 +766,7 @@ async def test_dns_graph_structure(bbot_scanner):
         }
     )
     events = [e async for e in scan.async_start()]
-    assert len(events) == 5
+    assert len(events) == 6
     non_scan_events = [e for e in events if e.type != "SCAN"]
     assert sorted([e.type for e in non_scan_events]) == ["DNS_NAME", "DNS_NAME", "DNS_NAME", "URL_UNVERIFIED"]
     events_by_data = {e.data: e for e in non_scan_events}
@@ -744,16 +781,16 @@ async def test_dns_graph_structure(bbot_scanner):
 
 @pytest.mark.asyncio
 async def test_dns_helpers(bbot_scanner):
-    assert service_record("") == False
-    assert service_record("localhost") == False
-    assert service_record("www.example.com") == False
-    assert service_record("www.example.com", "SRV") == True
-    assert service_record("_custom._service.example.com", "SRV") == True
-    assert service_record("_custom._service.example.com", "A") == False
+    assert service_record("") is False
+    assert service_record("localhost") is False
+    assert service_record("www.example.com") is False
+    assert service_record("www.example.com", "SRV") is True
+    assert service_record("_custom._service.example.com", "SRV") is True
+    assert service_record("_custom._service.example.com", "A") is False
     # top 100 most common SRV records
     for srv_record in common_srvs[:100]:
         hostname = f"{srv_record}.example.com"
-        assert service_record(hostname) == True
+        assert service_record(hostname) is True
 
     # make sure system nameservers are excluded from use by DNS brute force
     brute_nameservers = tempwordlist(["1.2.3.4", "8.8.4.4", "4.3.2.1", "8.8.8.8"])

bbot/test/test_step_1/test_engine.py

@@ -72,7 +72,7 @@ async def test_engine():
 
     # test async generator
     assert counter == 0
-    assert yield_cancelled == False
+    assert yield_cancelled is False
     yield_res = [r async for r in test_engine.yield_stuff(13)]
     assert yield_res == [f"thing{i}" for i in range(13)]
     assert len(yield_res) == 13
@@ -88,8 +88,8 @@ async def test_engine():
             await agen.aclose()
             break
     await asyncio.sleep(5)
-    assert yield_cancelled == True
-    assert yield_errored == False
+    assert yield_cancelled is True
+    assert yield_errored is False
     assert counter < 15
 
     # test async generator with error
@@ -99,8 +99,8 @@ async def test_engine():
     with pytest.raises(BBOTEngineError):
         async for _ in agen:
             pass
-    assert yield_cancelled == False
-    assert yield_errored == True
+    assert yield_cancelled is False
+    assert yield_errored is True
 
     # test return with cancellation
     return_started = False
@@ -113,10 +113,10 @@ async def test_engine():
     with pytest.raises(asyncio.CancelledError):
         await task
     await asyncio.sleep(0.1)
-    assert return_started == True
-    assert return_finished == False
-    assert return_cancelled == True
-    assert return_errored == False
+    assert return_started is True
+    assert return_finished is False
+    assert return_cancelled is True
+    assert return_errored is False
 
     # test return with late cancellation
     return_started = False
@@ -128,10 +128,10 @@ async def test_engine():
     task.cancel()
     result = await task
     assert result == "thing1"
-    assert return_started == True
-    assert return_finished == True
-    assert return_cancelled == False
-    assert return_errored == False
+    assert return_started is True
+    assert return_finished is True
+    assert return_cancelled is False
+    assert return_errored is False
 
     # test return with error
     return_started = False
@@ -140,9 +140,9 @@ async def test_engine():
     return_errored = False
     with pytest.raises(BBOTEngineError):
         result = await test_engine.return_thing(None)
-    assert return_started == True
-    assert return_finished == False
-    assert return_cancelled == False
-    assert return_errored == True
+    assert return_started is True
+    assert return_finished is False
+    assert return_cancelled is False
+    assert return_errored is True
 
     await test_engine.shutdown()

bbot/test/test_step_1/test_events.py

@@ -4,6 +4,7 @@ import ipaddress
 
 from ..bbot_fixtures import *
 from bbot.scanner import Scanner
+from bbot.core.helpers.regexes import event_uuid_regex
 
 
 @pytest.mark.asyncio
@@ -13,10 +14,19 @@ async def test_events(events, helpers):
     await scan._prep()
 
     assert events.ipv4.type == "IP_ADDRESS"
+    assert events.ipv4.netloc == "8.8.8.8"
+    assert events.ipv4.port is None
     assert events.ipv6.type == "IP_ADDRESS"
+    assert events.ipv6.netloc == "[2001:4860:4860::8888]"
+    assert events.ipv6.port is None
+    assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443"
     assert events.netv4.type == "IP_RANGE"
+    assert events.netv4.netloc is None
+    assert "netloc" not in events.netv4.json()
     assert events.netv6.type == "IP_RANGE"
     assert events.domain.type == "DNS_NAME"
+    assert events.domain.netloc == "publicapis.org"
+    assert events.domain.port is None
     assert "domain" in events.domain.tags
     assert events.subdomain.type == "DNS_NAME"
     assert "subdomain" in events.subdomain.tags
@@ -32,6 +42,7 @@ async def test_events(events, helpers):
     # ip tests
     assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True)
     assert "8.8.8.8" in events.ipv4
+    assert events.ipv4.host_filterable == "8.8.8.8"
     assert "8.8.8.8" == events.ipv4
     assert "8.8.8.8" in events.netv4
     assert "8.8.8.9" not in events.ipv4
@@ -49,11 +60,19 @@ async def test_events(events, helpers):
     assert events.emoji not in events.ipv4
     assert events.emoji not in events.netv6
     assert events.netv6 not in events.emoji
-    assert "dead::c0de" == scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    assert "dead::c0de" == ipv6_event
+    assert ipv6_event.host_filterable == "dead::c0de"
+    range_to_ip = scan.make_event("1.2.3.4/32", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
+    range_to_ip = scan.make_event("dead::beef/128", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
 
     # hostname tests
     assert events.domain.host == "publicapis.org"
+    assert events.domain.host_filterable == "publicapis.org"
     assert events.subdomain.host == "api.publicapis.org"
+    assert events.subdomain.host_filterable == "api.publicapis.org"
     assert events.domain.host_stem == "publicapis"
     assert events.subdomain.host_stem == "api.publicapis"
     assert "api.publicapis.org" in events.domain
@@ -62,23 +81,39 @@ async def test_events(events, helpers):
     assert "fsocie.ty" not in events.subdomain
     assert events.subdomain in events.domain
     assert events.domain not in events.subdomain
-    assert not events.ipv4 in events.domain
-    assert not events.netv6 in events.domain
+    assert events.ipv4 not in events.domain
+    assert events.netv6 not in events.domain
     assert events.emoji not in events.domain
     assert events.domain not in events.emoji
-    assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
-    assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
+    dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    for e in (open_port_event, dns_event):
+        assert "evilcorp.com" == e
+        assert e.netloc == "evilcorp.com"
+        assert e.json()["netloc"] == "evilcorp.com"
+        assert e.port is None
+        assert "port" not in e.json()
 
     # url tests
-    assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
+    url_no_trailing_slash = scan.make_event("http://evilcorp.com", dummy=True)
+    url_trailing_slash = scan.make_event("http://evilcorp.com/", dummy=True)
+    assert url_no_trailing_slash == url_trailing_slash
+    assert url_no_trailing_slash.host_filterable == "http://evilcorp.com/"
+    assert url_trailing_slash.host_filterable == "http://evilcorp.com/"
     assert events.url_unverified.host == "api.publicapis.org"
     assert events.url_unverified in events.domain
     assert events.url_unverified in events.subdomain
     assert "api.publicapis.org:443" in events.url_unverified
     assert "publicapis.org" not in events.url_unverified
     assert events.ipv4_url_unverified in events.ipv4
+    assert events.ipv4_url_unverified.netloc == "8.8.8.8:443"
+    assert events.ipv4_url_unverified.port == 443
+    assert events.ipv4_url_unverified.json()["port"] == 443
     assert events.ipv4_url_unverified in events.netv4
     assert events.ipv6_url_unverified in events.ipv6
+    assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443"
+    assert events.ipv6_url_unverified.port == 443
+    assert events.ipv6_url_unverified.json()["port"] == 443
     assert events.ipv6_url_unverified in events.netv6
     assert events.emoji not in events.url_unverified
     assert events.emoji not in events.ipv6_url_unverified
@@ -107,6 +142,7 @@ async def test_events(events, helpers):
     assert events.http_response.port == 80
     assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
+    assert events.http_response.host_filterable == "http://example.com/"
 
     http_response = scan.make_event(
         {
@@ -171,7 +207,7 @@ async def test_events(events, helpers):
 
     # scope distance
     event1 = scan.make_event("1.2.3.4", dummy=True)
-    assert event1._scope_distance == None
+    assert event1._scope_distance is None
     event1.scope_distance = 0
     assert event1._scope_distance == 0
     event2 = scan.make_event("2.3.4.5", parent=event1)
@@ -192,6 +228,8 @@ async def test_events(events, helpers):
 
     org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
+    assert org_stub_1.netloc is None
+    assert "netloc" not in org_stub_1.json()
     org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 
@@ -199,7 +237,7 @@ async def test_events(events, helpers):
     root_event = scan.make_event("0.0.0.0", dummy=True)
     root_event.scope_distance = 0
     internal_event1 = scan.make_event("1.2.3.4", parent=root_event, internal=True)
-    assert internal_event1._internal == True
+    assert internal_event1._internal is True
     assert "internal" in internal_event1.tags
 
     # tag inheritance
@@ -231,8 +269,8 @@ async def test_events(events, helpers):
     # updating module
     event3 = scan.make_event("127.0.0.1", parent=scan.root_event)
     updated_event = scan.make_event(event3, internal=True)
-    assert event3.internal == False
-    assert updated_event.internal == True
+    assert event3.internal is False
+    assert updated_event.internal is True
 
     # event sorting
     parent1 = scan.make_event("127.0.0.1", parent=scan.root_event)
@@ -412,17 +450,55 @@ async def test_events(events, helpers):
         == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ"
     )
 
+    # test event uuid
+    import uuid
+
+    parent_event1 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+    parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+
+    event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context")
+    assert hasattr(event1, "_uuid")
+    assert hasattr(event1, "uuid")
+    assert isinstance(event1._uuid, uuid.UUID)
+    assert isinstance(event1.uuid, str)
+    assert event1.uuid == f"{event1.type}:{event1._uuid}"
+    event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context")
+    assert hasattr(event2, "_uuid")
+    assert hasattr(event2, "uuid")
+    assert isinstance(event2._uuid, uuid.UUID)
+    assert isinstance(event2.uuid, str)
+    assert event2.uuid == f"{event2.type}:{event2._uuid}"
+    # ids should match because the event type + data is the same
+    assert event1.id == event2.id
+    # but uuids should be unique!
+    assert event1.uuid != event2.uuid
+    # parent ids should match
+    assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id
+    # uuids should not
+    assert event1.parent_uuid == parent_event1.uuid
+    assert event2.parent_uuid == parent_event2.uuid
+    assert event1.parent_uuid != event2.parent_uuid
+
     # test event serialization
     from bbot.core.event import event_from_json
 
     db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context")
+    assert db_event.parent == scan.root_event
+    assert db_event.parent is scan.root_event
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     assert db_event.discovery_context == "test context"
     assert db_event.discovery_path == ["test context"]
-    assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert len(db_event.parent_chain) == 1
+    assert all(event_uuid_regex.match(u) for u in db_event.parent_chain)
+    assert db_event.parent_chain[0] == str(db_event.uuid)
+    assert db_event.parent.uuid == scan.root_event.uuid
+    assert db_event.parent_uuid == scan.root_event.uuid
     timestamp = db_event.timestamp.isoformat()
     json_event = db_event.json()
+    assert isinstance(json_event["uuid"], str)
+    assert json_event["uuid"] == str(db_event.uuid)
+    assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"
     assert json_event["type"] == "OPEN_TCP_PORT"
@@ -430,8 +506,14 @@ async def test_events(events, helpers):
     assert json_event["timestamp"] == timestamp
     assert json_event["discovery_context"] == "test context"
     assert json_event["discovery_path"] == ["test context"]
-    assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert json_event["parent_chain"] == db_event.parent_chain
+    assert json_event["parent_chain"][0] == str(db_event.uuid)
     reconstituted_event = event_from_json(json_event)
+    assert isinstance(reconstituted_event._uuid, uuid.UUID)
+    assert str(reconstituted_event.uuid) == json_event["uuid"]
+    assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"]
+    assert reconstituted_event.uuid == db_event.uuid
+    assert reconstituted_event.parent_uuid == scan.root_event.uuid
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.isoformat() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
@@ -439,13 +521,13 @@ async def test_events(events, helpers):
     assert reconstituted_event.host == "evilcorp.com"
     assert reconstituted_event.discovery_context == "test context"
     assert reconstituted_event.discovery_path == ["test context"]
-    assert reconstituted_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert reconstituted_event.parent_chain == db_event.parent_chain
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
     assert hostless_event_json["type"] == "ASDF"
     assert hostless_event_json["data"] == "asdf"
-    assert not "host" in hostless_event_json
+    assert "host" not in hostless_event_json
 
     # SIEM-friendly serialize/deserialize
     json_event_siemfriendly = db_event.json(siem_friendly=True)
@@ -616,7 +698,7 @@ async def test_event_discovery_context():
     )
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 6
+    assert len(events) == 7
 
     assert 1 == len(
         [
@@ -723,7 +805,7 @@ async def test_event_web_spider_distance(bbot_scanner):
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     social_event = scan.make_event(
         {"platform": "github", "url": "http://www.evilcorp.com/test4"}, "SOCIAL", parent=url_event_3
     )
@@ -746,42 +828,42 @@ async def test_event_web_spider_distance(bbot_scanner):
 
     url_event = scan.make_event("http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event)
     assert url_event.web_spider_distance == 0
-    assert not "spider-danger" in url_event.tags
-    assert not "spider-max" in url_event.tags
+    assert "spider-danger" not in url_event.tags
+    assert "spider-max" not in url_event.tags
     url_event_2 = scan.make_event(
         "http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event, tags="spider-danger"
     )
     # spider distance shouldn't increment because it's not the same host
     assert url_event_2.web_spider_distance == 0
     assert "spider-danger" in url_event_2.tags
-    assert not "spider-max" in url_event_2.tags
+    assert "spider-max" not in url_event_2.tags
     url_event_3 = scan.make_event(
         "http://www.evilcorp.com/3", "URL_UNVERIFIED", parent=url_event_2, tags="spider-danger"
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     url_event_4 = scan.make_event("http://evilcorp.com", "URL_UNVERIFIED", parent=url_event_3)
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.add_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
     assert "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.remove_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_5 = scan.make_event("http://evilcorp.com/5", "URL_UNVERIFIED", parent=url_event_4)
     assert url_event_5.web_spider_distance == 0
-    assert not "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-danger" not in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
     url_event_5.add_tag("spider-danger")
     # if host is the same as parent, web spider distance should auto-increment after adding spider-danger tag
     assert url_event_5.web_spider_distance == 1
     assert "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
 
 
 def test_event_confidence():
@@ -856,3 +938,76 @@ def test_event_closest_host():
         vuln = scan.make_event(
             {"path": "/tmp/asdf.txt", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
         )
+
+
+def test_event_magic():
+    from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+    import base64
+
+    zip_base64 = "UEsDBAoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIAAAAYXNkZi50eHRhc2RmClBLAQI/AwoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIACQAAAAAAAAAIICkgQAAAABhc2RmLnR4dAoAIAAAAAAAAQAYAICi2B77MNsBgKLYHvsw2wGAotge+zDbAVBLBQYAAAAAAQABAFoAAAArAAAAAAA="
+    zip_bytes = base64.b64decode(zip_base64)
+    zip_file = Path("/tmp/.bbottestzipasdkfjalsdf.zip")
+    with open(zip_file, "wb") as f:
+        f.write(zip_bytes)
+
+    # test magic helpers
+    extension, mime_type, description, confidence = get_magic_info(zip_file)
+    assert extension == ".zip"
+    assert mime_type == "application/zip"
+    assert description == "PKZIP Archive file"
+    assert confidence > 0
+    assert get_compression(mime_type) == "zip"
+
+    # test filesystem event - file
+    scan = Scanner()
+    event = scan.make_event({"path": zip_file}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {
+        "path": "/tmp/.bbottestzipasdkfjalsdf.zip",
+        "magic_extension": ".zip",
+        "magic_mime_type": "application/zip",
+        "magic_description": "PKZIP Archive file",
+        "magic_confidence": 0.9,
+        "compression": "zip",
+    }
+    assert event.tags == {"file", "zip-archive", "compressed"}
+
+    # test filesystem event - folder
+    scan = Scanner()
+    event = scan.make_event({"path": "/tmp"}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {"path": "/tmp"}
+    assert event.tags == {"folder"}
+
+    zip_file.unlink()
+
+
+def test_event_hashing():
+    scan = Scanner("example.com")
+    url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
+    host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
+    host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
+    finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
+    finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
+    finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+    finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+
+    assert finding1.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "www.example.com",
+    }
+    assert finding2.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding3.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding1.id != finding2.id
+    assert finding2.id == finding3.id
+    assert finding1.data_id != finding2.data_id
+    assert finding2.data_id == finding3.data_id
+    assert finding1.data_hash != finding2.data_hash
+    assert finding2.data_hash == finding3.data_hash
+    assert hash(finding1) != hash(finding2)
+    assert hash(finding2) == hash(finding3)