bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/test/test_step_1/test_events.py

@@ -4,6 +4,7 @@ import ipaddress
 
 from ..bbot_fixtures import *
 from bbot.scanner import Scanner
+from bbot.core.helpers.regexes import event_uuid_regex
 
 
 @pytest.mark.asyncio
@@ -13,10 +14,19 @@ async def test_events(events, helpers):
     await scan._prep()
 
     assert events.ipv4.type == "IP_ADDRESS"
+    assert events.ipv4.netloc == "8.8.8.8"
+    assert events.ipv4.port is None
     assert events.ipv6.type == "IP_ADDRESS"
+    assert events.ipv6.netloc == "[2001:4860:4860::8888]"
+    assert events.ipv6.port is None
+    assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443"
     assert events.netv4.type == "IP_RANGE"
+    assert events.netv4.netloc is None
+    assert "netloc" not in events.netv4.json()
     assert events.netv6.type == "IP_RANGE"
     assert events.domain.type == "DNS_NAME"
+    assert events.domain.netloc == "publicapis.org"
+    assert events.domain.port is None
     assert "domain" in events.domain.tags
     assert events.subdomain.type == "DNS_NAME"
     assert "subdomain" in events.subdomain.tags
@@ -32,6 +42,7 @@ async def test_events(events, helpers):
     # ip tests
     assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True)
     assert "8.8.8.8" in events.ipv4
+    assert events.ipv4.host_filterable == "8.8.8.8"
     assert "8.8.8.8" == events.ipv4
     assert "8.8.8.8" in events.netv4
     assert "8.8.8.9" not in events.ipv4
@@ -49,11 +60,19 @@ async def test_events(events, helpers):
     assert events.emoji not in events.ipv4
     assert events.emoji not in events.netv6
     assert events.netv6 not in events.emoji
-    assert "dead::c0de" == scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    assert "dead::c0de" == ipv6_event
+    assert ipv6_event.host_filterable == "dead::c0de"
+    range_to_ip = scan.make_event("1.2.3.4/32", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
+    range_to_ip = scan.make_event("dead::beef/128", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
 
     # hostname tests
     assert events.domain.host == "publicapis.org"
+    assert events.domain.host_filterable == "publicapis.org"
     assert events.subdomain.host == "api.publicapis.org"
+    assert events.subdomain.host_filterable == "api.publicapis.org"
     assert events.domain.host_stem == "publicapis"
     assert events.subdomain.host_stem == "api.publicapis"
     assert "api.publicapis.org" in events.domain
@@ -62,23 +81,39 @@ async def test_events(events, helpers):
     assert "fsocie.ty" not in events.subdomain
     assert events.subdomain in events.domain
     assert events.domain not in events.subdomain
-    assert not events.ipv4 in events.domain
-    assert not events.netv6 in events.domain
+    assert events.ipv4 not in events.domain
+    assert events.netv6 not in events.domain
     assert events.emoji not in events.domain
     assert events.domain not in events.emoji
-    assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
-    assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
+    dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    for e in (open_port_event, dns_event):
+        assert "evilcorp.com" == e
+        assert e.netloc == "evilcorp.com"
+        assert e.json()["netloc"] == "evilcorp.com"
+        assert e.port is None
+        assert "port" not in e.json()
 
     # url tests
-    assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
+    url_no_trailing_slash = scan.make_event("http://evilcorp.com", dummy=True)
+    url_trailing_slash = scan.make_event("http://evilcorp.com/", dummy=True)
+    assert url_no_trailing_slash == url_trailing_slash
+    assert url_no_trailing_slash.host_filterable == "http://evilcorp.com/"
+    assert url_trailing_slash.host_filterable == "http://evilcorp.com/"
     assert events.url_unverified.host == "api.publicapis.org"
     assert events.url_unverified in events.domain
     assert events.url_unverified in events.subdomain
     assert "api.publicapis.org:443" in events.url_unverified
     assert "publicapis.org" not in events.url_unverified
     assert events.ipv4_url_unverified in events.ipv4
+    assert events.ipv4_url_unverified.netloc == "8.8.8.8:443"
+    assert events.ipv4_url_unverified.port == 443
+    assert events.ipv4_url_unverified.json()["port"] == 443
     assert events.ipv4_url_unverified in events.netv4
     assert events.ipv6_url_unverified in events.ipv6
+    assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443"
+    assert events.ipv6_url_unverified.port == 443
+    assert events.ipv6_url_unverified.json()["port"] == 443
     assert events.ipv6_url_unverified in events.netv6
     assert events.emoji not in events.url_unverified
     assert events.emoji not in events.ipv6_url_unverified
@@ -107,6 +142,7 @@ async def test_events(events, helpers):
     assert events.http_response.port == 80
     assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
+    assert events.http_response.host_filterable == "http://example.com/"
 
     http_response = scan.make_event(
         {
@@ -171,7 +207,7 @@ async def test_events(events, helpers):
 
     # scope distance
     event1 = scan.make_event("1.2.3.4", dummy=True)
-    assert event1._scope_distance == None
+    assert event1._scope_distance is None
     event1.scope_distance = 0
     assert event1._scope_distance == 0
     event2 = scan.make_event("2.3.4.5", parent=event1)
@@ -192,6 +228,8 @@ async def test_events(events, helpers):
 
     org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
+    assert org_stub_1.netloc is None
+    assert "netloc" not in org_stub_1.json()
     org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 
@@ -199,7 +237,7 @@ async def test_events(events, helpers):
     root_event = scan.make_event("0.0.0.0", dummy=True)
     root_event.scope_distance = 0
     internal_event1 = scan.make_event("1.2.3.4", parent=root_event, internal=True)
-    assert internal_event1._internal == True
+    assert internal_event1._internal is True
     assert "internal" in internal_event1.tags
 
     # tag inheritance
@@ -231,8 +269,8 @@ async def test_events(events, helpers):
     # updating module
     event3 = scan.make_event("127.0.0.1", parent=scan.root_event)
     updated_event = scan.make_event(event3, internal=True)
-    assert event3.internal == False
-    assert updated_event.internal == True
+    assert event3.internal is False
+    assert updated_event.internal is True
 
     # event sorting
     parent1 = scan.make_event("127.0.0.1", parent=scan.root_event)
@@ -311,6 +349,16 @@ async def test_events(events, helpers):
             {"host": "evilcorp.com", "severity": "WACK", "description": "asdf"}, "VULNERABILITY", dummy=True
         )
 
+    # test tagging
+    ip_event_1 = scan.make_event("8.8.8.8", dummy=True)
+    assert "private-ip" not in ip_event_1.tags
+    ip_event_2 = scan.make_event("192.168.0.1", dummy=True)
+    assert "private-ip" in ip_event_2.tags
+    dns_event_1 = scan.make_event("evilcorp.com", dummy=True)
+    assert "domain" in dns_event_1.tags
+    dns_event_2 = scan.make_event("www.evilcorp.com", dummy=True)
+    assert "subdomain" in dns_event_2.tags
+
     # punycode - event type detection
 
     # japanese
@@ -402,39 +450,84 @@ async def test_events(events, helpers):
         == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ"
     )
 
+    # test event uuid
+    import uuid
+
+    parent_event1 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+    parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+
+    event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context")
+    assert hasattr(event1, "_uuid")
+    assert hasattr(event1, "uuid")
+    assert isinstance(event1._uuid, uuid.UUID)
+    assert isinstance(event1.uuid, str)
+    assert event1.uuid == f"{event1.type}:{event1._uuid}"
+    event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context")
+    assert hasattr(event2, "_uuid")
+    assert hasattr(event2, "uuid")
+    assert isinstance(event2._uuid, uuid.UUID)
+    assert isinstance(event2.uuid, str)
+    assert event2.uuid == f"{event2.type}:{event2._uuid}"
+    # ids should match because the event type + data is the same
+    assert event1.id == event2.id
+    # but uuids should be unique!
+    assert event1.uuid != event2.uuid
+    # parent ids should match
+    assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id
+    # uuids should not
+    assert event1.parent_uuid == parent_event1.uuid
+    assert event2.parent_uuid == parent_event2.uuid
+    assert event1.parent_uuid != event2.parent_uuid
+
     # test event serialization
     from bbot.core.event import event_from_json
 
     db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context")
+    assert db_event.parent == scan.root_event
+    assert db_event.parent is scan.root_event
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     assert db_event.discovery_context == "test context"
-    assert db_event.discovery_path == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]]
+    assert db_event.discovery_path == ["test context"]
+    assert len(db_event.parent_chain) == 1
+    assert all(event_uuid_regex.match(u) for u in db_event.parent_chain)
+    assert db_event.parent_chain[0] == str(db_event.uuid)
+    assert db_event.parent.uuid == scan.root_event.uuid
+    assert db_event.parent_uuid == scan.root_event.uuid
     timestamp = db_event.timestamp.isoformat()
     json_event = db_event.json()
+    assert isinstance(json_event["uuid"], str)
+    assert json_event["uuid"] == str(db_event.uuid)
+    assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"
     assert json_event["type"] == "OPEN_TCP_PORT"
     assert json_event["host"] == "evilcorp.com"
     assert json_event["timestamp"] == timestamp
     assert json_event["discovery_context"] == "test context"
-    assert json_event["discovery_path"] == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]]
+    assert json_event["discovery_path"] == ["test context"]
+    assert json_event["parent_chain"] == db_event.parent_chain
+    assert json_event["parent_chain"][0] == str(db_event.uuid)
     reconstituted_event = event_from_json(json_event)
+    assert isinstance(reconstituted_event._uuid, uuid.UUID)
+    assert str(reconstituted_event.uuid) == json_event["uuid"]
+    assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"]
+    assert reconstituted_event.uuid == db_event.uuid
+    assert reconstituted_event.parent_uuid == scan.root_event.uuid
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.isoformat() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
     assert reconstituted_event.type == "OPEN_TCP_PORT"
     assert reconstituted_event.host == "evilcorp.com"
     assert reconstituted_event.discovery_context == "test context"
-    assert reconstituted_event.discovery_path == [
-        ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]
-    ]
+    assert reconstituted_event.discovery_path == ["test context"]
+    assert reconstituted_event.parent_chain == db_event.parent_chain
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
     assert hostless_event_json["type"] == "ASDF"
     assert hostless_event_json["data"] == "asdf"
-    assert not "host" in hostless_event_json
+    assert "host" not in hostless_event_json
 
     # SIEM-friendly serialize/deserialize
     json_event_siemfriendly = db_event.json(siem_friendly=True)
@@ -605,7 +698,7 @@ async def test_event_discovery_context():
     )
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 6
+    assert len(events) == 7
 
     assert 1 == len(
         [
@@ -614,7 +707,7 @@ async def test_event_discovery_context():
             if e.type == "DNS_NAME"
             and e.data == "evilcorp.com"
             and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"
-            and [_[-1] for _ in e.discovery_path] == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
+            and e.discovery_path == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
         ]
     )
     assert 1 == len(
@@ -624,7 +717,7 @@ async def test_event_discovery_context():
             if e.type == "DNS_NAME"
             and e.data == "one.evilcorp.com"
             and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com"
-            and [_[-1] for _ in e.discovery_path]
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -639,7 +732,7 @@ async def test_event_discovery_context():
             and e.data == "two.evilcorp.com"
             and e.discovery_context
             == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com"
-            and [_[-1] for _ in e.discovery_path]
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -654,7 +747,7 @@ async def test_event_discovery_context():
             if e.type == "DNS_NAME"
             and e.data == "three.evilcorp.com"
             and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com"
-            and [_[-1] for _ in e.discovery_path]
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -676,11 +769,11 @@ async def test_event_discovery_context():
         if e.type == "DNS_NAME"
         and e.data == "four.evilcorp.com"
         and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com"
-        and [_[-1] for _ in e.discovery_path] == final_path
+        and e.discovery_path == final_path
     ]
     assert 1 == len(final_event)
     j = final_event[0].json()
-    assert [_[-1] for _ in j["discovery_path"]] == final_path
+    assert j["discovery_path"] == final_path
 
     await scan._cleanup()
 
@@ -693,7 +786,7 @@ async def test_event_discovery_context():
     events = [e async for e in scan.async_start()]
     blsops_event = [e for e in events if e.type == "DNS_NAME" and e.data == "blsops.com"]
     assert len(blsops_event) == 1
-    assert blsops_event[0].discovery_path[1][-1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com"
+    assert blsops_event[0].discovery_path[1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com"
 
     await scan._cleanup()
 
@@ -712,7 +805,7 @@ async def test_event_web_spider_distance(bbot_scanner):
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     social_event = scan.make_event(
         {"platform": "github", "url": "http://www.evilcorp.com/test4"}, "SOCIAL", parent=url_event_3
     )
@@ -735,42 +828,42 @@ async def test_event_web_spider_distance(bbot_scanner):
 
     url_event = scan.make_event("http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event)
     assert url_event.web_spider_distance == 0
-    assert not "spider-danger" in url_event.tags
-    assert not "spider-max" in url_event.tags
+    assert "spider-danger" not in url_event.tags
+    assert "spider-max" not in url_event.tags
     url_event_2 = scan.make_event(
         "http://www.evilcorp.com", "URL_UNVERIFIED", parent=scan.root_event, tags="spider-danger"
     )
     # spider distance shouldn't increment because it's not the same host
     assert url_event_2.web_spider_distance == 0
     assert "spider-danger" in url_event_2.tags
-    assert not "spider-max" in url_event_2.tags
+    assert "spider-max" not in url_event_2.tags
     url_event_3 = scan.make_event(
         "http://www.evilcorp.com/3", "URL_UNVERIFIED", parent=url_event_2, tags="spider-danger"
     )
     assert url_event_3.web_spider_distance == 1
     assert "spider-danger" in url_event_3.tags
-    assert not "spider-max" in url_event_3.tags
+    assert "spider-max" not in url_event_3.tags
     url_event_4 = scan.make_event("http://evilcorp.com", "URL_UNVERIFIED", parent=url_event_3)
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.add_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
     assert "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_4.remove_tag("spider-danger")
     assert url_event_4.web_spider_distance == 0
-    assert not "spider-danger" in url_event_4.tags
-    assert not "spider-max" in url_event_4.tags
+    assert "spider-danger" not in url_event_4.tags
+    assert "spider-max" not in url_event_4.tags
     url_event_5 = scan.make_event("http://evilcorp.com/5", "URL_UNVERIFIED", parent=url_event_4)
     assert url_event_5.web_spider_distance == 0
-    assert not "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-danger" not in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
     url_event_5.add_tag("spider-danger")
     # if host is the same as parent, web spider distance should auto-increment after adding spider-danger tag
     assert url_event_5.web_spider_distance == 1
     assert "spider-danger" in url_event_5.tags
-    assert not "spider-max" in url_event_5.tags
+    assert "spider-max" not in url_event_5.tags
 
 
 def test_event_confidence():
@@ -845,3 +938,76 @@ def test_event_closest_host():
         vuln = scan.make_event(
             {"path": "/tmp/asdf.txt", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
         )
+
+
+def test_event_magic():
+    from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+    import base64
+
+    zip_base64 = "UEsDBAoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIAAAAYXNkZi50eHRhc2RmClBLAQI/AwoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIACQAAAAAAAAAIICkgQAAAABhc2RmLnR4dAoAIAAAAAAAAQAYAICi2B77MNsBgKLYHvsw2wGAotge+zDbAVBLBQYAAAAAAQABAFoAAAArAAAAAAA="
+    zip_bytes = base64.b64decode(zip_base64)
+    zip_file = Path("/tmp/.bbottestzipasdkfjalsdf.zip")
+    with open(zip_file, "wb") as f:
+        f.write(zip_bytes)
+
+    # test magic helpers
+    extension, mime_type, description, confidence = get_magic_info(zip_file)
+    assert extension == ".zip"
+    assert mime_type == "application/zip"
+    assert description == "PKZIP Archive file"
+    assert confidence > 0
+    assert get_compression(mime_type) == "zip"
+
+    # test filesystem event - file
+    scan = Scanner()
+    event = scan.make_event({"path": zip_file}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {
+        "path": "/tmp/.bbottestzipasdkfjalsdf.zip",
+        "magic_extension": ".zip",
+        "magic_mime_type": "application/zip",
+        "magic_description": "PKZIP Archive file",
+        "magic_confidence": 0.9,
+        "compression": "zip",
+    }
+    assert event.tags == {"file", "zip-archive", "compressed"}
+
+    # test filesystem event - folder
+    scan = Scanner()
+    event = scan.make_event({"path": "/tmp"}, "FILESYSTEM", parent=scan.root_event)
+    assert event.data == {"path": "/tmp"}
+    assert event.tags == {"folder"}
+
+    zip_file.unlink()
+
+
+def test_event_hashing():
+    scan = Scanner("example.com")
+    url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event)
+    host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event)
+    host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event)
+    finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"}
+    finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1)
+    finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+    finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2)
+
+    assert finding1.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "www.example.com",
+    }
+    assert finding2.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding3.data == {
+        "description": "Custom Yara Rule [find_string] Matched via identifier [str1]",
+        "host": "test.example.com",
+    }
+    assert finding1.id != finding2.id
+    assert finding2.id == finding3.id
+    assert finding1.data_id != finding2.data_id
+    assert finding2.data_id == finding3.data_id
+    assert finding1.data_hash != finding2.data_hash
+    assert finding2.data_hash == finding3.data_hash
+    assert hash(finding1) != hash(finding2)
+    assert hash(finding2) == hash(finding3)