bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/test/test_step_1/test_dns.py

@@ -23,7 +23,7 @@ async def test_dns_engine(bbot_scanner):
     )
     result = await scan.helpers.resolve("one.one.one.one")
     assert "1.1.1.1" in result
-    assert not "2606:4700:4700::1111" in result
+    assert "2606:4700:4700::1111" not in result
 
     results = [_ async for _ in scan.helpers.resolve_batch(("one.one.one.one", "1.1.1.1"))]
     pass_1 = False
@@ -85,12 +85,12 @@ async def test_dns_resolution(bbot_scanner):
     for answer in answers:
         responses += list(extract_targets(answer))
     assert ("A", "1.1.1.1") in responses
-    assert not ("AAAA", "2606:4700:4700::1111") in responses
+    assert ("AAAA", "2606:4700:4700::1111") not in responses
     answers, errors = await dnsengine.resolve_raw("one.one.one.one", type="AAAA")
     responses = []
     for answer in answers:
         responses += list(extract_targets(answer))
-    assert not ("A", "1.1.1.1") in responses
+    assert ("A", "1.1.1.1") not in responses
     assert ("AAAA", "2606:4700:4700::1111") in responses
     answers, errors = await dnsengine.resolve_raw("1.1.1.1")
     responses = []
@@ -106,13 +106,14 @@ async def test_dns_resolution(bbot_scanner):
     assert "2606:4700:4700::1111" in await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert "one.one.one.one" in await dnsengine.resolve("1.1.1.1")
     for rdtype in ("NS", "SOA", "MX", "TXT"):
-        assert len(await dnsengine.resolve("google.com", type=rdtype)) > 0
+        results = await dnsengine.resolve("google.com", type=rdtype)
+        assert len(results) > 0
 
     # batch resolution
     batch_results = [r async for r in dnsengine.resolve_batch(["1.1.1.1", "one.one.one.one"])]
     assert len(batch_results) == 2
     batch_results = dict(batch_results)
-    assert any([x in batch_results["one.one.one.one"] for x in ("1.1.1.1", "1.0.0.1")])
+    assert any(x in batch_results["one.one.one.one"] for x in ("1.1.1.1", "1.0.0.1"))
     assert "one.one.one.one" in batch_results["1.1.1.1"]
 
     # custom batch resolution
@@ -140,11 +141,11 @@ async def test_dns_resolution(bbot_scanner):
     assert hash(("1.1.1.1", "PTR")) in dnsengine._dns_cache
     await dnsengine.resolve("one.one.one.one", type="A")
     assert hash(("one.one.one.one", "A")) in dnsengine._dns_cache
-    assert not hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache
     dnsengine._dns_cache.clear()
     await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
-    assert not hash(("one.one.one.one", "A")) in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache
 
     await dnsengine._shutdown()
 
@@ -164,7 +165,7 @@ async def test_dns_resolution(bbot_scanner):
     assert "A" in resolved_hosts_event1.raw_dns_records
     assert "AAAA" in resolved_hosts_event1.raw_dns_records
     assert "a-record" in resolved_hosts_event1.tags
-    assert not "a-record" in resolved_hosts_event2.tags
+    assert "a-record" not in resolved_hosts_event2.tags
 
     scan2 = bbot_scanner("evilcorp.com", config={"dns": {"minimal": False}})
     await scan2.helpers.dns._mock_dns(
@@ -184,7 +185,6 @@ async def test_dns_resolution(bbot_scanner):
 
 @pytest.mark.asyncio
 async def test_wildcards(bbot_scanner):
-
     scan = bbot_scanner("1.1.1.1")
     helpers = scan.helpers
 
@@ -197,7 +197,7 @@ async def test_wildcards(bbot_scanner):
     assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2)
     for rdtype in all_rdtypes:
         assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache
-        if not rdtype in ("A", "AAAA"):
+        if rdtype not in ("A", "AAAA"):
             assert hash(("asdf.github.io", rdtype)) in dnsengine._wildcard_cache
     assert "github.io" in wildcard_domains
     assert "A" in wildcard_domains["github.io"]
@@ -263,7 +263,7 @@ def custom_lookup(query, rdtype):
     await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 11
+    assert len(events) == 12
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -320,7 +320,7 @@ def custom_lookup(query, rdtype):
     await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
 
     events = [e async for e in scan.async_start()]
-    assert len(events) == 11
+    assert len(events) == 12
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -418,7 +418,7 @@ def custom_lookup(query, rdtype):
 
     events = [e async for e in scan.async_start()]
 
-    assert len(events) == 10
+    assert len(events) == 11
     assert len([e for e in events if e.type == "DNS_NAME"]) == 5
     assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
     assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
@@ -546,8 +546,8 @@ def custom_lookup(query, rdtype):
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
-    assert len(events) == 3
-    assert 1 == len([e for e in events if e.type == "SCAN"])
+    assert len(events) == 4
+    assert 2 == len([e for e in events if e.type == "SCAN"])
     unmodified_wildcard_events = [
         e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io"
     ]
@@ -592,8 +592,8 @@ def custom_lookup(query, rdtype):
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
-    assert len(events) == 3
-    assert 1 == len([e for e in events if e.type == "SCAN"])
+    assert len(events) == 4
+    assert 2 == len([e for e in events if e.type == "SCAN"])
     unmodified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and "_wildcard" not in e.data]
     assert len(unmodified_wildcard_events) == 2
     assert 1 == len(
@@ -632,8 +632,42 @@ def custom_lookup(query, rdtype):
 
 
 @pytest.mark.asyncio
-async def test_dns_raw_records(bbot_scanner):
+async def test_wildcard_deduplication(bbot_scanner):
+    custom_lookup = """
+def custom_lookup(query, rdtype):
+    if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"):
+        return {""}
+"""
+
+    mock_data = {
+        "evilcorp.com": {"A": ["127.0.0.1"]},
+    }
+
+    from bbot.modules.base import BaseModule
+
+    class DummyModule(BaseModule):
+        watched_events = ["DNS_NAME"]
+        per_domain_only = True
+
+        async def handle_event(self, event):
+            for i in range(30):
+                await self.emit_event(f"www{i}.evilcorp.com", "DNS_NAME", parent=event)
+
+    # scan without omitted event type
+    scan = bbot_scanner(
+        "evilcorp.com", config={"dns": {"minimal": False, "wildcard_ignore": []}, "omit_event_types": []}
+    )
+    await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
+    dummy_module = DummyModule(scan)
+    scan.modules["dummy_module"] = dummy_module
+    events = [e async for e in scan.async_start()]
+    dns_name_events = [e for e in events if e.type == "DNS_NAME"]
+    assert len(dns_name_events) == 2
+    assert 1 == len([e for e in dns_name_events if e.data == "_wildcard.evilcorp.com"])
+
 
+@pytest.mark.asyncio
+async def test_dns_raw_records(bbot_scanner):
     from bbot.modules.base import BaseModule
 
     class DummyModule(BaseModule):
@@ -696,7 +730,7 @@ async def test_dns_raw_records(bbot_scanner):
     dummy_module = DummyModule(scan)
     scan.modules["dummy_module"] = dummy_module
     events = [e async for e in scan.async_start()]
-    # no raw records should be ouptut
+    # no raw records should be output
     assert 0 == len([e for e in events if e.type == "RAW_DNS_RECORD"])
     # but they should still make it to the module
     assert 1 == len(
@@ -729,7 +763,7 @@ async def test_dns_graph_structure(bbot_scanner):
         }
     )
     events = [e async for e in scan.async_start()]
-    assert len(events) == 5
+    assert len(events) == 6
     non_scan_events = [e for e in events if e.type != "SCAN"]
     assert sorted([e.type for e in non_scan_events]) == ["DNS_NAME", "DNS_NAME", "DNS_NAME", "URL_UNVERIFIED"]
     events_by_data = {e.data: e for e in non_scan_events}
@@ -744,16 +778,16 @@ async def test_dns_graph_structure(bbot_scanner):
 
 @pytest.mark.asyncio
 async def test_dns_helpers(bbot_scanner):
-    assert service_record("") == False
-    assert service_record("localhost") == False
-    assert service_record("www.example.com") == False
-    assert service_record("www.example.com", "SRV") == True
-    assert service_record("_custom._service.example.com", "SRV") == True
-    assert service_record("_custom._service.example.com", "A") == False
+    assert service_record("") is False
+    assert service_record("localhost") is False
+    assert service_record("www.example.com") is False
+    assert service_record("www.example.com", "SRV") is True
+    assert service_record("_custom._service.example.com", "SRV") is True
+    assert service_record("_custom._service.example.com", "A") is False
     # top 100 most common SRV records
     for srv_record in common_srvs[:100]:
         hostname = f"{srv_record}.example.com"
-        assert service_record(hostname) == True
+        assert service_record(hostname) is True
 
     # make sure system nameservers are excluded from use by DNS brute force
     brute_nameservers = tempwordlist(["1.2.3.4", "8.8.4.4", "4.3.2.1", "8.8.8.8"])

bbot/test/test_step_1/test_engine.py

@@ -14,7 +14,6 @@ async def test_engine():
     return_errored = False
 
     class TestEngineServer(EngineServer):
-
         CMDS = {
             0: "return_thing",
             1: "yield_stuff",
@@ -54,7 +53,6 @@ async def test_engine():
                 raise
 
     class TestEngineClient(EngineClient):
-
         SERVER_CLASS = TestEngineServer
 
         async def return_thing(self, n):
@@ -72,7 +70,7 @@ async def test_engine():
 
     # test async generator
     assert counter == 0
-    assert yield_cancelled == False
+    assert yield_cancelled is False
     yield_res = [r async for r in test_engine.yield_stuff(13)]
     assert yield_res == [f"thing{i}" for i in range(13)]
     assert len(yield_res) == 13
@@ -88,8 +86,8 @@ async def test_engine():
             await agen.aclose()
             break
     await asyncio.sleep(5)
-    assert yield_cancelled == True
-    assert yield_errored == False
+    assert yield_cancelled is True
+    assert yield_errored is False
     assert counter < 15
 
     # test async generator with error
@@ -99,8 +97,8 @@ async def test_engine():
     with pytest.raises(BBOTEngineError):
         async for _ in agen:
             pass
-    assert yield_cancelled == False
-    assert yield_errored == True
+    assert yield_cancelled is False
+    assert yield_errored is True
 
     # test return with cancellation
     return_started = False
@@ -113,10 +111,10 @@ async def test_engine():
     with pytest.raises(asyncio.CancelledError):
         await task
     await asyncio.sleep(0.1)
-    assert return_started == True
-    assert return_finished == False
-    assert return_cancelled == True
-    assert return_errored == False
+    assert return_started is True
+    assert return_finished is False
+    assert return_cancelled is True
+    assert return_errored is False
 
     # test return with late cancellation
     return_started = False
@@ -128,10 +126,10 @@ async def test_engine():
     task.cancel()
     result = await task
     assert result == "thing1"
-    assert return_started == True
-    assert return_finished == True
-    assert return_cancelled == False
-    assert return_errored == False
+    assert return_started is True
+    assert return_finished is True
+    assert return_cancelled is False
+    assert return_errored is False
 
     # test return with error
     return_started = False
@@ -140,9 +138,9 @@ async def test_engine():
     return_errored = False
     with pytest.raises(BBOTEngineError):
         result = await test_engine.return_thing(None)
-    assert return_started == True
-    assert return_finished == False
-    assert return_cancelled == False
-    assert return_errored == True
+    assert return_started is True
+    assert return_finished is False
+    assert return_cancelled is False
+    assert return_errored is True
 
     await test_engine.shutdown()