bbot 2.2.0.5263rc0__py3-none-any.whl → 2.2.0.5309rc0__py3-none-any.whl

bbot/test/bbot_fixtures.py

@@ -15,8 +15,8 @@ from werkzeug.wrappers import Request
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
 log = logging.getLogger(f"bbot.test.fixtures")
@@ -229,4 +229,7 @@ def install_all_python_deps():
     deps_pip = set()
     for module in DEFAULT_PRESET.module_loader.preloaded().values():
         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+
+    constraint_file = tempwordlist(get_python_constraints())
+
+    subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))

bbot/test/conftest.py

@@ -94,9 +94,21 @@ def bbot_httpserver_ssl():
     server.clear()
 
 
-@pytest.fixture
-def non_mocked_hosts() -> list:
-    return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+def should_mock(request):
+    return not request.url.host in ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+
+
+def pytest_collection_modifyitems(config, items):
+    # make sure all tests have the httpx_mock marker
+    for item in items:
+        item.add_marker(
+            pytest.mark.httpx_mock(
+                should_mock=should_mock,
+                assert_all_requests_were_expected=False,
+                assert_all_responses_were_requested=False,
+                can_send_already_matched_responses=True,
+            )
+        )
 
 
 @pytest.fixture
@@ -239,80 +251,80 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):  # pragma: no
 
 
 # BELOW: debugging for frozen/hung tests
-# import psutil
-# import traceback
-# import inspect
-
-
-# def _print_detailed_info():  # pragma: no cover
-#     """
-#     Debugging pytests hanging
-#     """
-#     print("=== Detailed Thread and Process Information ===\n")
-#     try:
-#         print("=== Threads ===")
-#         for thread in threading.enumerate():
-#             print(f"Thread Name: {thread.name}")
-#             print(f"Thread ID: {thread.ident}")
-#             print(f"Is Alive: {thread.is_alive()}")
-#             print(f"Daemon: {thread.daemon}")
-
-#             if hasattr(thread, "_target"):
-#                 target = thread._target
-#                 if target:
-#                     qualname = (
-#                         f"{target.__module__}.{target.__qualname__}"
-#                         if hasattr(target, "__qualname__")
-#                         else str(target)
-#                     )
-#                     print(f"Target Function: {qualname}")
-
-#                     if hasattr(thread, "_args"):
-#                         args = thread._args
-#                         kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
-#                         arg_spec = inspect.getfullargspec(target)
-
-#                         all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
-
-#                         if inspect.ismethod(target) and arg_spec.args[0] == "self":
-#                             arg_spec.args.pop(0)
-
-#                         named_args = list(zip(arg_spec.args, all_args))
-#                         if arg_spec.varargs:
-#                             named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
-
-#                         print("Arguments:")
-#                         for name, value in named_args:
-#                             print(f"  {name}: {value}")
-#                 else:
-#                     print("Target Function: None")
-#             else:
-#                 print("Target Function: Unknown")
-
-#             print()
-
-#         print("=== Processes ===")
-#         current_process = psutil.Process()
-#         for child in current_process.children(recursive=True):
-#             print(f"Process ID: {child.pid}")
-#             print(f"Name: {child.name()}")
-#             print(f"Status: {child.status()}")
-#             print(f"CPU Times: {child.cpu_times()}")
-#             print(f"Memory Info: {child.memory_info()}")
-#             print()
-
-#         print("=== Current Process ===")
-#         print(f"Process ID: {current_process.pid}")
-#         print(f"Name: {current_process.name()}")
-#         print(f"Status: {current_process.status()}")
-#         print(f"CPU Times: {current_process.cpu_times()}")
-#         print(f"Memory Info: {current_process.memory_info()}")
-#         print()
-
-#     except Exception as e:
-#         print(f"An error occurred: {str(e)}")
-#         print("Traceback:")
-#         traceback.print_exc()
+import psutil
+import traceback
+import inspect
+
+
+def _print_detailed_info():  # pragma: no cover
+    """
+    Debugging pytests hanging
+    """
+    print("=== Detailed Thread and Process Information ===\n")
+    try:
+        print("=== Threads ===")
+        for thread in threading.enumerate():
+            print(f"Thread Name: {thread.name}")
+            print(f"Thread ID: {thread.ident}")
+            print(f"Is Alive: {thread.is_alive()}")
+            print(f"Daemon: {thread.daemon}")
+
+            if hasattr(thread, "_target"):
+                target = thread._target
+                if target:
+                    qualname = (
+                        f"{target.__module__}.{target.__qualname__}"
+                        if hasattr(target, "__qualname__")
+                        else str(target)
+                    )
+                    print(f"Target Function: {qualname}")
+
+                    if hasattr(thread, "_args"):
+                        args = thread._args
+                        kwargs = thread._kwargs if hasattr(thread, "_kwargs") else {}
+                        arg_spec = inspect.getfullargspec(target)
+
+                        all_args = list(args) + [f"{k}={v}" for k, v in kwargs.items()]
+
+                        if inspect.ismethod(target) and arg_spec.args[0] == "self":
+                            arg_spec.args.pop(0)
+
+                        named_args = list(zip(arg_spec.args, all_args))
+                        if arg_spec.varargs:
+                            named_args.extend((f"*{arg_spec.varargs}", arg) for arg in all_args[len(arg_spec.args) :])
+
+                        print("Arguments:")
+                        for name, value in named_args:
+                            print(f"  {name}: {value}")
+                else:
+                    print("Target Function: None")
+            else:
+                print("Target Function: Unknown")
+
+            print()
+
+        print("=== Processes ===")
+        current_process = psutil.Process()
+        for child in current_process.children(recursive=True):
+            print(f"Process ID: {child.pid}")
+            print(f"Name: {child.name()}")
+            print(f"Status: {child.status()}")
+            print(f"CPU Times: {child.cpu_times()}")
+            print(f"Memory Info: {child.memory_info()}")
+            print()
+
+        print("=== Current Process ===")
+        print(f"Process ID: {current_process.pid}")
+        print(f"Name: {current_process.name()}")
+        print(f"Status: {current_process.status()}")
+        print(f"CPU Times: {current_process.cpu_times()}")
+        print(f"Memory Info: {current_process.memory_info()}")
+        print()
+
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+        print("Traceback:")
+        traceback.print_exc()
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -330,11 +342,11 @@ def pytest_sessionfinish(session, exitstatus):
     yield
 
     # temporarily suspend stdout capture and print detailed thread info
-    # capmanager = session.config.pluginmanager.get_plugin("capturemanager")
-    # if capmanager:
-    #     capmanager.suspend_global_capture(in_=True)
+    capmanager = session.config.pluginmanager.get_plugin("capturemanager")
+    if capmanager:
+        capmanager.suspend_global_capture(in_=True)
 
-    # _print_detailed_info()
+    _print_detailed_info()
 
-    # if capmanager:
-    #     capmanager.resume_global_capture()
+    if capmanager:
+        capmanager.resume_global_capture()

bbot/test/test_step_1/test_bloom_filter.py

@@ -66,4 +66,6 @@ async def test_bloom_filter():
     # ensure false positives are less than .02 percent
     assert false_positive_percent < 0.02
 
+    bloom_filter.close()
+
     await scan._cleanup()

bbot/test/test_step_1/test_cli.py

@@ -535,6 +535,13 @@ def test_cli_module_validation(monkeypatch, caplog):
         ]
     )
 
+    # bad target
+    caplog.clear()
+    assert not caplog.text
+    monkeypatch.setattr("sys.argv", ["bbot", "-t", "asdf:::sdf"])
+    cli.main()
+    assert 'Unable to autodetect event type from "asdf:::sdf"' in caplog.text
+
     # incorrect flag
     caplog.clear()
     assert not caplog.text
@@ -626,6 +633,35 @@ config:
     stdout_preset = yaml.safe_load(captured.out)
     assert stdout_preset["config"]["web"]["http_proxy"] == "http://proxy2"
 
+    # --fast-mode
+    monkeypatch.setattr("sys.argv", ["bbot", "--current-preset"])
+    cli.main()
+    captured = capsys.readouterr()
+    stdout_preset = yaml.safe_load(captured.out)
+    assert list(stdout_preset) == ["description"]
+
+    monkeypatch.setattr("sys.argv", ["bbot", "--fast", "--current-preset"])
+    cli.main()
+    captured = capsys.readouterr()
+    stdout_preset = yaml.safe_load(captured.out)
+    stdout_preset.pop("description")
+    assert stdout_preset == {
+        "config": {
+            "scope": {"strict": True},
+            "dns": {"minimal": True},
+            "modules": {"speculate": {"essential_only": True}},
+        },
+        "exclude_modules": ["excavate"],
+    }
+
+    # --proxy
+    monkeypatch.setattr("sys.argv", ["bbot", "--proxy", "http://127.0.0.1:8080", "--current-preset"])
+    cli.main()
+    captured = capsys.readouterr()
+    stdout_preset = yaml.safe_load(captured.out)
+    stdout_preset.pop("description")
+    assert stdout_preset == {"config": {"web": {"http_proxy": "http://127.0.0.1:8080"}}}
+
     # cli config overrides all presets
     monkeypatch.setattr(
         "sys.argv",

bbot/test/test_step_1/test_dns.py

@@ -106,7 +106,8 @@ async def test_dns_resolution(bbot_scanner):
     assert "2606:4700:4700::1111" in await dnsengine.resolve("one.one.one.one", type="AAAA")
     assert "one.one.one.one" in await dnsengine.resolve("1.1.1.1")
     for rdtype in ("NS", "SOA", "MX", "TXT"):
-        assert len(await dnsengine.resolve("google.com", type=rdtype)) > 0
+        results = await dnsengine.resolve("google.com", type=rdtype)
+        assert len(results) > 0
 
     # batch resolution
     batch_results = [r async for r in dnsengine.resolve_batch(["1.1.1.1", "one.one.one.one"])]

bbot/test/test_step_1/test_events.py

@@ -42,6 +42,7 @@ async def test_events(events, helpers):
     # ip tests
     assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True)
     assert "8.8.8.8" in events.ipv4
+    assert events.ipv4.host_filterable == "8.8.8.8"
     assert "8.8.8.8" == events.ipv4
     assert "8.8.8.8" in events.netv4
     assert "8.8.8.9" not in events.ipv4
@@ -59,11 +60,19 @@ async def test_events(events, helpers):
     assert events.emoji not in events.ipv4
     assert events.emoji not in events.netv6
     assert events.netv6 not in events.emoji
-    assert "dead::c0de" == scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True)
+    assert "dead::c0de" == ipv6_event
+    assert ipv6_event.host_filterable == "dead::c0de"
+    range_to_ip = scan.make_event("1.2.3.4/32", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
+    range_to_ip = scan.make_event("dead::beef/128", dummy=True)
+    assert range_to_ip.type == "IP_ADDRESS"
 
     # hostname tests
     assert events.domain.host == "publicapis.org"
+    assert events.domain.host_filterable == "publicapis.org"
     assert events.subdomain.host == "api.publicapis.org"
+    assert events.subdomain.host_filterable == "api.publicapis.org"
     assert events.domain.host_stem == "publicapis"
     assert events.subdomain.host_stem == "api.publicapis"
     assert "api.publicapis.org" in events.domain
@@ -86,7 +95,11 @@ async def test_events(events, helpers):
         assert "port" not in e.json()
 
     # url tests
-    assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
+    url_no_trailing_slash = scan.make_event("http://evilcorp.com", dummy=True)
+    url_trailing_slash = scan.make_event("http://evilcorp.com/", dummy=True)
+    assert url_no_trailing_slash == url_trailing_slash
+    assert url_no_trailing_slash.host_filterable == "http://evilcorp.com/"
+    assert url_trailing_slash.host_filterable == "http://evilcorp.com/"
     assert events.url_unverified.host == "api.publicapis.org"
     assert events.url_unverified in events.domain
     assert events.url_unverified in events.subdomain
@@ -129,6 +142,7 @@ async def test_events(events, helpers):
     assert events.http_response.port == 80
     assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
+    assert events.http_response.host_filterable == "http://example.com/"
 
     http_response = scan.make_event(
         {
@@ -484,7 +498,6 @@ async def test_events(events, helpers):
     json_event = db_event.json()
     assert isinstance(json_event["uuid"], str)
     assert json_event["uuid"] == str(db_event.uuid)
-    print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}")
     assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"

bbot/test/test_step_1/test_helpers.py

@@ -93,8 +93,23 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         ipaddress.ip_network("0.0.0.0/0"),
     ]
     assert helpers.is_ip("127.0.0.1")
+    assert helpers.is_ip("127.0.0.1", include_network=True)
+    assert helpers.is_ip("127.0.0.1", version=4)
+    assert not helpers.is_ip("127.0.0.1", version=6)
     assert not helpers.is_ip("127.0.0.0.1")
 
+    assert helpers.is_ip("dead::beef")
+    assert helpers.is_ip("dead::beef", include_network=True)
+    assert not helpers.is_ip("dead::beef", version=4)
+    assert helpers.is_ip("dead::beef", version=6)
+    assert not helpers.is_ip("dead:::beef")
+
+    assert not helpers.is_ip("1.2.3.4/24")
+    assert helpers.is_ip("1.2.3.4/24", include_network=True)
+    assert not helpers.is_ip("1.2.3.4/24", version=4)
+    assert helpers.is_ip("1.2.3.4/24", include_network=True, version=4)
+    assert not helpers.is_ip("1.2.3.4/24", include_network=True, version=6)
+
     assert not helpers.is_ip_type("127.0.0.1")
     assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"))
     assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True)
@@ -104,6 +119,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False)
 
     assert helpers.is_dns_name("evilcorp.com")
+    assert not helpers.is_dns_name("evilcorp.com:80")
+    assert not helpers.is_dns_name("http://evilcorp.com:80")
     assert helpers.is_dns_name("evilcorp")
     assert not helpers.is_dns_name("evilcorp", include_local=False)
     assert helpers.is_dns_name("ドメイン.テスト")

bbot/test/test_step_1/test_modules_basic.py

@@ -10,9 +10,6 @@ from bbot.modules.internal.base import BaseInternalModule
 
 @pytest.mark.asyncio
 async def test_modules_basic_checks(events, httpx_mock):
-    for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"):
-        httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"})
-
     from bbot.scanner import Scanner
 
     scan = Scanner(config={"omit_event_types": ["URL_UNVERIFIED"]})

bbot/test/test_step_1/test_presets.py

@@ -86,12 +86,15 @@ def test_preset_yaml(clean_default_config):
         debug=False,
         silent=True,
         config={"preset_test_asdf": 1},
-        strict_scope=False,
     )
     preset1 = preset1.bake()
-    assert "evilcorp.com" in preset1.target
+    assert "evilcorp.com" in preset1.target.seeds
+    assert "evilcorp.ce" not in preset1.target.seeds
+    assert "asdf.www.evilcorp.ce" in preset1.target.seeds
     assert "evilcorp.ce" in preset1.whitelist
+    assert "asdf.evilcorp.ce" in preset1.whitelist
     assert "test.www.evilcorp.ce" in preset1.blacklist
+    assert "asdf.test.www.evilcorp.ce" in preset1.blacklist
     assert "sslcert" in preset1.scan_modules
     assert preset1.whitelisted("evilcorp.ce")
     assert preset1.whitelisted("www.evilcorp.ce")
@@ -171,12 +174,14 @@ def test_preset_scope():
 
     # test target merging
     scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
-    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4", "evilcorp.com"}
-    assert set([e.data for e in scan.target]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4/32", "evilcorp.com"}
+    assert set([e.data for e in scan.target.seeds]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([e.data for e in scan.target.whitelist]) == {"1.2.3.4", "evilcorp.com"}
 
     blank_preset = Preset()
     blank_preset = blank_preset.bake()
-    assert not blank_preset.target
+    assert not blank_preset.target.seeds
+    assert not blank_preset.target.whitelist
     assert blank_preset.strict_scope == False
 
     preset1 = Preset(
@@ -188,10 +193,11 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # make sure target logic works as expected
-    assert "evilcorp.com" in preset1_baked.target
-    assert "asdf.evilcorp.com" in preset1_baked.target
-    assert "asdf.www.evilcorp.ce" in preset1_baked.target
-    assert not "evilcorp.ce" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert not "evilcorp.com" in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.com" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.com" in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "test.www.evilcorp.ce" in preset1_baked.blacklist
     assert not "evilcorp.ce" in preset1_baked.blacklist
@@ -210,7 +216,7 @@ def test_preset_scope():
         "evilcorp.org",
         whitelist=["evilcorp.de"],
         blacklist=["test.www.evilcorp.de"],
-        strict_scope=True,
+        config={"scope": {"strict": True}},
     )
 
     preset1.merge(preset3)
@@ -218,17 +224,21 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # targets should be merged
-    assert "evilcorp.com" in preset1_baked.target
-    assert "www.evilcorp.ce" in preset1_baked.target
-    assert "evilcorp.org" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert "www.evilcorp.ce" in preset1_baked.target.seeds
+    assert "evilcorp.org" in preset1_baked.target.seeds
     # strict scope is enabled
-    assert not "asdf.evilcorp.com" in preset1_baked.target
-    assert not "asdf.www.evilcorp.ce" in preset1_baked.target
+    assert not "asdf.www.evilcorp.ce" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.org" in preset1_baked.target.seeds
+    assert not "asdf.evilcorp.com" in preset1_baked.target.seeds
+    assert not "asdf.www.evilcorp.ce" in preset1_baked.target.seeds
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.de" in preset1_baked.whitelist
     assert not "asdf.evilcorp.de" in preset1_baked.whitelist
     assert not "asdf.evilcorp.ce" in preset1_baked.whitelist
     # blacklist should be merged, strict scope does not apply
+    assert "test.www.evilcorp.ce" in preset1_baked.blacklist
+    assert "test.www.evilcorp.de" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.ce" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.de" in preset1_baked.blacklist
     assert not "asdf.test.www.evilcorp.org" in preset1_baked.blacklist
@@ -264,14 +274,14 @@ def test_preset_scope():
     }
     assert preset_whitelist_baked.to_dict(include_target=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}},
     }
     assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"otherthing": "asdf"}}},
     }
 
@@ -279,7 +289,8 @@ def test_preset_scope():
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.de")
     assert not preset_nowhitelist_baked.in_scope("1.2.3.4/24")
 
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
+    assert "www.evilcorp.org" in preset_whitelist_baked.target.seeds
+    assert not "www.evilcorp.org" in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.de")
@@ -292,17 +303,17 @@ def test_preset_scope():
     assert preset_whitelist_baked.whitelisted("1.2.3.4/28")
     assert preset_whitelist_baked.whitelisted("1.2.3.4/24")
 
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist_baked.seeds]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
+    assert set([e.data for e in preset_whitelist_baked.seeds]) == {"evilcorp.org"}
+    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24", "http://evilcorp.net/"}
 
     preset_nowhitelist.merge(preset_whitelist)
     preset_nowhitelist_baked = preset_nowhitelist.bake()
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
-    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
-    assert "www.evilcorp.org" in preset_nowhitelist_baked.target
-    assert "www.evilcorp.com" in preset_nowhitelist_baked.target
+    assert set([e.data for e in preset_nowhitelist_baked.seeds]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"1.2.3.0/24", "http://evilcorp.net/"}
+    assert "www.evilcorp.org" in preset_nowhitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_nowhitelist_baked.seeds
     assert "1.2.3.4" in preset_nowhitelist_baked.whitelist
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.com")
@@ -314,10 +325,12 @@ def test_preset_scope():
     preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
     preset_whitelist.merge(preset_nowhitelist)
     preset_whitelist_baked = preset_whitelist.bake()
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_whitelist_baked.seeds]) == {"evilcorp.com", "evilcorp.org"}
     assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24"}
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
-    assert "www.evilcorp.com" in preset_whitelist_baked.target
+    assert "www.evilcorp.org" in preset_whitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_whitelist_baked.seeds
+    assert not "www.evilcorp.org" in preset_whitelist_baked.target.whitelist
+    assert not "www.evilcorp.com" in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.com")
@@ -329,18 +342,18 @@ def test_preset_scope():
     preset_nowhitelist2 = Preset("evilcorp.de")
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
     preset_nowhitelist1.merge(preset_nowhitelist2)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com", "evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
-    assert "www.evilcorp.com" in preset_nowhitelist1_baked.target
-    assert "www.evilcorp.de" in preset_nowhitelist1_baked.target
+    assert "www.evilcorp.com" in preset_nowhitelist1_baked.seeds
+    assert "www.evilcorp.de" in preset_nowhitelist1_baked.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.de" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.whitelist
@@ -357,8 +370,8 @@ def test_preset_scope():
     preset_nowhitelist2.merge(preset_nowhitelist1)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist1_baked.seeds]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2_baked.seeds]) == {"evilcorp.com", "evilcorp.de"}
     assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
     assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
 

bbot/test/test_step_1/test_python_api.py

@@ -84,6 +84,10 @@ def test_python_api_sync():
 def test_python_api_validation():
     from bbot.scanner import Scanner, Preset
 
+    # invalid target
+    with pytest.raises(ValidationError) as error:
+        Scanner("asdf:::asdf")
+    assert str(error.value) == 'Unable to autodetect event type from "asdf:::asdf"'
     # invalid module
     with pytest.raises(ValidationError) as error:
         Scanner(modules=["asdf"])

bbot/test/test_step_1/test_scan.py

@@ -1,3 +1,5 @@
+from ipaddress import ip_network
+
 from ..bbot_fixtures import *
 
 
@@ -12,6 +14,7 @@ async def test_scan(
         "1.1.1.0",
         "1.1.1.1/31",
         "evilcorp.com",
+        "test.evilcorp.com",
         blacklist=["1.1.1.1/28", "www.evilcorp.com"],
         modules=["ipneighbor"],
     )
@@ -31,8 +34,11 @@ async def test_scan(
     assert not scan0.in_scope("test.www.evilcorp.com")
     assert not scan0.in_scope("www.evilcorp.co.uk")
     j = scan0.json
-    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com"}
-    assert set(j["target"]["whitelist"]) == {"1.1.1.0/31", "evilcorp.com"}
+    assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # we preserve the original whitelist inputs
+    assert set(j["target"]["whitelist"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"}
+    # but in the background they are collapsed
+    assert scan0.target.whitelist.hosts == {ip_network("1.1.1.0/31"), "evilcorp.com"}
     assert set(j["target"]["blacklist"]) == {"1.1.1.0/28", "www.evilcorp.com"}
     assert "ipneighbor" in j["preset"]["modules"]