bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/test/test_step_1/test_presets.py

@@ -16,7 +16,7 @@ from bbot.scanner import Scanner, Preset
 def test_preset_descriptions():
     # ensure very preset has a description
     preset = Preset()
-    for yaml_file, (loaded_preset, category, preset_path, original_filename) in preset.all_presets.items():
+    for loaded_preset, category, preset_path, original_filename in preset.all_presets.values():
         assert (
             loaded_preset.description
         ), f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.'
@@ -68,7 +68,6 @@ def test_core():
 
 
 def test_preset_yaml(clean_default_config):
-
     import yaml
 
     preset1 = Preset(
@@ -86,12 +85,15 @@ def test_preset_yaml(clean_default_config):
         debug=False,
         silent=True,
         config={"preset_test_asdf": 1},
-        strict_scope=False,
     )
     preset1 = preset1.bake()
-    assert "evilcorp.com" in preset1.target
+    assert "evilcorp.com" in preset1.target.seeds
+    assert "evilcorp.ce" not in preset1.target.seeds
+    assert "asdf.www.evilcorp.ce" in preset1.target.seeds
     assert "evilcorp.ce" in preset1.whitelist
+    assert "asdf.evilcorp.ce" in preset1.whitelist
     assert "test.www.evilcorp.ce" in preset1.blacklist
+    assert "asdf.test.www.evilcorp.ce" in preset1.blacklist
     assert "sslcert" in preset1.scan_modules
     assert preset1.whitelisted("evilcorp.ce")
     assert preset1.whitelisted("www.evilcorp.ce")
@@ -168,11 +170,17 @@ exclude_flags:
 
 
 def test_preset_scope():
+    # test target merging
+    scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
+    assert {str(h) for h in scan.preset.target.seeds.hosts} == {"1.2.3.4/32", "evilcorp.com"}
+    assert {e.data for e in scan.target.seeds} == {"1.2.3.4", "evilcorp.com"}
+    assert {e.data for e in scan.target.whitelist} == {"1.2.3.4", "evilcorp.com"}
 
     blank_preset = Preset()
     blank_preset = blank_preset.bake()
-    assert not blank_preset.target
-    assert blank_preset.strict_scope == False
+    assert not blank_preset.target.seeds
+    assert not blank_preset.target.whitelist
+    assert blank_preset.strict_scope is False
 
     preset1 = Preset(
         "evilcorp.com",
@@ -183,13 +191,14 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # make sure target logic works as expected
-    assert "evilcorp.com" in preset1_baked.target
-    assert "asdf.evilcorp.com" in preset1_baked.target
-    assert "asdf.www.evilcorp.ce" in preset1_baked.target
-    assert not "evilcorp.ce" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert "evilcorp.com" not in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.com" in preset1_baked.target.seeds
+    assert "asdf.evilcorp.com" not in preset1_baked.target.whitelist
+    assert "asdf.evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "test.www.evilcorp.ce" in preset1_baked.blacklist
-    assert not "evilcorp.ce" in preset1_baked.blacklist
+    assert "evilcorp.ce" not in preset1_baked.blacklist
     assert preset1_baked.in_scope("www.evilcorp.ce")
     assert not preset1_baked.in_scope("evilcorp.com")
     assert not preset1_baked.in_scope("asdf.test.www.evilcorp.ce")
@@ -205,7 +214,7 @@ def test_preset_scope():
         "evilcorp.org",
         whitelist=["evilcorp.de"],
         blacklist=["test.www.evilcorp.de"],
-        strict_scope=True,
+        config={"scope": {"strict": True}},
     )
 
     preset1.merge(preset3)
@@ -213,20 +222,24 @@ def test_preset_scope():
     preset1_baked = preset1.bake()
 
     # targets should be merged
-    assert "evilcorp.com" in preset1_baked.target
-    assert "www.evilcorp.ce" in preset1_baked.target
-    assert "evilcorp.org" in preset1_baked.target
+    assert "evilcorp.com" in preset1_baked.target.seeds
+    assert "www.evilcorp.ce" in preset1_baked.target.seeds
+    assert "evilcorp.org" in preset1_baked.target.seeds
     # strict scope is enabled
-    assert not "asdf.evilcorp.com" in preset1_baked.target
-    assert not "asdf.www.evilcorp.ce" in preset1_baked.target
+    assert "asdf.www.evilcorp.ce" not in preset1_baked.target.seeds
+    assert "asdf.evilcorp.org" not in preset1_baked.target.seeds
+    assert "asdf.evilcorp.com" not in preset1_baked.target.seeds
+    assert "asdf.www.evilcorp.ce" not in preset1_baked.target.seeds
     assert "evilcorp.ce" in preset1_baked.whitelist
     assert "evilcorp.de" in preset1_baked.whitelist
-    assert not "asdf.evilcorp.de" in preset1_baked.whitelist
-    assert not "asdf.evilcorp.ce" in preset1_baked.whitelist
+    assert "asdf.evilcorp.de" not in preset1_baked.whitelist
+    assert "asdf.evilcorp.ce" not in preset1_baked.whitelist
     # blacklist should be merged, strict scope does not apply
+    assert "test.www.evilcorp.ce" in preset1_baked.blacklist
+    assert "test.www.evilcorp.de" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.ce" in preset1_baked.blacklist
     assert "asdf.test.www.evilcorp.de" in preset1_baked.blacklist
-    assert not "asdf.test.www.evilcorp.org" in preset1_baked.blacklist
+    assert "asdf.test.www.evilcorp.org" not in preset1_baked.blacklist
     # only the base domain of evilcorp.de should be in scope
     assert not preset1_baked.in_scope("evilcorp.com")
     assert not preset1_baked.in_scope("evilcorp.org")
@@ -259,14 +272,14 @@ def test_preset_scope():
     }
     assert preset_whitelist_baked.to_dict(include_target=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}},
     }
     assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == {
         "target": ["evilcorp.org"],
-        "whitelist": ["1.2.3.0/24", "evilcorp.net"],
-        "blacklist": ["evilcorp.co.uk"],
+        "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
+        "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
         "config": {"modules": {"secretsdb": {"otherthing": "asdf"}}},
     }
 
@@ -274,7 +287,8 @@ def test_preset_scope():
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.de")
     assert not preset_nowhitelist_baked.in_scope("1.2.3.4/24")
 
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
+    assert "www.evilcorp.org" in preset_whitelist_baked.target.seeds
+    assert "www.evilcorp.org" not in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.de")
@@ -287,17 +301,17 @@ def test_preset_scope():
     assert preset_whitelist_baked.whitelisted("1.2.3.4/28")
     assert preset_whitelist_baked.whitelisted("1.2.3.4/24")
 
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.org"}
-    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
+    assert {e.data for e in preset_nowhitelist_baked.seeds} == {"evilcorp.com"}
+    assert {e.data for e in preset_nowhitelist_baked.whitelist} == {"evilcorp.com"}
+    assert {e.data for e in preset_whitelist_baked.seeds} == {"evilcorp.org"}
+    assert {e.data for e in preset_whitelist_baked.whitelist} == {"1.2.3.0/24", "http://evilcorp.net/"}
 
     preset_nowhitelist.merge(preset_whitelist)
     preset_nowhitelist_baked = preset_nowhitelist.bake()
-    assert set([e.data for e in preset_nowhitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
-    assert set([e.data for e in preset_nowhitelist_baked.whitelist]) == {"1.2.3.0/24", "evilcorp.net"}
-    assert "www.evilcorp.org" in preset_nowhitelist_baked.target
-    assert "www.evilcorp.com" in preset_nowhitelist_baked.target
+    assert {e.data for e in preset_nowhitelist_baked.seeds} == {"evilcorp.com", "evilcorp.org"}
+    assert {e.data for e in preset_nowhitelist_baked.whitelist} == {"1.2.3.0/24", "http://evilcorp.net/"}
+    assert "www.evilcorp.org" in preset_nowhitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_nowhitelist_baked.seeds
     assert "1.2.3.4" in preset_nowhitelist_baked.whitelist
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_nowhitelist_baked.in_scope("www.evilcorp.com")
@@ -309,10 +323,12 @@ def test_preset_scope():
     preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
     preset_whitelist.merge(preset_nowhitelist)
     preset_whitelist_baked = preset_whitelist.bake()
-    assert set([e.data for e in preset_whitelist_baked.target]) == {"evilcorp.com", "evilcorp.org"}
-    assert set([e.data for e in preset_whitelist_baked.whitelist]) == {"1.2.3.0/24"}
-    assert "www.evilcorp.org" in preset_whitelist_baked.target
-    assert "www.evilcorp.com" in preset_whitelist_baked.target
+    assert {e.data for e in preset_whitelist_baked.seeds} == {"evilcorp.com", "evilcorp.org"}
+    assert {e.data for e in preset_whitelist_baked.whitelist} == {"1.2.3.0/24"}
+    assert "www.evilcorp.org" in preset_whitelist_baked.seeds
+    assert "www.evilcorp.com" in preset_whitelist_baked.seeds
+    assert "www.evilcorp.org" not in preset_whitelist_baked.target.whitelist
+    assert "www.evilcorp.com" not in preset_whitelist_baked.target.whitelist
     assert "1.2.3.4" in preset_whitelist_baked.whitelist
     assert not preset_whitelist_baked.in_scope("www.evilcorp.org")
     assert not preset_whitelist_baked.in_scope("www.evilcorp.com")
@@ -324,18 +340,18 @@ def test_preset_scope():
     preset_nowhitelist2 = Preset("evilcorp.de")
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com"}
+    assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com"}
+    assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.de"}
     preset_nowhitelist1.merge(preset_nowhitelist2)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com", "evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.de"}
-    assert "www.evilcorp.com" in preset_nowhitelist1_baked.target
-    assert "www.evilcorp.de" in preset_nowhitelist1_baked.target
+    assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com", "evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com", "evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.de"}
+    assert "www.evilcorp.com" in preset_nowhitelist1_baked.seeds
+    assert "www.evilcorp.de" in preset_nowhitelist1_baked.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.de" in preset_nowhitelist1_baked.target.seeds
     assert "www.evilcorp.com" in preset_nowhitelist1_baked.whitelist
@@ -352,15 +368,14 @@ def test_preset_scope():
     preset_nowhitelist2.merge(preset_nowhitelist1)
     preset_nowhitelist1_baked = preset_nowhitelist1.bake()
     preset_nowhitelist2_baked = preset_nowhitelist2.bake()
-    assert set([e.data for e in preset_nowhitelist1_baked.target]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.target]) == {"evilcorp.com", "evilcorp.de"}
-    assert set([e.data for e in preset_nowhitelist1_baked.whitelist]) == {"evilcorp.com"}
-    assert set([e.data for e in preset_nowhitelist2_baked.whitelist]) == {"evilcorp.com", "evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com"}
+    assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.com", "evilcorp.de"}
+    assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com"}
+    assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.com", "evilcorp.de"}
 
 
 @pytest.mark.asyncio
 async def test_preset_logging():
-
     scan = Scanner()
 
     # test individual verbosity levels
@@ -369,30 +384,30 @@ async def test_preset_logging():
 
     try:
         silent_preset = Preset(silent=True)
-        assert silent_preset.silent == True
-        assert silent_preset.debug == False
-        assert silent_preset.verbose == False
+        assert silent_preset.silent is True
+        assert silent_preset.debug is False
+        assert silent_preset.verbose is False
         assert original_log_level == CORE.logger.log_level
         debug_preset = Preset(debug=True)
-        assert debug_preset.silent == False
-        assert debug_preset.debug == True
-        assert debug_preset.verbose == False
+        assert debug_preset.silent is False
+        assert debug_preset.debug is True
+        assert debug_preset.verbose is False
         assert original_log_level == CORE.logger.log_level
         verbose_preset = Preset(verbose=True)
-        assert verbose_preset.silent == False
-        assert verbose_preset.debug == False
-        assert verbose_preset.verbose == True
+        assert verbose_preset.silent is False
+        assert verbose_preset.debug is False
+        assert verbose_preset.verbose is True
         assert original_log_level == CORE.logger.log_level
 
         # test conflicting verbosity levels
         silent_and_verbose = Preset(silent=True, verbose=True)
-        assert silent_and_verbose.silent == True
-        assert silent_and_verbose.debug == False
-        assert silent_and_verbose.verbose == True
+        assert silent_and_verbose.silent is True
+        assert silent_and_verbose.debug is False
+        assert silent_and_verbose.verbose is True
         baked = silent_and_verbose.bake()
-        assert baked.silent == True
-        assert baked.debug == False
-        assert baked.verbose == False
+        assert baked.silent is True
+        assert baked.debug is False
+        assert baked.verbose is False
         assert baked.core.logger.log_level == original_log_level
         baked = silent_and_verbose.bake(scan=scan)
         assert baked.core.logger.log_level == logging.CRITICAL
@@ -402,13 +417,13 @@ async def test_preset_logging():
         assert CORE.logger.log_level == original_log_level
 
         silent_and_debug = Preset(silent=True, debug=True)
-        assert silent_and_debug.silent == True
-        assert silent_and_debug.debug == True
-        assert silent_and_debug.verbose == False
+        assert silent_and_debug.silent is True
+        assert silent_and_debug.debug is True
+        assert silent_and_debug.verbose is False
         baked = silent_and_debug.bake()
-        assert baked.silent == True
-        assert baked.debug == False
-        assert baked.verbose == False
+        assert baked.silent is True
+        assert baked.debug is False
+        assert baked.verbose is False
         assert baked.core.logger.log_level == original_log_level
         baked = silent_and_debug.bake(scan=scan)
         assert baked.core.logger.log_level == logging.CRITICAL
@@ -418,13 +433,13 @@ async def test_preset_logging():
         assert CORE.logger.log_level == original_log_level
 
         debug_and_verbose = Preset(verbose=True, debug=True)
-        assert debug_and_verbose.silent == False
-        assert debug_and_verbose.debug == True
-        assert debug_and_verbose.verbose == True
+        assert debug_and_verbose.silent is False
+        assert debug_and_verbose.debug is True
+        assert debug_and_verbose.verbose is True
         baked = debug_and_verbose.bake()
-        assert baked.silent == False
-        assert baked.debug == True
-        assert baked.verbose == False
+        assert baked.silent is False
+        assert baked.debug is True
+        assert baked.verbose is False
         assert baked.core.logger.log_level == original_log_level
         baked = debug_and_verbose.bake(scan=scan)
         assert baked.core.logger.log_level == logging.DEBUG
@@ -434,13 +449,13 @@ async def test_preset_logging():
         assert CORE.logger.log_level == original_log_level
 
         all_preset = Preset(verbose=True, debug=True, silent=True)
-        assert all_preset.silent == True
-        assert all_preset.debug == True
-        assert all_preset.verbose == True
+        assert all_preset.silent is True
+        assert all_preset.debug is True
+        assert all_preset.verbose is True
         baked = all_preset.bake()
-        assert baked.silent == True
-        assert baked.debug == False
-        assert baked.verbose == False
+        assert baked.silent is True
+        assert baked.debug is False
+        assert baked.verbose is False
         assert baked.core.logger.log_level == original_log_level
         baked = all_preset.bake(scan=scan)
         assert baked.core.logger.log_level == logging.CRITICAL
@@ -527,12 +542,26 @@ def test_preset_module_resolution(clean_default_config):
     assert set(preset.scan_modules) == {"wayback"}
 
     # modules + module exclusions
-    with pytest.raises(ValidationError) as error:
-        preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake()
-    assert str(error.value) == 'Unable to add scan module "sslcert" because the module has been excluded'
+    preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake()
+    baked_preset = preset.bake()
+    assert baked_preset.modules == {
+        "wayback",
+        "cloudcheck",
+        "python",
+        "json",
+        "speculate",
+        "dnsresolve",
+        "aggregate",
+        "excavate",
+        "txt",
+        "httpx",
+        "csv",
+        "wappalyzer",
+    }
 
 
-def test_preset_module_loader():
+@pytest.mark.asyncio
+async def test_preset_module_loader():
     custom_module_dir = bbot_test_dir / "custom_module_dir"
     custom_module_dir_2 = custom_module_dir / "asdf"
     custom_output_module_dir = custom_module_dir / "output"
@@ -640,9 +669,45 @@ class TestModule4(BaseModule):
     # reset module_loader
     preset2.module_loader.__init__()
 
+    # custom module dir via preset
+    custom_module_dir_3 = bbot_test_dir / "custom_module_dir_3"
+    custom_module_dir_3.mkdir(exist_ok=True, parents=True)
+    custom_module_5 = custom_module_dir_3 / "testmodule5.py"
+    with open(custom_module_5, "w") as f:
+        f.write(
+            """
+from bbot.modules.base import BaseModule
 
-def test_preset_include():
+class TestModule5(BaseModule):
+    watched_events = ["TECHNOLOGY"]
+    produced_events = ["FINDING"]
+"""
+        )
 
+    preset = Preset.from_yaml_string(
+        """
+modules:
+  - testmodule5
+"""
+    )
+    # should fail
+    with pytest.raises(ValidationError):
+        scan = Scanner(preset=preset)
+
+    preset = Preset.from_yaml_string(
+        f"""
+module_dirs:
+  - {custom_module_dir_3}
+modules:
+  - testmodule5
+"""
+    )
+    scan = Scanner(preset=preset)
+    await scan._prep()
+    assert "testmodule5" in scan.modules
+
+
+def test_preset_include():
     # test recursive preset inclusion
 
     custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir"
@@ -814,7 +879,6 @@ def test_preset_module_disablement(clean_default_config):
 
 
 def test_preset_require_exclude():
-
     def get_module_flags(p):
         for m in p.scan_modules:
             preloaded = p.preloaded_module(m)
@@ -827,9 +891,9 @@ def test_preset_require_exclude():
     dnsbrute_flags = preset.preloaded_module("dnsbrute").get("flags", [])
     assert "subdomain-enum" in dnsbrute_flags
     assert "active" in dnsbrute_flags
-    assert not "passive" in dnsbrute_flags
+    assert "passive" not in dnsbrute_flags
     assert "aggressive" in dnsbrute_flags
-    assert not "safe" in dnsbrute_flags
+    assert "safe" not in dnsbrute_flags
     assert "dnsbrute" in [x[0] for x in module_flags]
     assert "certspotter" in [x[0] for x in module_flags]
     assert "c99" in [x[0] for x in module_flags]
@@ -843,7 +907,7 @@ def test_preset_require_exclude():
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
     assert "chaos" in [x[0] for x in module_flags]
-    assert not "httpx" in [x[0] for x in module_flags]
+    assert "httpx" not in [x[0] for x in module_flags]
     assert all("passive" in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
@@ -854,7 +918,7 @@ def test_preset_require_exclude():
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
     assert "chaos" in [x[0] for x in module_flags]
-    assert not "httpx" in [x[0] for x in module_flags]
+    assert "httpx" not in [x[0] for x in module_flags]
     assert all("passive" in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
@@ -864,7 +928,7 @@ def test_preset_require_exclude():
     preset = Preset(flags=["subdomain-enum"], exclude_modules=["dnsbrute"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "dnsbrute" in [x[0] for x in module_flags]
+    assert "dnsbrute" not in [x[0] for x in module_flags]
     assert "httpx" in [x[0] for x in module_flags]
     assert any("passive" in flags for module, flags in module_flags)
     assert any("active" in flags for module, flags in module_flags)
@@ -875,7 +939,7 @@ def test_preset_require_exclude():
     preset = Preset(flags=["subdomain-enum"], require_flags=["safe", "passive"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "dnsbrute" in [x[0] for x in module_flags]
+    assert "dnsbrute" not in [x[0] for x in module_flags]
     assert all("passive" in flags and "safe" in flags for module, flags in module_flags)
     assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
@@ -885,7 +949,7 @@ def test_preset_require_exclude():
     preset = Preset(flags=["subdomain-enum"], exclude_flags=["aggressive", "active"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "dnsbrute" in [x[0] for x in module_flags]
+    assert "dnsbrute" not in [x[0] for x in module_flags]
     assert all("passive" in flags and "safe" in flags for module, flags in module_flags)
     assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
@@ -895,10 +959,29 @@ def test_preset_require_exclude():
     preset = Preset(flags=["subdomain-enum"], exclude_modules=["dnsbrute", "c99"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "dnsbrute" in [x[0] for x in module_flags]
+    assert "dnsbrute" not in [x[0] for x in module_flags]
     assert "certspotter" in [x[0] for x in module_flags]
-    assert not "c99" in [x[0] for x in module_flags]
+    assert "c99" not in [x[0] for x in module_flags]
     assert any("passive" in flags for module, flags in module_flags)
     assert any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
     assert any("aggressive" in flags for module, flags in module_flags)
+
+
+@pytest.mark.asyncio
+async def test_preset_output_dir():
+    output_dir = bbot_test_dir / "preset_output_dir"
+    preset = Preset.from_yaml_string(
+        f"""
+output_dir: {output_dir}
+scan_name: bbot_test
+"""
+    )
+    scan = Scanner(preset=preset)
+    await scan.async_start_without_generator()
+    scan_dir = output_dir / "bbot_test"
+    assert scan_dir.is_dir()
+    output_file = scan_dir / "output.txt"
+    assert output_file.is_file()
+
+    shutil.rmtree(output_dir, ignore_errors=True)

bbot/test/test_step_1/test_python_api.py

@@ -84,6 +84,10 @@ def test_python_api_sync():
 def test_python_api_validation():
     from bbot.scanner import Scanner, Preset
 
+    # invalid target
+    with pytest.raises(ValidationError) as error:
+        Scanner("asdf:::asdf")
+    assert str(error.value) == 'Unable to autodetect event type from "asdf:::asdf"'
     # invalid module
     with pytest.raises(ValidationError) as error:
         Scanner(modules=["asdf"])
@@ -122,7 +126,8 @@ def test_python_api_validation():
     assert str(error.value) == 'Preset must be of type Preset, not "str"'
     # include nonexistent preset
     with pytest.raises(ValidationError) as error:
-        Preset(include=["asdf"])
+        Preset(include=["nonexistent"])
     assert (
-        str(error.value) == 'Could not find preset at "asdf" - file does not exist. Use -lp to list available presets'
+        str(error.value)
+        == 'Could not find preset at "nonexistent" - file does not exist. Use -lp to list available presets'
     )

bbot/test/test_step_1/test_regexes.py

@@ -91,7 +91,7 @@ def test_ip_regexes():
                     ip == "2001:db8::1/128" and event_type == "IP_RANGE"
                 ), f"Event type for IP_ADDRESS {ip} was not properly detected"
         else:
-            matches = list(r.match(ip) for r in ip_address_regexes)
+            matches = [r.match(ip) for r in ip_address_regexes]
             assert any(matches), f"Good IP ADDRESS {ip} did not match regexes"
 
 
@@ -138,7 +138,7 @@ def test_ip_range_regexes():
             pytest.fail(f"BAD IP_RANGE: {bad_ip_range} raised unknown error: {e}: {traceback.format_exc()}")
 
     for good_ip_range in good_ip_ranges:
-        matches = list(r.match(good_ip_range) for r in ip_range_regexes)
+        matches = [r.match(good_ip_range) for r in ip_range_regexes]
         assert any(matches), f"Good IP_RANGE {good_ip_range} did not match regexes"
 
 
@@ -191,7 +191,7 @@ def test_dns_name_regexes():
             pytest.fail(f"BAD DNS NAME: {dns} raised unknown error: {e}")
 
     for dns in good_dns:
-        matches = list(r.match(dns) for r in dns_name_regexes)
+        matches = [r.match(dns) for r in dns_name_regexes]
         assert any(matches), f"Good DNS_NAME {dns} did not match regexes"
         event_type, _ = get_event_type(dns)
         if not event_type == "DNS_NAME":
@@ -253,7 +253,7 @@ def test_open_port_regexes():
             pytest.fail(f"BAD OPEN_TCP_PORT: {open_port} raised unknown error: {e}")
 
     for open_port in good_ports:
-        matches = list(r.match(open_port) for r in open_port_regexes)
+        matches = [r.match(open_port) for r in open_port_regexes]
         assert any(matches), f"Good OPEN_TCP_PORT {open_port} did not match regexes"
         event_type, _ = get_event_type(open_port)
         assert event_type == "OPEN_TCP_PORT"
@@ -318,7 +318,7 @@ def test_url_regexes():
             pytest.fail(f"BAD URL: {bad_url} raised unknown error: {e}: {traceback.format_exc()}")
 
     for good_url in good_urls:
-        matches = list(r.match(good_url) for r in url_regexes)
+        matches = [r.match(good_url) for r in url_regexes]
         assert any(matches), f"Good URL {good_url} did not match regexes"
         assert (
             get_event_type(good_url)[0] == "URL_UNVERIFIED"
@@ -372,3 +372,33 @@ async def test_regex_helper():
         assert matches.count(s) == 2
 
     await scan._cleanup()
+
+    # test yara hostname extractor helper
+    scan = Scanner("evilcorp.com", "www.evilcorp.net", "evilcorp.co.uk")
+    host_blob = """
+    https://evilcorp.com/
+    https://asdf.evilcorp.com/
+    https://asdf.www.evilcorp.net/
+    https://asdf.www.evilcorp.co.uk/
+    https://asdf.www.evilcorp.com/
+    https://asdf.www.evilcorp.com/
+    https://test.api.www.evilcorp.net/
+    """
+    extracted = await scan.extract_in_scope_hostnames(host_blob)
+    assert extracted == {
+        "evilcorp.co.uk",
+        "evilcorp.com",
+        "www.evilcorp.com",
+        "asdf.evilcorp.com",
+        "asdf.www.evilcorp.com",
+        "www.evilcorp.net",
+        "api.www.evilcorp.net",
+        "asdf.www.evilcorp.net",
+        "test.api.www.evilcorp.net",
+        "asdf.www.evilcorp.co.uk",
+        "www.evilcorp.co.uk",
+    }
+
+    scan = Scanner()
+    extracted = await scan.extract_in_scope_hostnames(host_blob)
+    assert extracted == set()