bbot 2.3.0.5546rc0__py3-none-any.whl → 2.3.1.5815rc0__py3-none-any.whl

bbot/scanner/scanner.py

@@ -334,7 +334,7 @@ class Scanner:
             await self._prep()
 
             self._start_log_handlers()
-            self.trace(f'Ran BBOT {__version__} at {self.start_time}, command: {" ".join(sys.argv)}')
+            self.trace(f"Ran BBOT {__version__} at {self.start_time}, command: {' '.join(sys.argv)}")
             self.trace(f"Target: {self.preset.target.json}")
             self.trace(f"Preset: {self.preset.to_dict(redact_secrets=True)}")
 
@@ -683,14 +683,14 @@ class Scanner:
             event_type_summary = sorted(self.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True)
             if event_type_summary:
                 self.info(
-                    f'{self.name}: Events produced so far: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}'
+                    f"{self.name}: Events produced so far: {', '.join([f'{k}: {v}' for k, v in event_type_summary])}"
                 )
             else:
                 self.info(f"{self.name}: No events produced yet")
 
             if modules_errored:
                 self.verbose(
-                    f'{self.name}: Modules errored: {len(modules_errored):,} ({", ".join(list(modules_errored))})'
+                    f"{self.name}: Modules errored: {len(modules_errored):,} ({', '.join(list(modules_errored))})"
                 )
 
             num_queued_events = self.num_queued_events
@@ -1089,9 +1089,19 @@ class Scanner:
             regexes_component_list = []
             for i, r in enumerate(self.dns_regexes_yara):
                 regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase")
-            if regexes_component_list:
-                regexes_component = " ".join(regexes_component_list)
-                self._dns_yara_rules_uncompiled = f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}'
+
+            # Chunk the regexes into groups of 10,000
+            chunk_size = 10000
+            rules = {}
+            for chunk_index in range(0, len(regexes_component_list), chunk_size):
+                chunk = regexes_component_list[chunk_index : chunk_index + chunk_size]
+                if chunk:
+                    regexes_component = " ".join(chunk)
+                    rule_name = f"hostname_extraction_{chunk_index // chunk_size}"
+                    rule = f'rule {rule_name} {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}'
+                    rules[rule_name] = rule
+
+            self._dns_yara_rules_uncompiled = rules
         return self._dns_yara_rules_uncompiled
 
     async def dns_yara_rules(self):
@@ -1100,7 +1110,7 @@ class Scanner:
                 import yara
 
                 self._dns_yara_rules = await self.helpers.run_in_executor(
-                    yara.compile, source=self.dns_yara_rules_uncompiled
+                    yara.compile, source="\n".join(self.dns_yara_rules_uncompiled.values())
                 )
         return self._dns_yara_rules
 

bbot/test/bbot_fixtures.py

@@ -224,12 +224,12 @@ def events(scan):
     return bbot_events
 
 
-@pytest.fixture(scope="session", autouse=True)
-def install_all_python_deps():
-    deps_pip = set()
-    for module in DEFAULT_PRESET.module_loader.preloaded().values():
-        deps_pip.update(set(module.get("deps", {}).get("pip", [])))
+# @pytest.fixture(scope="session", autouse=True)
+# def install_all_python_deps():
+#     deps_pip = set()
+#     for module in DEFAULT_PRESET.module_loader.preloaded().values():
+#         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
 
-    constraint_file = tempwordlist(get_python_constraints())
+#     constraint_file = tempwordlist(get_python_constraints())
 
-    subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))
+#     subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip))

bbot/test/test_step_1/test_bloom_filter.py

@@ -38,7 +38,7 @@ async def test_bloom_filter():
         test_set.add(hash(item))
     end = time.time()
     elapsed = end - start
-    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test/elapsed)}/s)")
+    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test / elapsed)}/s)")
     # this shouldn't take longer than 5 seconds
     assert elapsed < 5
 
@@ -48,7 +48,7 @@ async def test_bloom_filter():
         assert item in bloom_filter
     end = time.time()
     elapsed = end - start
-    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test/elapsed)}/s)")
+    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test / elapsed)}/s)")
     # this shouldn't take longer than 5 seconds
     assert elapsed < 5
 

bbot/test/test_step_1/test_cli.py

@@ -271,7 +271,7 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     result = await cli._main()
     out, err = capsys.readouterr()
     assert result is None
-    assert "Target:\n  -t TARGET [TARGET ...]" in out
+    assert "-t TARGET [TARGET ...]" in out
 
     # list modules
     monkeypatch.setattr("sys.argv", ["bbot", "-l"])
@@ -342,17 +342,17 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     monkeypatch.setattr("sys.argv", ["bbot", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 5/5 internal modules (aggregate,cloudcheck,dnsresolve,excavate,speculate)" in caplog.text
+    assert "Loaded 6/6 internal modules (aggregate,cloudcheck,dnsresolve,excavate,speculate,unarchive)" in caplog.text
     caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-em", "excavate", "speculate", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 3/3 internal modules (aggregate,cloudcheck,dnsresolve)" in caplog.text
+    assert "Loaded 4/4 internal modules (aggregate,cloudcheck,dnsresolve,unarchive)" in caplog.text
     caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-c", "speculate=false", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 4/4 internal modules (aggregate,cloudcheck,dnsresolve,excavate)" in caplog.text
+    assert "Loaded 5/5 internal modules (aggregate,cloudcheck,dnsresolve,excavate,unarchive)" in caplog.text
 
     # custom target type
     out, err = capsys.readouterr()
@@ -607,7 +607,7 @@ def test_cli_presets(monkeypatch, capsys, caplog):
     assert "    http_proxy: currentpresettest" in captured.out
 
     # show current preset (full)
-    monkeypatch.setattr("sys.argv", ["bbot", "-c" "modules.c99.api_key=asdf", "--current-preset-full"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-cmodules.c99.api_key=asdf", "--current-preset-full"])
     cli.main()
     captured = capsys.readouterr()
     assert "      api_key: asdf" in captured.out

bbot/test/test_step_1/test_dns.py

@@ -776,6 +776,39 @@ async def test_dns_graph_structure(bbot_scanner):
     assert str(events_by_data["evilcorp.com"].module) == "host"
 
 
+@pytest.mark.asyncio
+async def test_hostname_extraction(bbot_scanner):
+    scan = bbot_scanner("evilcorp.com", config={"dns": {"minimal": False}})
+    await scan.helpers.dns._mock_dns(
+        {
+            "evilcorp.com": {
+                "A": ["127.0.0.1"],
+                "TXT": [
+                    "v=spf1 include:spf-a.evilcorp.com include:spf-b.evilcorp.com include:icpbounce.com include:shops.shopify.com include:_spf.qemailserver.com include:spf.mandrillapp.com include:spf.protection.office365.us include:spf-003ea501.gpphosted.com 127.0.0.1 -all"
+                ],
+            }
+        }
+    )
+    events = [e async for e in scan.async_start()]
+    dns_name_events = [e for e in events if e.type == "DNS_NAME"]
+    main_dns_event = [e for e in dns_name_events if e.data == "evilcorp.com"]
+    assert len(main_dns_event) == 1
+    main_dns_event = main_dns_event[0]
+    dns_children = main_dns_event.dns_children
+    assert dns_children["A"] == {"127.0.0.1"}
+    assert dns_children["TXT"] == {
+        "spf-a.evilcorp.com",
+        "spf-b.evilcorp.com",
+        "icpbounce.com",
+        "shops.shopify.com",
+        "_spf.qemailserver.com",
+        "spf.mandrillapp.com",
+        "spf.protection.office365.us",
+        "spf-003ea501.gpphosted.com",
+        "127.0.0.1",
+    }
+
+
 @pytest.mark.asyncio
 async def test_dns_helpers(bbot_scanner):
     assert service_record("") is False

bbot/test/test_step_1/test_events.py

@@ -964,15 +964,25 @@ def test_event_closest_host():
     assert vuln.data["path"] == "/tmp/asdf.txt"
     assert vuln.host == "www.evilcorp.com"
 
-    # no host == not allowed
+    # no host and no path == not allowed
     event3 = scan.make_event("wat", "ASDF", parent=scan.root_event)
     assert not event3.host
     with pytest.raises(ValueError):
-        finding = scan.make_event({"path": "/tmp/asdf.txt", "description": "test"}, "FINDING", parent=event3)
+        finding = scan.make_event({"description": "test"}, "FINDING", parent=event3)
+    finding = scan.make_event({"path": "/tmp/asdf.txt", "description": "test"}, "FINDING", parent=event3)
+    assert finding is not None
+    finding = scan.make_event({"host": "evilcorp.com", "description": "test"}, "FINDING", parent=event3)
+    assert finding is not None
     with pytest.raises(ValueError):
-        vuln = scan.make_event(
-            {"path": "/tmp/asdf.txt", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
-        )
+        vuln = scan.make_event({"description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3)
+    vuln = scan.make_event(
+        {"path": "/tmp/asdf.txt", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
+    )
+    assert vuln is not None
+    vuln = scan.make_event(
+        {"host": "evilcorp.com", "description": "test", "severity": "HIGH"}, "VULNERABILITY", parent=event3
+    )
+    assert vuln is not None
 
 
 def test_event_magic():

bbot/test/test_step_1/test_modules_basic.py

@@ -147,15 +147,15 @@ async def test_modules_basic_checks(events, httpx_mock):
         for flag in flags:
             all_flags.add(flag)
         if preloaded["type"] == "scan":
-            assert ("active" in flags and "passive" not in flags) or (
-                "active" not in flags and "passive" in flags
-            ), f'module "{module_name}" must have either "active" or "passive" flag'
+            assert ("active" in flags and "passive" not in flags) or ("active" not in flags and "passive" in flags), (
+                f'module "{module_name}" must have either "active" or "passive" flag'
+            )
             assert ("safe" in flags and "aggressive" not in flags) or (
                 "safe" not in flags and "aggressive" in flags
             ), f'module "{module_name}" must have either "safe" or "aggressive" flag'
-            assert not (
-                "web-basic" in flags and "web-thorough" in flags
-            ), f'module "{module_name}" should have either "web-basic" or "web-thorough" flags, not both'
+            assert not ("web-basic" in flags and "web-thorough" in flags), (
+                f'module "{module_name}" should have either "web-basic" or "web-thorough" flags, not both'
+            )
         meta = preloaded.get("meta", {})
         # make sure every module has a description
         assert meta.get("description", ""), f"{module_name} must have a description"
@@ -176,29 +176,29 @@ async def test_modules_basic_checks(events, httpx_mock):
             assert watched_events, f"{module_name}.watched_events must not be empty"
         assert type(watched_events) == list, f"{module_name}.watched_events must be of type list"
         assert type(produced_events) == list, f"{module_name}.produced_events must be of type list"
-        assert all(
-            type(t) == str for t in watched_events
-        ), f"{module_name}.watched_events entries must be of type string"
-        assert all(
-            type(t) == str for t in produced_events
-        ), f"{module_name}.produced_events entries must be of type string"
+        assert all(type(t) == str for t in watched_events), (
+            f"{module_name}.watched_events entries must be of type string"
+        )
+        assert all(type(t) == str for t in produced_events), (
+            f"{module_name}.produced_events entries must be of type string"
+        )
 
         assert type(preloaded.get("deps_pip", [])) == list, f"{module_name}.deps_pip must be of type list"
-        assert (
-            type(preloaded.get("deps_pip_constraints", [])) == list
-        ), f"{module_name}.deps_pip_constraints must be of type list"
+        assert type(preloaded.get("deps_pip_constraints", [])) == list, (
+            f"{module_name}.deps_pip_constraints must be of type list"
+        )
         assert type(preloaded.get("deps_apt", [])) == list, f"{module_name}.deps_apt must be of type list"
         assert type(preloaded.get("deps_shell", [])) == list, f"{module_name}.deps_shell must be of type list"
         assert type(preloaded.get("config", None)) == dict, f"{module_name}.options must be of type list"
         assert type(preloaded.get("options_desc", None)) == dict, f"{module_name}.options_desc must be of type list"
         # options must have descriptions
-        assert set(preloaded.get("config", {})) == set(
-            preloaded.get("options_desc", {})
-        ), f"{module_name}.options do not match options_desc"
+        assert set(preloaded.get("config", {})) == set(preloaded.get("options_desc", {})), (
+            f"{module_name}.options do not match options_desc"
+        )
         # descriptions most not be blank
-        assert all(
-            o for o in preloaded.get("options_desc", {}).values()
-        ), f"{module_name}.options_desc descriptions must not be blank"
+        assert all(o for o in preloaded.get("options_desc", {}).values()), (
+            f"{module_name}.options_desc descriptions must not be blank"
+        )
 
     from bbot.core.flags import flag_descriptions
 

bbot/test/test_step_1/test_presets.py

@@ -17,9 +17,9 @@ def test_preset_descriptions():
     # ensure very preset has a description
     preset = Preset()
     for loaded_preset, category, preset_path, original_filename in preset.all_presets.values():
-        assert (
-            loaded_preset.description
-        ), f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.'
+        assert loaded_preset.description, (
+            f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.'
+        )
 
 
 def test_core():
@@ -493,7 +493,14 @@ def test_preset_module_resolution(clean_default_config):
     # make sure we have the expected defaults
     assert not preset.scan_modules
     assert set(preset.output_modules) == {"python", "csv", "txt", "json"}
-    assert set(preset.internal_modules) == {"aggregate", "excavate", "speculate", "cloudcheck", "dnsresolve"}
+    assert set(preset.internal_modules) == {
+        "aggregate",
+        "excavate",
+        "unarchive",
+        "speculate",
+        "cloudcheck",
+        "dnsresolve",
+    }
     assert preset.modules == set(preset.output_modules).union(set(preset.internal_modules))
 
     # make sure dependency resolution works as expected
@@ -553,6 +560,7 @@ def test_preset_module_resolution(clean_default_config):
         "dnsresolve",
         "aggregate",
         "excavate",
+        "unarchive",
         "txt",
         "httpx",
         "csv",
@@ -878,6 +886,88 @@ def test_preset_module_disablement(clean_default_config):
     assert set(preset.output_modules) == {"json"}
 
 
+def test_preset_override():
+    # tests to make sure a preset's config settings override others it includes
+    preset_1_yaml = """
+name: override1
+scan_name: override1
+target: ["evilcorp1.com"]
+silent: True
+modules:
+  - robots
+config:
+  modules:
+    asdf:
+      option1: asdf
+"""
+    preset_2_yaml = """
+name: override2
+scan_name: override2
+target: ["evilcorp2.com"]
+debug: true
+modules:
+  - c99
+config:
+  modules:
+    asdf:
+      option1: fdsa
+"""
+    preset_3_yaml = """
+name: override3
+scan_name: override3
+target: ["evilcorp3.com"]
+modules:
+  - securitytrails
+# test ordering priority
+include:
+  - override1
+  - override2
+config:
+  web:
+    spider_distance: 2
+    spider_depth: 3
+"""
+    preset_4_yaml = """
+name: override4
+scan_name: override4
+target: ["evilcorp4.com"]
+modules:
+  - virustotal
+include:
+  - override3
+config:
+  web:
+    spider_distance: 1
+    spider_depth: 2
+"""
+    custom_preset_dir = bbot_test_dir / "custom_preset_dir_override"
+    custom_preset_dir.mkdir(parents=True, exist_ok=True)
+    preset_1_file = custom_preset_dir / "override1.yml"
+    preset_1_file.write_text(preset_1_yaml)
+    preset_2_file = custom_preset_dir / "override2.yml"
+    preset_2_file.write_text(preset_2_yaml)
+    preset_3_file = custom_preset_dir / "override3.yml"
+    preset_3_file.write_text(preset_3_yaml)
+    preset_4_file = custom_preset_dir / "override4.yml"
+    preset_4_file.write_text(preset_4_yaml)
+
+    preset = Preset.from_yaml_file(preset_4_file.resolve())
+    assert preset.debug is True
+    assert preset.silent is True
+    assert preset.name == "override4"
+    preset = preset.bake()
+    assert preset.debug is False
+    assert preset.silent is True
+    assert preset.name == "override4"
+    assert preset.scan_name == "override4"
+    targets = set([str(e.data) for e in preset.target.seeds])
+    assert targets == {"evilcorp1.com", "evilcorp2.com", "evilcorp3.com", "evilcorp4.com"}
+    assert preset.config["web"]["spider_distance"] == 1
+    assert preset.config["web"]["spider_depth"] == 2
+    assert preset.config["modules"]["asdf"]["option1"] == "fdsa"
+    assert set(preset.scan_modules) == {"httpx", "c99", "robots", "virustotal", "securitytrails"}
+
+
 def test_preset_require_exclude():
     def get_module_flags(p):
         for m in p.scan_modules:

bbot/test/test_step_1/test_regexes.py

@@ -83,13 +83,13 @@ def test_ip_regexes():
         event_type, _ = get_event_type(ip)
         if not event_type == "IP_ADDRESS":
             if ip.endswith("/24"):
-                assert (
-                    ip == "203.0.113.0/24" and event_type == "IP_RANGE"
-                ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+                assert ip == "203.0.113.0/24" and event_type == "IP_RANGE", (
+                    f"Event type for IP_ADDRESS {ip} was not properly detected"
+                )
             else:
-                assert (
-                    ip == "2001:db8::1/128" and event_type == "IP_RANGE"
-                ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+                assert ip == "2001:db8::1/128" and event_type == "IP_RANGE", (
+                    f"Event type for IP_ADDRESS {ip} was not properly detected"
+                )
         else:
             matches = [r.match(ip) for r in ip_address_regexes]
             assert any(matches), f"Good IP ADDRESS {ip} did not match regexes"
@@ -195,9 +195,9 @@ def test_dns_name_regexes():
         assert any(matches), f"Good DNS_NAME {dns} did not match regexes"
         event_type, _ = get_event_type(dns)
         if not event_type == "DNS_NAME":
-            assert (
-                dns == "1.2.3.4" and event_type == "IP_ADDRESS"
-            ), f"Event type for DNS_NAME {dns} was not properly detected"
+            assert dns == "1.2.3.4" and event_type == "IP_ADDRESS", (
+                f"Event type for DNS_NAME {dns} was not properly detected"
+            )
 
 
 def test_open_port_regexes():
@@ -279,7 +279,7 @@ def test_url_regexes():
         "http://-evilcorp-.com",
         "http://evilcorp-.com/path",
         "http://evilcorp.com-/path",
-        "evilcorp.com/path" "asdfasdfasdfasdfgsdgasdfs.asdfasdfasdfasdfasdf.evilcorp.com/path",
+        "evilcorp.com/pathasdfasdfasdfasdfgsdgasdfs.asdfasdfasdfasdfasdf.evilcorp.com/path",
         "rhttps://evilcorp.com",
         "https://[e]",
         "https://[1]:80",
@@ -320,9 +320,9 @@ def test_url_regexes():
     for good_url in good_urls:
         matches = [r.match(good_url) for r in url_regexes]
         assert any(matches), f"Good URL {good_url} did not match regexes"
-        assert (
-            get_event_type(good_url)[0] == "URL_UNVERIFIED"
-        ), f"Event type for URL {good_url} was not properly detected"
+        assert get_event_type(good_url)[0] == "URL_UNVERIFIED", (
+            f"Event type for URL {good_url} was not properly detected"
+        )
 
 
 @pytest.mark.asyncio

bbot/test/test_step_1/test_scan.py

@@ -149,3 +149,81 @@ async def test_python_output_matches_json(bbot_scanner):
     assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1
     assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1
     assert events == json_events
+
+
+@pytest.mark.asyncio
+async def test_huge_target_list(bbot_scanner, monkeypatch):
+    # single target should only have one rule
+    scan = bbot_scanner("evilcorp.com", config={"excavate": True})
+    await scan._prep()
+    assert "hostname_extraction_0" in scan.modules["excavate"].yara_rules_dict
+    assert "hostname_extraction_1" not in scan.modules["excavate"].yara_rules_dict
+
+    # over 10000 targets should be broken into two rules
+    num_targets = 10005
+    targets = [f"evil{i}.com" for i in range(num_targets)]
+    scan = bbot_scanner(*targets, config={"excavate": True})
+    await scan._prep()
+    assert "hostname_extraction_0" in scan.modules["excavate"].yara_rules_dict
+    assert "hostname_extraction_1" in scan.modules["excavate"].yara_rules_dict
+    assert "hostname_extraction_2" not in scan.modules["excavate"].yara_rules_dict
+
+
+@pytest.mark.asyncio
+async def test_exclude_cdn(bbot_scanner, monkeypatch):
+    # test that CDN exclusion works
+
+    from bbot import Preset
+
+    dns_mock = {
+        "evilcorp.com": {"A": ["127.0.0.1"]},
+        "www.evilcorp.com": {"A": ["127.0.0.1"]},
+    }
+
+    # first, run a scan with no CDN exclusion
+    scan = bbot_scanner("evilcorp.com")
+    await scan.helpers._mock_dns(dns_mock)
+
+    from bbot.modules.base import BaseModule
+
+    class DummyModule(BaseModule):
+        watched_events = ["DNS_NAME"]
+
+        async def handle_event(self, event):
+            if event.type == "DNS_NAME" and event.data == "evilcorp.com":
+                await self.emit_event("www.evilcorp.com", "DNS_NAME", parent=event, tags=["cdn-cloudflare"])
+            if event.type == "DNS_NAME" and event.data == "www.evilcorp.com":
+                await self.emit_event("www.evilcorp.com:80", "OPEN_TCP_PORT", parent=event, tags=["cdn-cloudflare"])
+                await self.emit_event("www.evilcorp.com:443", "OPEN_TCP_PORT", parent=event, tags=["cdn-cloudflare"])
+                await self.emit_event("www.evilcorp.com:8080", "OPEN_TCP_PORT", parent=event, tags=["cdn-cloudflare"])
+
+    dummy = DummyModule(scan=scan)
+    await scan._prep()
+    scan.modules["dummy"] = dummy
+    events = [e async for e in scan.async_start() if e.type in ("DNS_NAME", "OPEN_TCP_PORT")]
+    assert set(e.data for e in events) == {
+        "evilcorp.com",
+        "www.evilcorp.com",
+        "www.evilcorp.com:80",
+        "www.evilcorp.com:443",
+        "www.evilcorp.com:8080",
+    }
+
+    monkeypatch.setattr("sys.argv", ["bbot", "-t", "evilcorp.com", "--exclude-cdn"])
+
+    # then run a scan with --exclude-cdn enabled
+    preset = Preset("evilcorp.com")
+    preset.parse_args()
+    assert preset.bake().to_yaml() == "modules:\n- portfilter\n"
+    scan = bbot_scanner("evilcorp.com", preset=preset)
+    await scan.helpers._mock_dns(dns_mock)
+    dummy = DummyModule(scan=scan)
+    await scan._prep()
+    scan.modules["dummy"] = dummy
+    events = [e async for e in scan.async_start() if e.type in ("DNS_NAME", "OPEN_TCP_PORT")]
+    assert set(e.data for e in events) == {
+        "evilcorp.com",
+        "www.evilcorp.com",
+        "www.evilcorp.com:80",
+        "www.evilcorp.com:443",
+    }

bbot/test/test_step_1/test_web.py

@@ -263,7 +263,7 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock):
     finally:
         await agen.aclose()
     assert not results
-    agen = module.api_page_iter(template_url, json=False)
+    agen = module.api_page_iter(template_url, _json=False)
     try:
         async for result in agen:
             if result and result.text.startswith("page"):
@@ -407,9 +407,9 @@ async def test_http_proxy(bbot_scanner, bbot_httpserver, proxy_server):
 
     r = await scan.helpers.request(url)
 
-    assert (
-        len(proxy_server.RequestHandlerClass.urls) == 1
-    ), f"Request to {url} did not go through proxy {proxy_address}"
+    assert len(proxy_server.RequestHandlerClass.urls) == 1, (
+        f"Request to {url} did not go through proxy {proxy_address}"
+    )
     visited_url = proxy_server.RequestHandlerClass.urls[0]
     assert visited_url.endswith(endpoint), f"There was a problem with request to {url}: {visited_url}"
     assert r.status_code == 200 and r.text == "test_http_proxy_yep"