bbot 2.1.0.5004rc0__py3-none-any.whl → 2.1.0.5028rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.0.5004rc"
+__version__ = "v2.1.0.5028rc"
 
 from .scanner import Scanner, Preset

bbot/core/event/base.py

@@ -25,6 +25,7 @@ from bbot.core.helpers import (
     is_domain,
     is_subdomain,
     is_ip,
+    is_ip_type,
     is_ptr,
     is_uri,
     url_depth,
@@ -157,7 +158,7 @@ class BaseEvent:
         Raises:
             ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
-        self.uuid = uuid.uuid4()
+        self._uuid = uuid.uuid4()
         self._id = None
         self._hash = None
         self._data = None
@@ -352,6 +353,12 @@ class BaseEvent:
                 return 80
         return self._port
 
+    @property
+    def netloc(self):
+        if self.host and is_ip_type(self.host, network=False):
+            return make_netloc(self.host, self.port)
+        return None
+
     @property
     def host_stem(self):
         """
@@ -449,6 +456,13 @@ class BaseEvent:
             self._id = f"{self.type}:{self.data_hash.hex()}"
         return self._id
 
+    @property
+    def uuid(self):
+        """
+        A universally unique identifier for the event
+        """
+        return f"{self.type}:{self._uuid}"
+
     @property
     def data_hash(self):
         """
@@ -741,7 +755,7 @@ class BaseEvent:
         """
         j = dict()
         # type, ID, scope description
-        for i in ("type", "id", "uuid", "scope_description"):
+        for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
             if v:
                 j.update({i: str(v)})
@@ -760,6 +774,8 @@ class BaseEvent:
             j["host"] = str(self.host)
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
             j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        if isinstance(self.port, int):
+            j["port"] = self.port
         # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
@@ -1709,7 +1725,7 @@ def event_from_json(j, siem_friendly=False):
         event = make_event(**kwargs)
         event_uuid = j.get("uuid", None)
         if event_uuid is not None:
-            event.uuid = uuid.UUID(event_uuid)
+            event._uuid = uuid.UUID(event_uuid.split(":")[-1])
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
@@ -1721,7 +1737,8 @@ def event_from_json(j, siem_friendly=False):
             event._parent_id = parent_id
         parent_uuid = j.get("parent_uuid", None)
         if parent_uuid is not None:
-            event._parent_uuid = uuid.UUID(parent_uuid)
+            parent_type, parent_uuid = parent_uuid.split(":", 1)
+            event._parent_uuid = parent_type + ":" + str(uuid.UUID(parent_uuid))
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")

bbot/core/helpers/misc.py

@@ -621,12 +621,13 @@ def is_ip(d, version=None):
     return False
 
 
-def is_ip_type(i):
+def is_ip_type(i, network=None):
     """
     Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module.
 
     Args:
         i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check.
+        network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False.
 
     Returns:
         bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise.
@@ -639,6 +640,12 @@ def is_ip_type(i):
         >>> is_ip_type("192.168.1.0/24")
         False
     """
+    if network is not None:
+        is_network = ipaddress._BaseNetwork in i.__class__.__mro__
+        if network:
+            return is_network
+        else:
+            return not is_network
     return ipaddress._IPAddressBase in i.__class__.__mro__
 
 
@@ -1260,7 +1267,7 @@ def gen_numbers(n, padding=2):
     return results
 
 
-def make_netloc(host, port):
+def make_netloc(host, port=None):
     """Constructs a network location string from a given host and port.
 
     Args:
@@ -1289,7 +1296,7 @@ def make_netloc(host, port):
     if is_ip(host, version=6):
         host = f"[{host}]"
     if port is None:
-        return host
+        return str(host)
     return f"{host}:{port}"
 
 

bbot/core/helpers/regexes.py

@@ -54,6 +54,9 @@ ptr_regex = re.compile(_ptr_regex)
 # uuid regex
 _uuid_regex = r"[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
 uuid_regex = re.compile(_uuid_regex, re.I)
+# event uuid regex
+_event_uuid_regex = r"[0-9A-Z_]+:[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
+event_uuid_regex = re.compile(_event_uuid_regex, re.I)
 
 _open_port_regexes = (
     _dns_name_regex + r":[0-9]{1,5}",

bbot/modules/base.py

@@ -984,8 +984,11 @@ class BaseModule:
     def _outgoing_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
+
+        We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules)
+        to inadvertently suppress a non-internal event.
         """
-        return hash((event, self.name))
+        return hash((event, self.name, event.internal, event.always_emit))
 
     def get_per_host_hash(self, event):
         """

bbot/modules/generic_ssrf.py

@@ -137,10 +137,10 @@ class Generic_XXE(BaseSubmodule):
         post_body = f"""<?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE foo [
 <!ELEMENT foo ANY >
-<!ENTITY % {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
+<!ENTITY {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
 ]>
 <foo>&{rand_entity};</foo>"""
-        test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
+        test_url = event.parsed_url.geturl()
         r = await self.parent_module.helpers.curl(
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )

bbot/modules/internal/dnsresolve.py

@@ -16,12 +16,6 @@ class DNSResolve(BaseInterceptModule):
         _name = "host"
         _type = "internal"
 
-        def _outgoing_dedup_hash(self, event):
-            # this exists to ensure a second, more interesting host isn't passed up
-            # because its ugly cousin spent its one dedup token before it arrived
-            # by removing those race conditions, this makes for more consistent results
-            return hash((event, self.name, event.always_emit))
-
     @property
     def module_threads(self):
         return self.dns_config.get("threads", 25)

bbot/modules/internal/excavate.py

@@ -876,6 +876,8 @@ class excavate(BaseInternalModule, BaseInterceptModule):
         yara_rules_combined = "\n".join(self.yara_rules_dict.values())
         try:
             self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules")
+            for rule_name, rule_content in self.yara_rules_dict.items():
+                self.debug(f"  - {rule_name}")
             self.yara_rules = yara.compile(source=yara_rules_combined)
         except yara.SyntaxError as e:
             self.debug(yara_rules_combined)

bbot/modules/internal/speculate.py

@@ -112,15 +112,15 @@ class speculate(BaseInternalModule):
         speculate_open_ports = self.emit_open_ports and event_in_scope_distance
 
         # URL --> OPEN_TCP_PORT
-        if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
+        event_is_url = event.type == "URL"
+        if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
             # only speculate port from a URL if it wouldn't be speculated naturally from the host
             if event.host and (event.port not in self.ports or not speculate_open_ports):
                 await self.emit_event(
                     self.helpers.make_netloc(event.host, event.port),
                     "OPEN_TCP_PORT",
                     parent=event,
-                    internal=True,
-                    quick=(event.type == "URL"),
+                    internal=not event_is_url,  # if the URL is verified, the port is definitely open
                     context=f"speculated {{event.type}} from {event.type}: {{event.data}}",
                 )
 
@@ -169,7 +169,6 @@ class speculate(BaseInternalModule):
                         "OPEN_TCP_PORT",
                         parent=event,
                         internal=True,
-                        quick=True,
                         context="speculated {event.type}: {event.data}",
                     )
 

bbot/modules/shodan_dns.py

@@ -16,13 +16,11 @@ class shodan_dns(shodan):
 
     base_url = "https://api.shodan.io"
 
-    async def request_url(self, query):
-        url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}"
-        response = await self.api_request(url)
-        return response
+    async def handle_event(self, event):
+        await self.handle_event_paginated(event)
 
-    def parse_results(self, r, query):
-        json = r.json()
-        if json:
-            for hostname in json.get("subdomains", []):
-                yield f"{hostname}.{query}"
+    def make_url(self, query):
+        return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
+
+    def parse_results(self, json, query):
+        return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

bbot/modules/templates/subdomain_enum.py

@@ -31,6 +31,11 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    # how many results to request per API call
+    page_size = 100
+    # arguments to pass to api_page_iter
+    api_page_iter_kwargs = {}
+
     @property
     def source_pretty_name(self):
         return f"{self.__class__.__name__} API"
@@ -61,10 +66,31 @@ class subdomain_enum(BaseModule):
                             context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
                         )
 
+    async def handle_event_paginated(self, event):
+        query = self.make_query(event)
+        async for result_batch in self.query_paginated(query):
+            for hostname in set(result_batch):
+                try:
+                    hostname = self.helpers.validators.validate_host(hostname)
+                except ValueError as e:
+                    self.verbose(e)
+                    continue
+                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
+                    await self.emit_event(
+                        hostname,
+                        "DNS_NAME",
+                        event,
+                        abort_if=self.abort_if,
+                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
+
     async def request_url(self, query):
-        url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
+        url = self.make_url(query)
         return await self.api_request(url)
 
+    def make_url(self, query):
+        return f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
+
     def make_query(self, event):
         query = event.data
         parents = list(self.helpers.domain_parents(event.data))
@@ -86,11 +112,11 @@ class subdomain_enum(BaseModule):
             for hostname in json:
                 yield hostname
 
-    async def query(self, query, parse_fn=None, request_fn=None):
-        if parse_fn is None:
-            parse_fn = self.parse_results
+    async def query(self, query, request_fn=None, parse_fn=None):
         if request_fn is None:
             request_fn = self.request_url
+        if parse_fn is None:
+            parse_fn = self.parse_results
         try:
             response = await request_fn(query)
             if response is None:
@@ -113,6 +139,19 @@ class subdomain_enum(BaseModule):
         except Exception as e:
             self.info(f"Error retrieving results for {query}: {e}", trace=True)
 
+    async def query_paginated(self, query):
+        url = self.make_url(query)
+        agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
+        try:
+            async for response in agen:
+                subdomains = self.parse_results(response, query)
+                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
+                if not subdomains:
+                    break
+                yield subdomains
+        finally:
+            agen.aclose()
+
     async def _is_wildcard(self, query):
         rdtypes = ("A", "AAAA", "CNAME")
         if self.helpers.is_dns_name(query):

bbot/modules/trickest.py

@@ -28,39 +28,15 @@ class Trickest(subdomain_enum_apikey):
         return url, kwargs
 
     async def handle_event(self, event):
-        query = self.make_query(event)
-        async for result_batch in self.query(query):
-            for hostname in set(result_batch):
-                try:
-                    hostname = self.helpers.validators.validate_host(hostname)
-                except ValueError as e:
-                    self.verbose(e)
-                    continue
-                if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                    await self.emit_event(
-                        hostname,
-                        "DNS_NAME",
-                        event,
-                        abort_if=self.abort_if,
-                        context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
-                    )
+        await self.handle_event_paginated(event)
 
-    async def query(self, query):
+    def make_url(self, query):
         url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22"
         url += f"&dataset_id={self.dataset_id}"
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
-        agen = self.api_page_iter(url, page_size=self.page_size)
-        try:
-            async for response in agen:
-                subdomains = self.parse_results(response)
-                self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
-                if not subdomains:
-                    break
-                yield subdomains
-        finally:
-            agen.aclose()
+        return url
 
-    def parse_results(self, j):
+    def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

bbot/modules/virustotal.py

@@ -15,6 +15,10 @@ class virustotal(subdomain_enum_apikey):
     options_desc = {"api_key": "VirusTotal API Key"}
 
     base_url = "https://www.virustotal.com/api/v3"
+    api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")}
+
+    def make_url(self, query):
+        return f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
 
     def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["x-apikey"] = self.api_key
@@ -28,17 +32,3 @@ class virustotal(subdomain_enum_apikey):
             if match.endswith(query):
                 results.add(match)
         return results
-
-    async def query(self, query):
-        results = set()
-        url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
-        agen = self.api_page_iter(url, json=False, next_key=lambda r: r.json().get("links", {}).get("next", ""))
-        try:
-            async for response in agen:
-                r = self.parse_results(response, query)
-                if not r:
-                    break
-                results.update(r)
-        finally:
-            agen.aclose()
-        return results

bbot/scanner/preset/preset.py

@@ -844,8 +844,6 @@ class Preset:
 
         if module in self.exclude_modules:
             reason = "the module has been excluded"
-            if raise_error:
-                raise ValidationError(f'Unable to add {module_type} module "{module}" because {reason}')
             return False, reason, {}
 
         module_flags = preloaded.get("flags", [])

bbot/scanner/scanner.py

@@ -130,20 +130,22 @@ class Scanner:
         else:
             self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}"
 
-        preset = kwargs.pop("preset", None)
+        custom_preset = kwargs.pop("preset", None)
         kwargs["_log"] = True
 
         from .preset import Preset
 
-        if preset is None:
-            preset = Preset(*targets, **kwargs)
-        else:
-            if not isinstance(preset, Preset):
-                raise ValidationError(f'Preset must be of type Preset, not "{type(preset).__name__}"')
-        self.preset = preset.bake(self)
+        base_preset = Preset(*targets, **kwargs)
+
+        if custom_preset is not None:
+            if not isinstance(custom_preset, Preset):
+                raise ValidationError(f'Preset must be of type Preset, not "{type(custom_preset).__name__}"')
+            base_preset.merge(custom_preset)
+
+        self.preset = base_preset.bake(self)
 
         # scan name
-        if preset.scan_name is None:
+        if self.preset.scan_name is None:
             tries = 0
             while 1:
                 if tries > 5:
@@ -158,7 +160,7 @@ class Scanner:
                     break
                 tries += 1
         else:
-            scan_name = str(preset.scan_name)
+            scan_name = str(self.preset.scan_name)
         self.name = scan_name
 
         # make sure the preset has a description
@@ -166,8 +168,8 @@ class Scanner:
             self.preset.description = self.name
 
         # scan output dir
-        if preset.output_dir is not None:
-            self.home = Path(preset.output_dir).resolve() / self.name
+        if self.preset.output_dir is not None:
+            self.home = Path(self.preset.output_dir).resolve() / self.name
         else:
             self.home = self.preset.bbot_home / "scans" / self.name
 

bbot/test/test_step_1/test_cli.py

@@ -351,14 +351,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     result = await cli._main()
     assert result == True
 
-    # enable and exclude the same module
-    caplog.clear()
-    assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-m", "ffuf_shortnames", "-em", "ffuf_shortnames"])
-    result = await cli._main()
-    assert result == None
-    assert 'Unable to add scan module "ffuf_shortnames" because the module has been excluded' in caplog.text
-
     # require flags
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "-rf", "passive"])
     result = await cli._main()

bbot/test/test_step_1/test_events.py

@@ -4,7 +4,7 @@ import ipaddress
 
 from ..bbot_fixtures import *
 from bbot.scanner import Scanner
-from bbot.core.helpers.regexes import uuid_regex
+from bbot.core.helpers.regexes import event_uuid_regex
 
 
 @pytest.mark.asyncio
@@ -14,10 +14,19 @@ async def test_events(events, helpers):
     await scan._prep()
 
     assert events.ipv4.type == "IP_ADDRESS"
+    assert events.ipv4.netloc == "8.8.8.8"
+    assert events.ipv4.port is None
     assert events.ipv6.type == "IP_ADDRESS"
+    assert events.ipv6.netloc == "[2001:4860:4860::8888]"
+    assert events.ipv6.port is None
+    assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443"
     assert events.netv4.type == "IP_RANGE"
+    assert events.netv4.netloc is None
+    assert "netloc" not in events.netv4.json()
     assert events.netv6.type == "IP_RANGE"
     assert events.domain.type == "DNS_NAME"
+    assert events.domain.netloc == "publicapis.org"
+    assert events.domain.port is None
     assert "domain" in events.domain.tags
     assert events.subdomain.type == "DNS_NAME"
     assert "subdomain" in events.subdomain.tags
@@ -67,8 +76,14 @@ async def test_events(events, helpers):
     assert not events.netv6 in events.domain
     assert events.emoji not in events.domain
     assert events.domain not in events.emoji
-    assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
-    assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True)
+    dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True)
+    for e in (open_port_event, dns_event):
+        assert "evilcorp.com" == e
+        assert e.netloc == "evilcorp.com"
+        assert e.json()["netloc"] == "evilcorp.com"
+        assert e.port is None
+        assert "port" not in e.json()
 
     # url tests
     assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True)
@@ -78,8 +93,14 @@ async def test_events(events, helpers):
     assert "api.publicapis.org:443" in events.url_unverified
     assert "publicapis.org" not in events.url_unverified
     assert events.ipv4_url_unverified in events.ipv4
+    assert events.ipv4_url_unverified.netloc == "8.8.8.8:443"
+    assert events.ipv4_url_unverified.port == 443
+    assert events.ipv4_url_unverified.json()["port"] == 443
     assert events.ipv4_url_unverified in events.netv4
     assert events.ipv6_url_unverified in events.ipv6
+    assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443"
+    assert events.ipv6_url_unverified.port == 443
+    assert events.ipv6_url_unverified.json()["port"] == 443
     assert events.ipv6_url_unverified in events.netv6
     assert events.emoji not in events.url_unverified
     assert events.emoji not in events.ipv6_url_unverified
@@ -193,6 +214,8 @@ async def test_events(events, helpers):
 
     org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
+    assert org_stub_1.netloc == None
+    assert "netloc" not in org_stub_1.json()
     org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 
@@ -420,11 +443,17 @@ async def test_events(events, helpers):
     parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
 
     event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context")
+    assert hasattr(event1, "_uuid")
     assert hasattr(event1, "uuid")
-    assert isinstance(event1.uuid, uuid.UUID)
+    assert isinstance(event1._uuid, uuid.UUID)
+    assert isinstance(event1.uuid, str)
+    assert event1.uuid == f"{event1.type}:{event1._uuid}"
     event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context")
+    assert hasattr(event2, "_uuid")
     assert hasattr(event2, "uuid")
-    assert isinstance(event2.uuid, uuid.UUID)
+    assert isinstance(event2._uuid, uuid.UUID)
+    assert isinstance(event2.uuid, str)
+    assert event2.uuid == f"{event2.type}:{event2._uuid}"
     # ids should match because the event type + data is the same
     assert event1.id == event2.id
     # but uuids should be unique!
@@ -447,7 +476,7 @@ async def test_events(events, helpers):
     assert db_event.discovery_context == "test context"
     assert db_event.discovery_path == ["test context"]
     assert len(db_event.parent_chain) == 1
-    assert all([uuid_regex.match(u) for u in db_event.parent_chain])
+    assert all([event_uuid_regex.match(u) for u in db_event.parent_chain])
     assert db_event.parent_chain[0] == str(db_event.uuid)
     assert db_event.parent.uuid == scan.root_event.uuid
     assert db_event.parent_uuid == scan.root_event.uuid
@@ -467,7 +496,7 @@ async def test_events(events, helpers):
     assert json_event["parent_chain"] == db_event.parent_chain
     assert json_event["parent_chain"][0] == str(db_event.uuid)
     reconstituted_event = event_from_json(json_event)
-    assert isinstance(reconstituted_event.uuid, uuid.UUID)
+    assert isinstance(reconstituted_event._uuid, uuid.UUID)
     assert str(reconstituted_event.uuid) == json_event["uuid"]
     assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"]
     assert reconstituted_event.uuid == db_event.uuid

bbot/test/test_step_1/test_helpers.py

@@ -94,6 +94,15 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     ]
     assert helpers.is_ip("127.0.0.1")
     assert not helpers.is_ip("127.0.0.0.1")
+
+    assert not helpers.is_ip_type("127.0.0.1")
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"))
+    assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True)
+    assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=False)
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"))
+    assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=True)
+    assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False)
+
     assert helpers.is_dns_name("evilcorp.com")
     assert helpers.is_dns_name("evilcorp")
     assert not helpers.is_dns_name("evilcorp", include_local=False)
@@ -212,8 +221,12 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
 
     ipv4_netloc = helpers.make_netloc("192.168.1.1", 80)
     assert ipv4_netloc == "192.168.1.1:80"
-    ipv6_netloc = helpers.make_netloc("dead::beef", "443")
-    assert ipv6_netloc == "[dead::beef]:443"
+    assert helpers.make_netloc("192.168.1.1") == "192.168.1.1"
+    assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1"), None) == "192.168.1.1"
+    assert helpers.make_netloc("dead::beef", "443") == "[dead::beef]:443"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), 443) == "[dead::beef]:443"
+    assert helpers.make_netloc("dead::beef", None) == "[dead::beef]"
+    assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), None) == "[dead::beef]"
 
     assert helpers.get_file_extension("https://evilcorp.com/evilcorp.com/test/asdf.TXT") == "txt"
     assert helpers.get_file_extension("/etc/conf/test.tar.gz") == "gz"

bbot/test/test_step_1/test_manager_scope_accuracy.py

@@ -575,7 +575,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
         },
     )
 
-    assert len(events) == 10
+    assert len(events) == 12
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"])
     assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -589,9 +589,9 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"])
     assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"])
-    assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"])
-    assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
     assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"])
     assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
@@ -603,7 +603,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"])
 
-    assert len(all_events) == 29
+    assert len(all_events) == 31
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -618,8 +618,10 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == True and e.scope_distance == 0])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == True and e.scope_distance == 0])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
@@ -660,7 +662,7 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 10
+        assert len(_graph_output_events) == 12
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"])
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
@@ -674,9 +676,9 @@ async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_
         assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"])
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33"])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"])
-        assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"])
-        assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"])
         assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"])
         assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])

bbot/test/test_step_1/test_modules_basic.py

@@ -380,15 +380,14 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
     scan.modules["dummy"] = dummy(scan)
     events = [e async for e in scan.async_start()]
 
-    assert len(events) == 10
-    for e in events:
-        log.critical(e)
+    assert len(events) == 11
     assert 2 == len([e for e in events if e.type == "SCAN"])
     assert 4 == len([e for e in events if e.type == "DNS_NAME"])
     # one from target and one from speculate
     assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.com"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.com:443"])
     assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp"])
     assert 1 == len([e for e in events if e.type == "FINDING"])
     assert 1 == len([e for e in events if e.type == "URL_UNVERIFIED"])
@@ -397,6 +396,7 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
         "SCAN": 1,
         "DNS_NAME": 4,
         "URL": 1,
+        "OPEN_TCP_PORT": 1,
         "ORG_STUB": 1,
         "URL_UNVERIFIED": 1,
         "FINDING": 1,
@@ -431,16 +431,17 @@ async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, mo
     assert python_stats.consumed == {
         "DNS_NAME": 4,
         "FINDING": 1,
+        "OPEN_TCP_PORT": 1,
         "ORG_STUB": 1,
         "SCAN": 1,
         "URL": 1,
         "URL_UNVERIFIED": 1,
     }
-    assert python_stats.consumed_total == 9
+    assert python_stats.consumed_total == 10
 
     speculate_stats = scan.stats.module_stats["speculate"]
-    assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1}
-    assert speculate_stats.produced_total == 3
+    assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1, "OPEN_TCP_PORT": 1}
+    assert speculate_stats.produced_total == 4
     assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 3, "URL_UNVERIFIED": 1, "IP_ADDRESS": 3}
     assert speculate_stats.consumed_total == 8
 

bbot/test/test_step_1/test_presets.py

@@ -169,6 +169,11 @@ exclude_flags:
 
 def test_preset_scope():
 
+    # test target merging
+    scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
+    assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4", "evilcorp.com"}
+    assert set([e.data for e in scan.target]) == {"1.2.3.4", "evilcorp.com"}
+
     blank_preset = Preset()
     blank_preset = blank_preset.bake()
     assert not blank_preset.target
@@ -527,9 +532,22 @@ def test_preset_module_resolution(clean_default_config):
     assert set(preset.scan_modules) == {"wayback"}
 
     # modules + module exclusions
-    with pytest.raises(ValidationError) as error:
-        preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake()
-    assert str(error.value) == 'Unable to add scan module "sslcert" because the module has been excluded'
+    preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake()
+    baked_preset = preset.bake()
+    assert baked_preset.modules == {
+        "wayback",
+        "cloudcheck",
+        "python",
+        "json",
+        "speculate",
+        "dnsresolve",
+        "aggregate",
+        "excavate",
+        "txt",
+        "httpx",
+        "csv",
+        "wappalyzer",
+    }
 
 
 @pytest.mark.asyncio

bbot/test/test_step_2/module_tests/test_module_shodan_dns.py

@@ -9,13 +9,32 @@ class TestShodan_DNS(ModuleTestBase):
             url="https://api.shodan.io/api-info?key=asdf",
         )
         module_test.httpx_mock.add_response(
-            url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf",
+            url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=1",
             json={
                 "subdomains": [
                     "asdf",
                 ],
             },
         )
+        module_test.httpx_mock.add_response(
+            url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=2",
+            json={
+                "subdomains": [
+                    "www",
+                ],
+            },
+        )
+        await module_test.mock_dns(
+            {
+                "blacklanternsecurity.com": {
+                    "A": ["127.0.0.11"],
+                },
+                "www.blacklanternsecurity.com": {"A": ["127.0.0.22"]},
+                "asdf.blacklanternsecurity.com": {"A": ["127.0.0.33"]},
+            }
+        )
 
     def check(self, module_test, events):
+        assert len([e for e in events if e.type == "DNS_NAME"]) == 3, "Failed to detect both subdomains"
         assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
+        assert any(e.data == "www.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"

{bbot-2.1.0.5004rc0.dist-info → bbot-2.1.0.5028rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.0.5004rc0
+Version: 2.1.0.5028rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.1.0.5004rc0.dist-info → bbot-2.1.0.5028rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=2MX0wg8qVd4uv2p2QWKxK-Lgomc0rk8etHVfyVf9jFo,130
+bbot/__init__.py,sha256=uGNlVZLVuhzc5H4GbnGYmOTRrPsfNQvnztWI-cJlTjE,130
 bbot/cli.py,sha256=7S3a4eB-Dl8yodc5WC-927Z30CNlLl9EXimGvIVypJo,10434
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -7,7 +7,7 @@ bbot/core/config/logger.py,sha256=C9txmKQtqSFnWU1AapwFS9cZgDBtNZQh4io_13d3AxA,95
 bbot/core/core.py,sha256=twd7-fiaaxzgcWTPwT1zbSWfAa_gHHfl7gAFvLYvFYg,6358
 bbot/core/engine.py,sha256=wGopKa2GNs61r16Pr_xtp6Si9AT6I-lE83iWhEgtxwA,29290
 bbot/core/event/__init__.py,sha256=8ut88ZUg0kbtWkOx2j3XzNr_3kTfgoM-3UdiWHFA_ag,56
-bbot/core/event/base.py,sha256=ZAz5jEd7sEX8Bl0EVdCpdOD7jXNjcdlqOJ83GpckCA8,58963
+bbot/core/event/base.py,sha256=YHWevdDo5sHIIMKz_9TkWFYGTVtoD3fv957TpMO-6DQ,59498
 bbot/core/event/helpers.py,sha256=PUN4Trq5_wpKVuhmwUQWAr40apgMXhJ9Gz-VfZ0j3lA,1554
 bbot/core/flags.py,sha256=Ltvm8Bc4D65I55HuU5bzyjO1R3yMDNpVmreGU83ZBXE,1266
 bbot/core/helpers/__init__.py,sha256=0UNwcZjNsX41hbHdo3yZPuARkYWch-okI68DScexve4,86
@@ -28,13 +28,13 @@ bbot/core/helpers/dns/mock.py,sha256=Ztkp2aOuwDJ0NTQSlAk2H0s3Stx9wIM22Qm3VtqWMKM
 bbot/core/helpers/files.py,sha256=GqrwNGJljUvGSzaOW5-Y357hkt7j88dOYbzQxJGsdTc,5787
 bbot/core/helpers/helper.py,sha256=3O96peNBvSkaJosft8w9-nKjCscEdykTayGcUlHRqLw,8394
 bbot/core/helpers/interactsh.py,sha256=Q9IHUzH-T7e1s4YTHevHe-VJj1Mokv0EHY16UZJdl8M,12627
-bbot/core/helpers/misc.py,sha256=jhQsCAESXin5eBOQ5n8Uq5UhB4lcBjuBv3SEEaCaOEQ,86526
+bbot/core/helpers/misc.py,sha256=rvfZmm8UHCChmbMorjPMybaCZTkERrKZhxvY9S4dVPc,86873
 bbot/core/helpers/names_generator.py,sha256=Sj_Q-7KQyElEpalzlUadSwaniESqrIVVEle9ycPIiho,10322
 bbot/core/helpers/ntlm.py,sha256=P2Xj4-GPos2iAzw4dfk0FJp6oGyycGhu2x6sLDVjYjs,2573
 bbot/core/helpers/process.py,sha256=6D9_LYZrhQ0Jb7Rn58rWMafmAZn7rVVA2LqMKwpR_xg,2271
 bbot/core/helpers/ratelimiter.py,sha256=K8qFIyJPJtfdb9kSW6_lL6ahWqxR2uWyCBkDlg6uJgo,1990
 bbot/core/helpers/regex.py,sha256=XURaY6ijpOYYU9lzWMAKg12G1VFtGJjlJl07_eN1xxk,4170
-bbot/core/helpers/regexes.py,sha256=BlbfziycPctQIvVSpD9dIz6HGyeNqRFnktCbwDRA56A,5686
+bbot/core/helpers/regexes.py,sha256=kz3y3hyEJvsnF7-4NrkrHTwwhkbRqEYi_wqA_rmZlgQ,5859
 bbot/core/helpers/url.py,sha256=1NDrvirODzzD6Mcssu-4WDNerMeMdekHCFzhRCS0m3g,5947
 bbot/core/helpers/validators.py,sha256=cglRDybXiFDh2vKwqjqv7Ruu1TWmSzPcbuQ3Rvky5JU,9696
 bbot/core/helpers/web/__init__.py,sha256=pIEkL3DhjaGTSmZ7D3yKKYwWpntoLRILekV2wWsbsws,27
@@ -58,7 +58,7 @@ bbot/modules/baddns.py,sha256=FFGpJ0AdcpYBh_i8VMYOUj6C_PhzkFls263DoVmUXGY,6376
 bbot/modules/baddns_direct.py,sha256=_Z_snPDU5VqF3jb2CAoX5Sb94GnhYVP22JIK5TdpF_Y,3820
 bbot/modules/baddns_zone.py,sha256=bTpYzQYLt4ZJo_Jan5K4NsYL7O1PMVQvJHmYPy6QyvY,1035
 bbot/modules/badsecrets.py,sha256=WMeIhjafD63sXShYGtbAk2kw12PgxMIZh8EMbYg7x20,5108
-bbot/modules/base.py,sha256=HiffBnsNcCjKIgp_IHg82NAsXDXlnvoPs2wvx6HBZ6k,70893
+bbot/modules/base.py,sha256=-nPgoPjMwcveZ81fzb4TwhbtBFqNYorj-zD3TG91ZPg,71115
 bbot/modules/bevigil.py,sha256=EFA4N_aJDF20KjrzbzdkHaTL1GfOF6ohBMlNIEJne6s,2850
 bbot/modules/binaryedge.py,sha256=W6VMbV7-tVAduUNbcmS6uLLJixb3sDJmqC9m4IGB6Yg,1391
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
@@ -96,7 +96,7 @@ bbot/modules/ffuf_shortnames.py,sha256=9Kh0kJsw7XXpXmCkiB5eAhG4h9rSo8Y-mB3p0EDa_
 bbot/modules/filedownload.py,sha256=1prC84wAQO-W1HstitKPQ0-eYEApjzFn3RHFa9oaqLc,8185
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=R70VpNZezUOeCOuYiYaG3oJCIOjOjB6lMPaVJ-xPVr0,1305
-bbot/modules/generic_ssrf.py,sha256=ZkaZuQE_sKlHH0uBUtJ45UKcGU91tdq8-n3ypneYdqA,7980
+bbot/modules/generic_ssrf.py,sha256=FZ7XZ2RbBk5PVwa7wHasSkEsatUFWvCH70ZvPbg-EQA,7946
 bbot/modules/git.py,sha256=CMDarsmBemZEzZSeQTzB70XD8IRdwdG39zXpwDdgZbw,1383
 bbot/modules/git_clone.py,sha256=XFZXx0k97EMY3E5PZzdNvqQzZddOfRMaVp5ol2gk11s,2468
 bbot/modules/github_codesearch.py,sha256=VbzSsnJKL1SWSl6rB88qWl26bAgszM7Gbo4HCOl6XUU,3540
@@ -115,9 +115,9 @@ bbot/modules/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 bbot/modules/internal/aggregate.py,sha256=csWYIt2fUp9K_CRxP3bndUMIjpNIh8rmBubp5Fr1-nc,395
 bbot/modules/internal/base.py,sha256=BXO4Hc7XKaAOaLzolF3krJX1KibPxtek2GTQUgnCHk0,387
 bbot/modules/internal/cloudcheck.py,sha256=kkD5DC5Y7EPCiVTdEspZktkflArU3unx--9ciBikxHo,4099
-bbot/modules/internal/dnsresolve.py,sha256=IYIbtwvHzsVHsnBkgAmRUNMc3nnG5DuH4tBnq4Ja0FU,14923
-bbot/modules/internal/excavate.py,sha256=eQNorIZz0NHnsRYPt1MOR2Oj4Uc6---q4OSfr3ErNC4,51118
-bbot/modules/internal/speculate.py,sha256=diAI1_RGdcoKCBTiwsplReHNxL7S85OmBc8hsvr2yuA,9832
+bbot/modules/internal/dnsresolve.py,sha256=rmXBb6r8MNYrHlnTbES1ok5M9q3JD3Oj6AOx6d0Bawc,14558
+bbot/modules/internal/excavate.py,sha256=1kPiaUlpJVT_xgPgmzqhpf04uanIe9ZayQsWANbzhd4,51238
+bbot/modules/internal/speculate.py,sha256=wP75lpdnxhoGiARmSdIlvPyYHQKx1Tu6oCu7_sKV9Qs,9850
 bbot/modules/internetdb.py,sha256=Edg0Z84dH8dPTZMd7RlzvYBYNq8JHs_ns_ldnFxwRKo,5415
 bbot/modules/ip2location.py,sha256=yGivX9fzvwvLpnqmYCP2a8SPjTarzrZxfRluog-nkME,2628
 bbot/modules/ipneighbor.py,sha256=Gr-HGtyZRDp_fPjpw-Mq1al7ocFdiZbKsAoAit-EUlA,1591
@@ -162,7 +162,7 @@ bbot/modules/robots.py,sha256=G2_IvLcI24iMXJxii44H1z6m5KWe2NMIDtVMbQGaXDE,2172
 bbot/modules/secretsdb.py,sha256=MA6IKo5rOvC0Dzt-F4QVrSwLkcPRwWLd9FpzqYkc8u8,3082
 bbot/modules/securitytrails.py,sha256=j2ZcY_qO48mQquWwS3FOaHgZiomr2ETMGv28y5qtRZM,1088
 bbot/modules/securitytxt.py,sha256=-GuIelHW04256VlcgfyV4ylfo7oSbAwToxtV2Cfo2p0,4558
-bbot/modules/shodan_dns.py,sha256=G0Wt8Uure-oY63n0ctRtQNzSGD6YFo3UnBz8Fjk4jJY,876
+bbot/modules/shodan_dns.py,sha256=PJaeqGlTfhQfKgc_3fDmC4mLWsOb4BnRUpVD_h3Y8cU,836
 bbot/modules/sitedossier.py,sha256=0GOGMt0x3wro2FvP4ngNa2lcTmcjjq5x1SM8l5HG8Cs,2283
 bbot/modules/skymem.py,sha256=NApG68Eh22jMcV1H590SXvQIlLJY4JOfGUeNaOJaMKs,1928
 bbot/modules/smuggler.py,sha256=v8NCRgzd7wpEFZJUTAArG04bN8nNTGiHxYpGBapzi14,1580
@@ -175,15 +175,15 @@ bbot/modules/templates/bucket.py,sha256=x-c_iAeMILux6wRm0xkUUJkc2P69hYWS6DxqD7g5
 bbot/modules/templates/github.py,sha256=ENnDWpzzmZBsTisDx6Cg9V_NwJKyVyPIOpGAPktigdI,1455
 bbot/modules/templates/postman.py,sha256=oxwVusW2EdNotVX7xnnxCTnWtj3xNPbfs8aff9s4phs,614
 bbot/modules/templates/shodan.py,sha256=BfI0mNPbqkykGmjMtARhmCGKmk1uq7yTlZoPgzzJ040,1175
-bbot/modules/templates/subdomain_enum.py,sha256=W6UC3r2qkEryJAiwTz523G2STql3f8SDKGmlF79XQM0,6805
+bbot/modules/templates/subdomain_enum.py,sha256=lT5MZF66OuzsyFFrj20wKlsZflzL9MOkPjDIbN3o65o,8375
 bbot/modules/templates/webhook.py,sha256=MYhKWrNYrsfM0a4PR6yVotudLyyCwgmy2eI-l9LvpBs,3706
-bbot/modules/trickest.py,sha256=dPUeUsOy0saUcFlpOyPXDh3ESKL2LIqH_oQkuAR4ToE,2657
+bbot/modules/trickest.py,sha256=HfAzjnawxXd9ypi3gumDHqImE5-C7uwNugo8d_b9HT0,1544
 bbot/modules/trufflehog.py,sha256=KMFYbjKEyoNaJDoID0SoDvbCRPaDwalMD6H2lQaI1QE,8555
 bbot/modules/unstructured.py,sha256=si3_Y__A36QOBdkIUocVXCHrmUqM0E-JSnoOeRpELYE,5311
 bbot/modules/url_manipulation.py,sha256=BI-OhlzNzP5xvwzHphL4qdehc4NiEYnL2BNK-JoEm90,4322
 bbot/modules/urlscan.py,sha256=ajhiX2sj-zZDlKU1q5rE8JTzxioj1mDLqZ9PRSQCpAw,3741
 bbot/modules/viewdns.py,sha256=f0vwoLpua2Ovw1gcrjoafUdaAP9fi4bHgTUiDOe8iWg,2596
-bbot/modules/virustotal.py,sha256=hQZe1jgcC65fNHrYV98EMFQfi-j7qVyT4NAlfxsiEgc,1505
+bbot/modules/virustotal.py,sha256=GsGaVF05IMgSNOQtUx1B8UXL5JA1Bt8M6ZDWJiiEQ1k,1213
 bbot/modules/wafw00f.py,sha256=I-jEnHWxO4Ga72ukdeBlTGJB9xeucCT3lpDhhFaVyAk,2536
 bbot/modules/wappalyzer.py,sha256=LL5QeY5DeG7LdaFzZZU-LXaVlJ-sHzOwQLgFtxW3TNg,2176
 bbot/modules/wayback.py,sha256=9cxd_HfHgLp4AChzA8C0Zjd6DIJ7c3NsJ02W2oLIXuU,3257
@@ -212,8 +212,8 @@ bbot/scanner/preset/args.py,sha256=9Nmir2dHJWzN66m6N-mA0QEKiOgt8vWq23O8BG50eMA,1
 bbot/scanner/preset/conditions.py,sha256=hFL9cSIWGEsv2TfM5UGurf0c91cyaM8egb5IngBmIjA,1569
 bbot/scanner/preset/environ.py,sha256=-wbFk1YHpU8IJLKVw23Q3btQTICeX0iulURo7D673L0,4732
 bbot/scanner/preset/path.py,sha256=p9tZC7XcgZv2jXpbEJAg1lU2b4ZLX5COFnCxEUOXz2g,2234
-bbot/scanner/preset/preset.py,sha256=nhAPjV-9P-udAs_CK5oOVuCeQP_c-a6Nem0SaNF98Ss,40216
-bbot/scanner/scanner.py,sha256=UyTpW1qTvegugUd2MWwtAE9CsAOs7bBmM8kMSqY7uCo,53583
+bbot/scanner/preset/preset.py,sha256=7q6PB9LalIzHyb4eiMDVKE6CapWBCKVw7350M0fSiwM,40083
+bbot/scanner/scanner.py,sha256=62DKCjgV1uLxNAwpxjvE5h1uzQCxG-nzBxp1PBCSVKc,53674
 bbot/scanner/stats.py,sha256=re93sArKXZSiD0Owgqk2J3Kdvfm3RL4Y9Qy_VOcaVk8,3623
 bbot/scanner/target.py,sha256=X25gpgRv5HmqQjGADiSe6b8744yOkRhAGAvKKYbXnSI,19886
 bbot/scripts/docs.py,sha256=kg2CzovmUVGJx9hBZjAjUdE1hXeIwC7Ry3CyrnE8GL8,10782
@@ -228,20 +228,20 @@ bbot/test/test_output.ndjson,sha256=Jfor8nUJ3QTEwXxD6UULrFXM4zhP5wflWo_UNekM3V8,
 bbot/test/test_step_1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bbot/test/test_step_1/test__module__tests.py,sha256=RpD4yuVuQRgbbUkfuasxUlyoVxhTm6TeDyi87y_AaK0,1461
 bbot/test/test_step_1/test_bloom_filter.py,sha256=OpiZXsBX-I8QdTK0LqSYkGMDLA6vL_6t0wco9ypxxtQ,2114
-bbot/test/test_step_1/test_cli.py,sha256=vJ_tERHP3-7PukHO0cfL_gEIKjHhAdTe4dsVR2yDct0,25039
+bbot/test/test_step_1/test_cli.py,sha256=m4LyIiiedR41f5SpnlkxgE-f42fF7qoGcuthpYxQ_m8,24688
 bbot/test/test_step_1/test_command.py,sha256=5IeGV6TKB0xtFEsfsU_0mNrOmEdIQiQ3FHkUmsBNoOI,6485
 bbot/test/test_step_1/test_config.py,sha256=Q38hygpke2GDcv8OguVZuiSOnfDJxEMrRy20dN5Qsn0,887
 bbot/test/test_step_1/test_depsinstaller.py,sha256=zr9f-wJDotD1ZvKXGEuDRWzFYMAYBI6209mI_PWPtTQ,703
 bbot/test/test_step_1/test_dns.py,sha256=YZtSbja-Z76KC9MWBieRExolVWHm0WqssL0WHUpUiC8,30932
 bbot/test/test_step_1/test_docs.py,sha256=YWVGNRfzcrvDmFekX0Cq9gutQplsqvhKTpZ0XK4tWvo,82
 bbot/test/test_step_1/test_engine.py,sha256=Bfid3-D9ziN93w4vym97tFEn_l2Iof08wjITTv_lAZw,4269
-bbot/test/test_step_1/test_events.py,sha256=-nBbVfJdsEvx_W0X4aO9RpXGBZ61y1QwTjreDq494s8,43765
+bbot/test/test_step_1/test_events.py,sha256=D9W3zGxRWUIm0SYklsWRE3IeAPcMdWLAOIMWkI24Rpc,45130
 bbot/test/test_step_1/test_files.py,sha256=5Q_3jPpMXULxDHsanSDUaj8zF8bXzKdiJZHOmoYpLhQ,699
-bbot/test/test_step_1/test_helpers.py,sha256=kRd64C_LD6VhBM1UJrWlkX1rGLFW0QtAj8mPX0dgbq4,37787
+bbot/test/test_step_1/test_helpers.py,sha256=oY2hWhgL-TCB67ve1bAyIwZO3wNRWpx4SjCHNUxHep8,38676
 bbot/test/test_step_1/test_manager_deduplication.py,sha256=hZQpDXzg6zvzxFolVOcJuY-ME8NXjZUsqS70BRNXp8A,15594
-bbot/test/test_step_1/test_manager_scope_accuracy.py,sha256=5CptaGVplTLsGx7THjrJva99M9rumd7JCUFjTAXYUVE,79454
-bbot/test/test_step_1/test_modules_basic.py,sha256=JlO1waj6JArpazCUIYaNzOIv1qnKs-ArrfXw1D2B_tA,20155
-bbot/test/test_step_1/test_presets.py,sha256=SY-A3q3ubwFPJ0dSEgo9DHud-VpNrJa-PS-BGw2P_j8,36870
+bbot/test/test_step_1/test_manager_scope_accuracy.py,sha256=_4O5bW9PA2DIeguvqzb4CMm0i1joqqBBppw7qElL2a0,79767
+bbot/test/test_step_1/test_modules_basic.py,sha256=h4eCe-UhniwXZGhTEa6tH-RGeBF8tyjpSHAPsSu-ssw,20295
+bbot/test/test_step_1/test_presets.py,sha256=RXdRCGBgwLBJk_npn5Pyph3IhLTfy0ULkb4v_aZjABA,37299
 bbot/test/test_step_1/test_python_api.py,sha256=BYIVREUy7rd7tKfUsnIX-w_eqQDTEo5UplMZeGMb4bs,5317
 bbot/test/test_step_1/test_regexes.py,sha256=btlDgwM5crSkXFZNydLYn8lCDmyLW_8oX_6Aos5xAHk,14376
 bbot/test/test_step_1/test_scan.py,sha256=yLQAZ5tvIwTULBGgUmXVg8KwMmujal7kB2p-S2_ORzA,5461
@@ -348,7 +348,7 @@ bbot/test/test_step_2/module_tests/test_module_robots.py,sha256=8rRw4GpGE6tN_W3o
 bbot/test/test_step_2/module_tests/test_module_secretsdb.py,sha256=EmqWGnqAQHGEZxbmJJXHwIgz5SQi4R3Fq982k6sYkZM,537
 bbot/test/test_step_2/module_tests/test_module_securitytrails.py,sha256=NB8_PhWN1-2s8wporRjI6rrQeQW4inoz4Z_mBhhycXo,758
 bbot/test/test_step_2/module_tests/test_module_securitytxt.py,sha256=i9Emp1utut1LPIe502Jb3XzN4GiVBeo4ATLFR9w4rbY,2382
-bbot/test/test_step_2/module_tests/test_module_shodan_dns.py,sha256=DQvs4e3_vtAfkzTCTHfFMh8UZHag6bUs_m6nbWL9rvM,713
+bbot/test/test_step_2/module_tests/test_module_shodan_dns.py,sha256=9DpFizZguP2aFCKI1HgXzxy3sB_afA03pzIx0BDfgqg,1515
 bbot/test/test_step_2/module_tests/test_module_sitedossier.py,sha256=YAaTzCL-lOYFu-42zk4dsV33V3Da3Md6rcAuBy9jgeY,6584
 bbot/test/test_step_2/module_tests/test_module_skymem.py,sha256=VaRhEmrZ0auKmxExeuYmzryXpZ0h78AqSVozkqJ5dXo,2321
 bbot/test/test_step_2/module_tests/test_module_slack.py,sha256=oJvUzSowPAhpbMFnzl-iS3XvLQBxCaO_vofhu_bgR4w,317
@@ -393,8 +393,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ruUQwVfia1_m2u
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=VciB-ww0y-O8Ii1wpTR6rJzGDiC2r-dhVsIJApS1ZYU,3309
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.1.0.5004rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.1.0.5004rc0.dist-info/METADATA,sha256=zye_cTeYT3jp0QSL3ThNRjBVFQVEHpADQQgJqjqWiow,16930
-bbot-2.1.0.5004rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-bbot-2.1.0.5004rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.1.0.5004rc0.dist-info/RECORD,,
+bbot-2.1.0.5028rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.1.0.5028rc0.dist-info/METADATA,sha256=3Y7ocHDJJpm5Ebb7PhjejhASPdJO6tK52qlUo_6RUQI,16930
+bbot-2.1.0.5028rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+bbot-2.1.0.5028rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.1.0.5028rc0.dist-info/RECORD,,