bbot 2.2.0.5263rc0__tar.gz → 2.2.0.5309rc0__tar.gz

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.2.0.5263rc0
+Version: 2.2.0.5309rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
-Keywords: python,cli,automation,osint,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
+Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
 Author: TheTechromancer
 Requires-Python: >=3.9,<4.0
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -20,7 +20,7 @@ Requires-Dist: ansible (>=7.3,<9.0)
 Requires-Dist: ansible-runner (>=2.3.2,<3.0.0)
 Requires-Dist: beautifulsoup4 (>=4.12.2,<5.0.0)
 Requires-Dist: cachetools (>=5.3.2,<6.0.0)
-Requires-Dist: cloudcheck (>=5.0.0.350,<6.0.0.0)
+Requires-Dist: cloudcheck (>=6.0.0.602,<7.0.0.0)
 Requires-Dist: deepdiff (>=6.2.3,<8.0.0)
 Requires-Dist: dnspython (>=2.4.2,<3.0.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0)
@@ -35,7 +35,7 @@ Requires-Dist: pycryptodome (>=3.17,<4.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pyjwt (>=2.7.0,<3.0.0)
 Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
-Requires-Dist: radixtarget (>=1.0.0.15,<2.0.0.0)
+Requires-Dist: radixtarget (>=2.0.0.50,<3.0.0.0)
 Requires-Dist: regex (>=2024.4.16,<2025.0.0)
 Requires-Dist: setproctitle (>=1.3.3,<2.0.0)
 Requires-Dist: socksio (>=1.0.0,<2.0.0)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.2.0.5263rc"
+__version__ = "v2.2.0.5309rc"
 
 from .scanner import Scanner, Preset

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/cli.py

@@ -174,7 +174,7 @@ async def _main():
             if sys.stdin.isatty():
 
                 # warn if any targets belong directly to a cloud provider
-                for event in scan.target.events:
+                for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":
                         cloudcheck_result = scan.helpers.cloudcheck(event.host)
                         if cloudcheck_result:

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/engine.py

@@ -641,7 +641,7 @@ class EngineServer(EngineBase):
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):
                     self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})")
-                    for task in tasks:
+                    for task in list(tasks):
                         task.cancel()
                         self._await_cancelled_task(task)
                 else:
@@ -683,5 +683,5 @@ class EngineServer(EngineBase):
         for client_id in list(self.tasks):
             await self.cancel_task(client_id)
         for client_id, tasks in self.child_tasks.items():
-            for task in tasks:
+            for task in list(tasks):
                 await self._await_cancelled_task(task)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/event/base.py

@@ -341,6 +341,21 @@ class BaseEvent:
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -1114,8 +1129,7 @@ class DnsEvent(BaseEvent):
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1689,6 +1703,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/bloom.py

@@ -64,8 +64,15 @@ class BloomFilter:
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -14,7 +14,7 @@ from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform, rm_at_exit
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -176,10 +176,13 @@ class DepsInstaller:
 
         command = [sys.executable, "-m", "pip", "install", "--upgrade"] + packages
 
-        if constraints:
-            constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
-            command.append("--constraint")
-            command.append(constraints_tempfile)
+        # if no custom constraints are provided, use the constraints of the currently installed version of bbot
+        if constraints is not None:
+            constraints = get_python_constraints()
+
+        constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
+        command.append("--constraint")
+        command.append(constraints_tempfile)
 
         process = None
         try:

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/dns/helpers.py

@@ -1,6 +1,6 @@
 import logging
 
-from bbot.core.helpers.regexes import dns_name_regex
+from bbot.core.helpers.regexes import dns_name_extraction_regex
 from bbot.core.helpers.misc import clean_dns_record, smart_decode
 
 log = logging.getLogger("bbot.core.helpers.dns")
@@ -198,7 +198,7 @@ def extract_targets(record):
     elif rdtype == "TXT":
         for s in record.strings:
             s = smart_decode(s)
-            for match in dns_name_regex.finditer(s):
+            for match in dns_name_extraction_regex.finditer(s):
                 start, end = match.span()
                 host = s[start:end]
                 add_result(rdtype, host)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/helper.py

@@ -12,10 +12,11 @@ from .diff import HttpCompare
 from .regex import RegexHelper
 from .wordcloud import WordCloud
 from .interactsh import Interactsh
-from ...scanner.target import Target
 from .depsinstaller import DepsInstaller
 from .async_helpers import get_event_loop
 
+from bbot.scanner.target import BaseTarget
+
 log = logging.getLogger("bbot.core.helpers")
 
 
@@ -155,8 +156,8 @@ class ConfigAwareHelper:
         _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events, **kwargs):
-        return Target(*events, **kwargs)
+    def make_target(self, *targets, **kwargs):
+        return BaseTarget(*targets, scan=self.scan, **kwargs)
 
     @property
     def config(self):

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/misc.py

@@ -586,17 +586,18 @@ def is_dns_name(d, include_local=True):
     if include_local:
         if bbot_regexes.hostname_regex.match(d):
             return True
-    if bbot_regexes.dns_name_regex.match(d):
+    if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False
 
 
-def is_ip(d, version=None):
+def is_ip(d, version=None, include_network=False):
     """
     Checks if the given string or object represents a valid IP address.
 
     Args:
         d (str or ipaddress.IPvXAddress): The IP address to check.
+        include_network (bool, optional): Whether to include network types (IPv4Network or IPv6Network). Defaults to False.
         version (int, optional): The IP version to validate (4 or 6). Default is None.
 
     Returns:
@@ -612,12 +613,17 @@ def is_ip(d, version=None):
         >>> is_ip('evilcorp.com')
         False
     """
+    ip = None
     try:
         ip = ipaddress.ip_address(d)
-        if version is None or ip.version == version:
-            return True
     except Exception:
-        pass
+        if include_network:
+            try:
+                ip = ipaddress.ip_network(d, strict=False)
+            except Exception:
+                pass
+    if ip is not None and (version is None or ip.version == version):
+        return True
     return False
 
 
@@ -2807,3 +2813,21 @@ def safe_format(s, **kwargs):
     Format string while ignoring unused keys (prevents KeyError)
     """
     return s.format_map(SafeDict(kwargs))
+
+
+def get_python_constraints():
+    req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE)
+
+    def clean_requirement(req_string):
+        # Extract package name and version constraints from format like "package (>=1.0,<2.0)"
+        match = req_regex.match(req_string)
+        if match:
+            name, constraints = match.groups()
+            return f"{name.strip()}{constraints}"
+
+        return req_string
+
+    from importlib.metadata import distribution
+
+    dist = distribution("bbot")
+    return [clean_requirement(r) for r in dist.requires]

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/regexes.py

@@ -40,7 +40,8 @@ ip_range_regexes = list(re.compile(r, re.I) for r in _ip_range_regexes)
 
 # dns names with periods
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
-dns_name_regex = re.compile(_dns_name_regex, re.I)
+dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
+dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
 # dns names without periods
 _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/core/helpers/web/web.py

@@ -58,7 +58,7 @@ class WebHelper(EngineClient):
         self.ssl_verify = self.config.get("ssl_verify", False)
         engine_debug = self.config.get("engine", {}).get("debug", False)
         super().__init__(
-            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.radix_only},
+            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.minimal},
             debug=engine_debug,
         )
 

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/defaults.yml

@@ -14,6 +14,9 @@ folder_blobs: false
 ### SCOPE ###
 
 scope:
+  # strict scope means only exact DNS names are considered in-scope
+  # subdomains are not included unless they are explicitly provided in the target list
+  strict: false
   # Filter by scope distance which events are displayed in the output
   # 0 == show only in-scope events (affiliates are always shown)
   # 1 == show all events up to distance-1 (1 hop from target)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/anubisdb.py

@@ -38,7 +38,7 @@ class anubisdb(subdomain_enum):
             return True, "DNS name is unresolved"
         return await super().abort_if(event)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json:

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/baddns.py

@@ -116,7 +116,7 @@ class baddns(BaseModule):
                                 context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
                         else:
-                            self.warning(f"Got unrecognized confidence level: {r['confidence']}")
+                            self.warning(f"Got unrecognized confidence level: {r_dict['confidence']}")
 
                         found_domains = r_dict.get("found_domains", None)
                         if found_domains:

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/bevigil.py

@@ -60,14 +60,14 @@ class bevigil(subdomain_enum_apikey):
         url = f"{self.base_url}/{self.helpers.quote(query)}/urls/"
         return await self.api_request(url)
 
-    def parse_subdomains(self, r, query=None):
+    async def parse_subdomains(self, r, query=None):
         results = set()
         subdomains = r.json().get("subdomains")
         if subdomains:
             results.update(subdomains)
         return results
 
-    def parse_urls(self, r, query=None):
+    async def parse_urls(self, r, query=None):
         results = set()
         urls = r.json().get("urls")
         if urls:

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/binaryedge.py

@@ -37,6 +37,6 @@ class binaryedge(subdomain_enum_apikey):
         url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         return j.get("events", [])

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/bufferoverrun.py

@@ -33,7 +33,7 @@ class BufferOverrun(subdomain_enum_apikey):
         url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):
@@ -44,5 +44,4 @@ class BufferOverrun(subdomain_enum_apikey):
                     subdomain = parts[4].strip()
                     if subdomain and subdomain.endswith(f".{query}"):
                         subdomains_set.add(subdomain)
-            for subdomain in subdomains_set:
-                yield subdomain
+        return subdomains_set

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/builtwith.py

@@ -62,7 +62,7 @@ class builtwith(subdomain_enum_apikey):
         url = f"{self.base_url}/redirect1/api.json?KEY={{api_key}}&LOOKUP={query}"
         return await self.api_request(url)
 
-    def parse_domains(self, r, query):
+    async def parse_domains(self, r, query):
         """
         This method returns a set of subdomains.
         Each subdomain is an "FQDN" that was reported in the "Detailed Technology Profile" page on builtwith.com
@@ -92,7 +92,7 @@ class builtwith(subdomain_enum_apikey):
                     self.verbose(f"No results for {query}: {error}")
         return results_set
 
-    def parse_redirects(self, r, query):
+    async def parse_redirects(self, r, query):
         """
         This method creates a set.
         Each entry in the set is either an Inbound or Outbound Redirect reported in the "Redirect Profile" page on builtwith.com

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/c99.py

@@ -26,7 +26,8 @@ class c99(subdomain_enum_apikey):
         url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             subdomains = j.get("subdomains", [])
@@ -34,4 +35,5 @@ class c99(subdomain_enum_apikey):
                 for s in subdomains:
                     subdomain = s.get("subdomain", "")
                     if subdomain:
-                        yield subdomain
+                        results.add(subdomain)
+        return results

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/certspotter.py

@@ -17,9 +17,11 @@ class certspotter(subdomain_enum):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
         return self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         json = r.json()
         if json:
             for r in json:
                 for dns_name in r.get("dns_names", []):
-                    yield dns_name.lstrip(".*").rstrip(".")
+                    results.add(dns_name.lstrip(".*").rstrip("."))
+        return results

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/chaos.py

@@ -26,7 +26,8 @@ class chaos(subdomain_enum_apikey):
         url = f"{self.base_url}/{domain}/subdomains"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):
@@ -39,4 +40,5 @@ class chaos(subdomain_enum_apikey):
                 for s in subdomains_set:
                     full_subdomain = f"{s}.{domain}"
                     if full_subdomain and full_subdomain.endswith(f".{query}"):
-                        yield full_subdomain
+                        results.add(full_subdomain)
+        return results

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/columbus.py

@@ -17,7 +17,7 @@ class columbus(subdomain_enum):
         url = f"{self.base_url}/{self.helpers.quote(query)}?days=365"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/crt.py

@@ -23,7 +23,8 @@ class crt(subdomain_enum):
         url = self.helpers.add_get_params(self.base_url, params).geturl()
         return await self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         for cert_info in j:
             if not type(cert_info) == dict:
@@ -35,4 +36,5 @@ class crt(subdomain_enum):
                     domain = cert_info.get("name_value")
                     if domain:
                         for d in domain.splitlines():
-                            yield d.lower()
+                            results.add(d.lower())
+        return results

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/digitorus.py

@@ -19,7 +19,7 @@ class digitorus(subdomain_enum):
         url = f"{self.base_url}/{self.helpers.quote(query)}"
         return await self.helpers.request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         content = getattr(r, "text", "")
         extract_regex = re.compile(r"[\w.-]+\." + query, re.I)

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/dnscaa.py

@@ -2,7 +2,7 @@
 #
 # Checks for and parses CAA DNS TXT records for IODEF reporting destination email addresses and/or URL's.
 #
-# NOTE: when the target domain is initially resolved basic "dns_name_regex" matched targets will be extracted so we do not perform that again here.
+# NOTE: when the target domain is initially resolved basic "dns_name_extraction_regex" matched targets will be extracted so we do not perform that again here.
 #
 # Example CAA records,
 #   0 iodef "mailto:dnsadmin@example.com"
@@ -23,7 +23,7 @@ from bbot.modules.base import BaseModule
 
 import re
 
-from bbot.core.helpers.regexes import dns_name_regex, email_regex, url_regexes
+from bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes
 
 # Handle '0 iodef "mailto:support@hcaptcha.com"'
 # Handle '1 iodef "https://some.host.tld/caa;"'
@@ -109,7 +109,7 @@ class dnscaa(BaseModule):
 
                     elif caa_match.group("property").lower().startswith("issue"):
                         if self._dns_names:
-                            for match in dns_name_regex.finditer(caa_match.group("text")):
+                            for match in dns_name_extraction_regex.finditer(caa_match.group("text")):
                                 start, end = match.span()
                                 name = caa_match.group("text")[start:end]
 

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/fullhunt.py

@@ -35,5 +35,5 @@ class fullhunt(subdomain_enum_apikey):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         return r.json().get("hosts", [])

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/hackertarget.py

@@ -18,12 +18,14 @@ class hackertarget(subdomain_enum):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for line in r.text.splitlines():
             host = line.split(",")[0]
             try:
                 self.helpers.validators.validate_host(host)
-                yield host
+                results.add(host)
             except ValueError:
                 self.debug(f"Error validating API result: {line}")
                 continue
+        return results

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/internal/excavate.py

@@ -527,9 +527,8 @@ class excavate(BaseInternalModule, BaseInterceptModule):
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
             for identifier in yara_results.keys():
                 for csp_str in yara_results[identifier]:
-                    domains = await self.helpers.re.findall(bbot_regexes.dns_name_regex, csp_str)
-                    unique_domains = set(domains)
-                    for domain in unique_domains:
+                    domains = await self.excavate.scan.extract_in_scope_hostnames(csp_str)
+                    for domain in domains:
                         await self.report(domain, event, yara_rule_settings, discovery_context, event_type="DNS_NAME")
 
     class EmailExtractor(ExcavateRule):

{bbot-2.2.0.5263rc0 → bbot-2.2.0.5309rc0}/bbot/modules/internal/speculate.py

@@ -32,10 +32,11 @@ class speculate(BaseInternalModule):
         "author": "@liquidsec",
     }
 
-    options = {"max_hosts": 65536, "ports": "80,443"}
+    options = {"max_hosts": 65536, "ports": "80,443", "essential_only": False}
     options_desc = {
         "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events",
         "ports": "The set of ports to speculate on",
+        "essential_only": "Only enable essential speculate features (no extra discovery)",
     }
     scope_distance_modifier = 1
     _priority = 4
@@ -52,6 +53,7 @@ class speculate(BaseInternalModule):
         self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
         self.range_to_ip = True
         self.dns_disable = self.scan.config.get("dns", {}).get("disable", False)
+        self.essential_only = self.config.get("essential_only", False)
         self.org_stubs_seen = set()
 
         port_string = self.config.get("ports", "80,443")
@@ -63,7 +65,7 @@ class speculate(BaseInternalModule):
         if not self.portscanner_enabled:
             self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
 
-        target_len = len(self.scan.target)
+        target_len = len(self.scan.target.seeds)
         if target_len > self.config.get("max_hosts", 65536):
             if not self.portscanner_enabled:
                 self.hugewarning(
@@ -75,6 +77,14 @@ class speculate(BaseInternalModule):
         return True
 
     async def handle_event(self, event):
+        ### BEGIN ESSENTIAL SPECULATION ###
+        # These features are required for smooth operation of bbot
+        # I.e. they are not "osinty" or intended to discover anything, they only compliment other modules
+
+        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
+        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
+        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
+
         # generate individual IP addresses from IP range
         if event.type == "IP_RANGE" and self.range_to_ip:
             net = ipaddress.ip_network(event.data)
@@ -89,6 +99,28 @@ class speculate(BaseInternalModule):
                     context=f"speculate converted range into individual IP_ADDRESS: {ip}",
                 )
 
+        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
+        if speculate_open_ports:
+            # don't act on unresolved DNS_NAMEs
+            usable_dns = False
+            if event.type == "DNS_NAME":
+                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
+                    usable_dns = True
+
+            if event.type == "IP_ADDRESS" or usable_dns:
+                for port in self.ports:
+                    await self.emit_event(
+                        self.helpers.make_netloc(event.data, port),
+                        "OPEN_TCP_PORT",
+                        parent=event,
+                        internal=True,
+                        context="speculated {event.type}: {event.data}",
+                    )
+
+        ### END ESSENTIAL SPECULATION ###
+        if self.essential_only:
+            return
+
         # parent domains
         if event.type.startswith("DNS_NAME"):
             parent = self.helpers.parent_domain(event.host_original)
@@ -97,10 +129,6 @@ class speculate(BaseInternalModule):
                     parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}"
                 )
 
-        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
-        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
-        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
-
         # URL --> OPEN_TCP_PORT
         event_is_url = event.type == "URL"
         if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
@@ -144,24 +172,6 @@ class speculate(BaseInternalModule):
                     context="speculated {event.type}: {event.data}",
                 )
 
-        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
-        if speculate_open_ports:
-            # don't act on unresolved DNS_NAMEs
-            usable_dns = False
-            if event.type == "DNS_NAME":
-                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
-                    usable_dns = True
-
-            if event.type == "IP_ADDRESS" or usable_dns:
-                for port in self.ports:
-                    await self.emit_event(
-                        self.helpers.make_netloc(event.data, port),
-                        "OPEN_TCP_PORT",
-                        parent=event,
-                        internal=True,
-                        context="speculated {event.type}: {event.data}",
-                    )
-
         # ORG_STUB from TLD, SOCIAL, AZURE_TENANT
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0: