bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/internal/dnsresolve.py

@@ -3,11 +3,11 @@ from contextlib import suppress
 
 from bbot.errors import ValidationError
 from bbot.core.helpers.dns.engine import all_rdtypes
-from bbot.modules.base import InterceptModule, BaseModule
 from bbot.core.helpers.dns.helpers import extract_targets
+from bbot.modules.base import BaseInterceptModule, BaseModule
 
 
-class DNSResolve(InterceptModule):
+class DNSResolve(BaseInterceptModule):
     watched_events = ["*"]
     _priority = 1
     scope_distance_modifier = None
@@ -16,12 +16,6 @@ class DNSResolve(InterceptModule):
         _name = "host"
         _type = "internal"
 
-        def _outgoing_dedup_hash(self, event):
-            # this exists to ensure a second, more interesting host isn't passed up
-            # because its ugly cousin spent its one dedup token before it arrived
-            # by removing those race conditions, this makes for more consistent results
-            return hash((event, self.name, event.always_emit))
-
     @property
     def module_threads(self):
         return self.dns_config.get("threads", 25)
@@ -85,11 +79,21 @@ class DNSResolve(InterceptModule):
                 await self.resolve_event(main_host_event, types=non_minimal_rdtypes)
                 # check for wildcards if the event is within the scan's search distance
                 if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance:
-                    await self.handle_wildcard_event(main_host_event)
+                    event_data_changed = await self.handle_wildcard_event(main_host_event)
+                    if event_data_changed:
+                        # since data has changed, we check again whether it's a duplicate
+                        if self.scan.ingress_module.is_incoming_duplicate(event, add=True):
+                            if not event._graph_important:
+                                return False, "event was already emitted by its module"
+                            else:
+                                self.debug(
+                                    f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass"
+                                )
 
         # if there weren't any DNS children and it's not an IP address, tag as unresolved
         if not main_host_event.raw_dns_records and not event_is_ip:
             main_host_event.add_tag("unresolved")
+            main_host_event.type = "DNS_NAME_UNRESOLVED"
 
         # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}")
 
@@ -109,10 +113,6 @@ class DNSResolve(InterceptModule):
             if not self.minimal:
                 await self.emit_dns_children(main_host_event)
 
-            # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
-            if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
-                main_host_event.type = "DNS_NAME_UNRESOLVED"
-
             # emit the main DNS_NAME or IP_ADDRESS
             if (
                 new_event
@@ -131,9 +131,9 @@ class DNSResolve(InterceptModule):
             event.host, rdtypes=rdtypes, raw_dns_records=event.raw_dns_records
         )
         for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():
-            if is_wildcard == False:
+            if is_wildcard is False:
                 continue
-            elif is_wildcard == True:
+            elif is_wildcard is True:
                 event.add_tag("wildcard")
                 wildcard_tag = "wildcard"
             else:
@@ -142,16 +142,16 @@ class DNSResolve(InterceptModule):
             event.add_tag(f"{rdtype}-{wildcard_tag}")
 
         # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
-        if wildcard_rdtypes and not "target" in event.tags:
+        if wildcard_rdtypes and "target" not in event.tags:
             # these are the rdtypes that have wildcards
             wildcard_rdtypes_set = set(wildcard_rdtypes)
             # consider the event a full wildcard if all its records are wildcards
             event_is_wildcard = False
             if wildcard_rdtypes_set:
-                event_is_wildcard = all(r[0] == True for r in wildcard_rdtypes.values())
+                event_is_wildcard = all(r[0] is True for r in wildcard_rdtypes.values())
 
             if event_is_wildcard:
-                if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."):
+                if event.type in ("DNS_NAME",) and "_wildcard" not in event.data.split("."):
                     wildcard_parent = self.helpers.parent_domain(event.host)
                     for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():
                         if _is_wildcard:
@@ -161,6 +161,8 @@ class DNSResolve(InterceptModule):
                     if wildcard_data != event.data:
                         self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"')
                         event.data = wildcard_data
+                        return True
+        return False
 
     async def emit_dns_children(self, event):
         for rdtype, children in event.dns_children.items():
@@ -271,7 +273,7 @@ class DNSResolve(InterceptModule):
         # tag event with errors
         for rdtype, errors in dns_errors.items():
             # only consider it an error if there weren't any results for that rdtype
-            if errors and not rdtype in event.dns_children:
+            if errors and rdtype not in event.dns_children:
                 event.add_tag(f"{rdtype}-error")
 
     def get_dns_parent(self, event):
@@ -305,7 +307,7 @@ class DNSResolve(InterceptModule):
     def emit_raw_records(self):
         if self._emit_raw_records is None:
             watching_raw_records = any(
-                ["RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()]
+                "RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()
             )
             omitted_event_types = self.scan.config.get("omit_event_types", [])
             omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types

bbot/modules/internal/excavate.py

@@ -6,6 +6,7 @@ import regex as re
 from pathlib import Path
 from bbot.errors import ExcavateError
 import bbot.core.helpers.regexes as bbot_regexes
+from bbot.modules.base import BaseInterceptModule
 from bbot.modules.internal.base import BaseInternalModule
 from urllib.parse import urlparse, urljoin, parse_qs, urlunparse
 
@@ -61,7 +62,6 @@ def _exclude_key(original_dict, key_to_exclude):
 
 
 def extract_params_url(parsed_url):
-
     params = parse_qs(parsed_url.query)
     flat_params = {k: v[0] for k, v in params.items()}
 
@@ -93,7 +93,6 @@ def extract_params_location(location_header_value, original_parsed_url):
 
 
 class YaraRuleSettings:
-
     def __init__(self, description, tags, emit_match):
         self.description = description
         self.tags = tags
@@ -153,7 +152,9 @@ class ExcavateRule:
         yara_rule_settings = YaraRuleSettings(description, tags, emit_match)
         yara_results = {}
         for h in r.strings:
-            yara_results[h.identifier.lstrip("$")] = sorted(set([i.matched_data.decode("utf-8") for i in h.instances]))
+            yara_results[h.identifier.lstrip("$")] = sorted(
+                {i.matched_data.decode("utf-8", errors="ignore") for i in h.instances}
+            )
         await self.process(yara_results, event, yara_rule_settings, discovery_context)
 
     async def process(self, yara_results, event, yara_rule_settings, discovery_context):
@@ -179,7 +180,7 @@ class ExcavateRule:
         Returns:
         None
         """
-        for identifier, results in yara_results.items():
+        for results in yara_results.values():
             for result in results:
                 event_data = {"description": f"{discovery_context} {yara_rule_settings.description}"}
                 if yara_rule_settings.emit_match:
@@ -260,7 +261,6 @@ class ExcavateRule:
 
 
 class CustomExtractor(ExcavateRule):
-
     def __init__(self, excavate):
         super().__init__(excavate)
 
@@ -279,7 +279,7 @@ class CustomExtractor(ExcavateRule):
                 await self.report(event_data, event, yara_rule_settings, discovery_context)
 
 
-class excavate(BaseInternalModule):
+class excavate(BaseInternalModule, BaseInterceptModule):
     """
     Example (simple) Excavate Rules:
 
@@ -310,10 +310,11 @@ class excavate(BaseInternalModule):
         "custom_yara_rules": "Include custom Yara rules",
     }
     scope_distance_modifier = None
+    accept_dupes = False
 
     _module_threads = 8
 
-    parameter_blacklist = set(
+    parameter_blacklist = {
         p.lower()
         for p in [
             "__VIEWSTATE",
@@ -328,7 +329,7 @@ class excavate(BaseInternalModule):
             "JSESSIONID",
             "PHPSESSID",
         ]
-    )
+    }
 
     yara_rule_name_regex = re.compile(r"rule\s(\w+)\s{")
     yara_rule_regex = re.compile(r"(?s)((?:rule\s+\w+\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\S*?}[^/]*?/)*)*})")
@@ -354,7 +355,6 @@ class excavate(BaseInternalModule):
         )
 
     class ParameterExtractor(ExcavateRule):
-
         yara_rules = {}
 
         class ParameterExtractorRule:
@@ -368,7 +368,6 @@ class excavate(BaseInternalModule):
                 self.result = result
 
         class GetJquery(ParameterExtractorRule):
-
             name = "GET jquery"
             discovery_regex = r"/\$.get\([^\)].+\)/ nocase"
             extraction_regex = re.compile(r"\$.get\([\'\"](.+)[\'\"].+(\{.+\})\)")
@@ -389,8 +388,12 @@ class excavate(BaseInternalModule):
                     for action, extracted_parameters in extracted_results:
                         extracted_parameters_dict = self.convert_to_dict(extracted_parameters)
                         for parameter_name, original_value in extracted_parameters_dict.items():
-                            yield self.output_type, parameter_name, original_value, action, _exclude_key(
-                                extracted_parameters_dict, parameter_name
+                            yield (
+                                self.output_type,
+                                parameter_name,
+                                original_value,
+                                action,
+                                _exclude_key(extracted_parameters_dict, parameter_name),
                             )
 
         class PostJquery(GetJquery):
@@ -414,8 +417,12 @@ class excavate(BaseInternalModule):
                         k: v[0] if isinstance(v, list) and len(v) == 1 else v for k, v in query_strings.items()
                     }
                     for parameter_name, original_value in query_strings_dict.items():
-                        yield self.output_type, parameter_name, original_value, url, _exclude_key(
-                            query_strings_dict, parameter_name
+                        yield (
+                            self.output_type,
+                            parameter_name,
+                            original_value,
+                            url,
+                            _exclude_key(query_strings_dict, parameter_name),
                         )
 
         class GetForm(ParameterExtractorRule):
@@ -440,8 +447,12 @@ class excavate(BaseInternalModule):
                             form_parameters[parameter_name] = original_value
 
                         for parameter_name, original_value in form_parameters.items():
-                            yield self.output_type, parameter_name, original_value, form_action, _exclude_key(
-                                form_parameters, parameter_name
+                            yield (
+                                self.output_type,
+                                parameter_name,
+                                original_value,
+                                form_action,
+                                _exclude_key(form_parameters, parameter_name),
                             )
 
         class PostForm(GetForm):
@@ -481,7 +492,6 @@ class excavate(BaseInternalModule):
                             endpoint,
                             additional_params,
                         ) in extracted_params:
-
                             self.excavate.debug(
                                 f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule"
                             )
@@ -493,7 +503,6 @@ class excavate(BaseInternalModule):
                             )
 
                             if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
-
                                 if self.excavate.in_bl(parameter_name) == False:
                                     parsed_url = urlparse(url)
                                     description = f"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)"
@@ -523,13 +532,11 @@ class excavate(BaseInternalModule):
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
             for identifier in yara_results.keys():
                 for csp_str in yara_results[identifier]:
-                    domains = await self.helpers.re.findall(bbot_regexes.dns_name_regex, csp_str)
-                    unique_domains = set(domains)
-                    for domain in unique_domains:
+                    domains = await self.excavate.scan.extract_in_scope_hostnames(csp_str)
+                    for domain in domains:
                         await self.report(domain, event, yara_rule_settings, discovery_context, event_type="DNS_NAME")
 
     class EmailExtractor(ExcavateRule):
-
         yara_rules = {
             "email": 'rule email { meta: description = "contains email address" strings: $email = /[^\\W_][\\w\\-\\.\\+\']{0,100}@[a-zA-Z0-9\\-]{1,100}(\\.[a-zA-Z0-9\\-]{1,100})*\\.[a-zA-Z]{2,63}/ nocase fullword condition: $email }',
         }
@@ -548,7 +555,6 @@ class excavate(BaseInternalModule):
         }
 
     class ErrorExtractor(ExcavateRule):
-
         signatures = {
             "PHP_1": r"/\.php on line [0-9]+/",
             "PHP_2": r"/\.php<\/b> on line <b>[0-9]+/",
@@ -586,7 +592,6 @@ class excavate(BaseInternalModule):
                     await self.report(event_data, event, yara_rule_settings, discovery_context, event_type="FINDING")
 
     class SerializationExtractor(ExcavateRule):
-
         regexes = {
             "Java": re.compile(r"[^a-zA-Z0-9\/+]rO0[a-zA-Z0-9+\/]+={0,2}"),
             "DOTNET": re.compile(r"[^a-zA-Z0-9\/+]AAEAAAD\/\/[a-zA-Z0-9\/+]+={0,2}"),
@@ -616,7 +621,6 @@ class excavate(BaseInternalModule):
                     await self.report(event_data, event, yara_rule_settings, discovery_context, event_type="FINDING")
 
     class FunctionalityExtractor(ExcavateRule):
-
         yara_rules = {
             "File_Upload_Functionality": r'rule File_Upload_Functionality { meta: description = "contains file upload functionality" strings: $fileuploadfunc = /<input[^>]+type=["\']?file["\']?[^>]+>/ nocase condition: $fileuploadfunc }',
             "Web_Service_WSDL": r'rule Web_Service_WSDL { meta: emit_match = "True" description = "contains a web service WSDL URL" strings: $wsdl = /https?:\/\/[^\s]*\.(wsdl)/ nocase condition: $wsdl }',
@@ -630,7 +634,7 @@ class excavate(BaseInternalModule):
         scheme_blacklist = ["javascript", "mailto", "tel", "data", "vbscript", "about", "file"]
 
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
-            for identifier, results in yara_results.items():
+            for results in yara_results.values():
                 for url_str in results:
                     scheme = url_str.split("://")[0]
                     if scheme in self.scheme_blacklist:
@@ -669,15 +673,38 @@ class excavate(BaseInternalModule):
 
     class URLExtractor(ExcavateRule):
         yara_rules = {
-            "url_full": r'rule url_full { meta: tags = "spider-danger" description = "contains full URL" strings: $url_full = /https?:\/\/([\w\.-]+)([:\/\w\.-]*)/ condition: $url_full }',
-            "url_attr": r'rule url_attr { meta: tags = "spider-danger" description = "contains tag with src or href attribute" strings: $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ condition: $url_attr }',
+            "url_full": (
+                r"""
+                rule url_full {
+                    meta:
+                        tags = "spider-danger"
+                        description = "contains full URL"
+                    strings:
+                        $url_full = /https?:\/\/([\w\.-]+)(:\d{1,5})?([\/\w\.-]*)/
+                    condition:
+                        $url_full
+                }
+                """
+            ),
+            "url_attr": (
+                r"""
+                rule url_attr {
+                    meta:
+                        tags = "spider-danger"
+                        description = "contains tag with src or href attribute"
+                    strings:
+                        $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/
+                    condition:
+                        $url_attr
+                }
+                """
+            ),
         }
         full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)")
         full_url_regex_strict = re.compile(r"^(https?):\/\/([\w.-]+)(?::\d{1,5})?(\/[\w\/\.-]*)?(\?[^\s]+)?$")
         tag_attribute_regex = bbot_regexes.tag_attribute_regex
 
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
-
             for identifier, results in yara_results.items():
                 urls_found = 0
                 final_url = ""
@@ -741,20 +768,33 @@ class excavate(BaseInternalModule):
 
         def __init__(self, excavate):
             super().__init__(excavate)
-            regexes_component_list = []
-            if excavate.scan.dns_regexes_yara:
-                for i, r in enumerate(excavate.scan.dns_regexes_yara):
-                    regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase")
-                regexes_component = " ".join(regexes_component_list)
-                self.yara_rules[f"hostname_extraction"] = (
-                    f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}'
-                )
+            if excavate.scan.dns_yara_rules_uncompiled:
+                self.yara_rules[f"hostname_extraction"] = excavate.scan.dns_yara_rules_uncompiled
 
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
             for identifier in yara_results.keys():
                 for domain_str in yara_results[identifier]:
                     await self.report(domain_str, event, yara_rule_settings, discovery_context, event_type="DNS_NAME")
 
+    class LoginPageExtractor(ExcavateRule):
+        yara_rules = {
+            "login_page": r"""
+            rule login_page {
+                meta:
+                    description = "Detects login pages with username and password fields"
+                strings:
+                    $username_field = /<input[^>]+name=["']?(user|login|email)/ nocase
+                    $password_field = /<input[^>]+name=["']?passw?/ nocase
+                condition:
+                    $username_field and $password_field
+            }
+            """
+        }
+
+        async def process(self, yara_results, event, yara_rule_settings, discovery_context):
+            if yara_results:
+                event.add_tag("login-page")
+
     def add_yara_rule(self, rule_name, rule_content, rule_instance):
         rule_instance.name = rule_name
         self.yara_rules_dict[rule_name] = rule_content
@@ -805,9 +845,9 @@ class excavate(BaseInternalModule):
             if Path(self.custom_yara_rules).is_file():
                 with open(self.custom_yara_rules) as f:
                     rules_content = f.read()
-                self.debug(f"Successfully loaded secrets file [{self.custom_yara_rules}]")
+                self.debug(f"Successfully loaded custom yara rules file [{self.custom_yara_rules}]")
             else:
-                self.debug(f"Custom secrets is NOT a file. Will attempt to treat it as rule content")
+                self.debug(f"Custom yara rules file is NOT a file. Will attempt to treat it as rule content")
                 rules_content = self.custom_yara_rules
 
             self.debug(f"Final combined yara rule contents: {rules_content}")
@@ -816,13 +856,11 @@ class excavate(BaseInternalModule):
                 try:
                     yara.compile(source=rule_content)
                 except yara.SyntaxError as e:
-                    self.hugewarning(f"Custom Yara rule failed to compile: {e}")
-                    return False
+                    return False, f"Custom Yara rule failed to compile: {e}"
 
                 rule_match = await self.helpers.re.search(self.yara_rule_name_regex, rule_content)
                 if not rule_match:
-                    self.hugewarning(f"Custom Yara formatted incorrectly: could not find rule name")
-                    return False
+                    return False, f"Custom Yara formatted incorrectly: could not find rule name"
 
                 rule_name = rule_match.groups(1)[0]
                 c = CustomExtractor(self)
@@ -836,11 +874,13 @@ class excavate(BaseInternalModule):
         yara.set_config(max_match_data=yara_max_match_data)
         yara_rules_combined = "\n".join(self.yara_rules_dict.values())
         try:
+            self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules")
+            for rule_name, rule_content in self.yara_rules_dict.items():
+                self.debug(f"  - {rule_name}")
             self.yara_rules = yara.compile(source=yara_rules_combined)
         except yara.SyntaxError as e:
-            self.hugewarning(f"Yara Rules failed to compile with error: [{e}]")
             self.debug(yara_rules_combined)
-            return False
+            return False, f"Yara Rules failed to compile with error: [{e}]"
 
         # pre-load valid URL schemes
         valid_schemes_filename = self.helpers.wordlist_dir / "valid_url_schemes.txt"
@@ -857,7 +897,6 @@ class excavate(BaseInternalModule):
         decoded_data = await self.helpers.re.recursive_decode(data)
 
         if self.parameter_extraction:
-
             content_type_lower = content_type.lower() if content_type else ""
             extraction_map = {
                 "json": self.helpers.extract_params_json,
@@ -894,7 +933,6 @@ class excavate(BaseInternalModule):
                 self.hugewarning(f"YARA Rule {rule_name} not found in pre-compiled rules")
 
     async def handle_event(self, event):
-
         if event.type == "HTTP_RESPONSE":
             # Harvest GET parameters from URL, if it came directly from the target, and parameter extraction is enabled
             if (
@@ -983,7 +1021,6 @@ class excavate(BaseInternalModule):
 
                             # Try to extract parameters from the redirect URL
                             if self.parameter_extraction:
-
                                 for (
                                     method,
                                     parsed_url,

bbot/modules/internal/speculate.py

@@ -32,10 +32,11 @@ class speculate(BaseInternalModule):
         "author": "@liquidsec",
     }
 
-    options = {"max_hosts": 65536, "ports": "80,443"}
+    options = {"max_hosts": 65536, "ports": "80,443", "essential_only": False}
     options_desc = {
         "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events",
         "ports": "The set of ports to speculate on",
+        "essential_only": "Only enable essential speculate features (no extra discovery)",
     }
     scope_distance_modifier = 1
     _priority = 4
@@ -44,14 +45,15 @@ class speculate(BaseInternalModule):
 
     async def setup(self):
         scan_modules = [m for m in self.scan.modules.values() if m._type == "scan"]
-        self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in scan_modules])
+        self.open_port_consumers = any("OPEN_TCP_PORT" in m.watched_events for m in scan_modules)
         # only consider active portscanners (still speculate if only passive ones are enabled)
         self.portscanner_enabled = any(
-            ["portscan" in m.flags and "active" in m.flags for m in self.scan.modules.values()]
+            "portscan" in m.flags and "active" in m.flags for m in self.scan.modules.values()
         )
         self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
         self.range_to_ip = True
         self.dns_disable = self.scan.config.get("dns", {}).get("disable", False)
+        self.essential_only = self.config.get("essential_only", False)
         self.org_stubs_seen = set()
 
         port_string = self.config.get("ports", "80,443")
@@ -63,18 +65,26 @@ class speculate(BaseInternalModule):
         if not self.portscanner_enabled:
             self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
 
-        target_len = len(self.scan.target)
+        target_len = len(self.scan.target.seeds)
         if target_len > self.config.get("max_hosts", 65536):
             if not self.portscanner_enabled:
                 self.hugewarning(
                     f"Selected target ({target_len:,} hosts) is too large, skipping IP_RANGE --> IP_ADDRESS speculation"
                 )
-                self.hugewarning(f'Enabling the "portscan" module is highly recommended')
+                self.hugewarning('Enabling the "portscan" module is highly recommended')
             self.range_to_ip = False
 
         return True
 
     async def handle_event(self, event):
+        ### BEGIN ESSENTIAL SPECULATION ###
+        # These features are required for smooth operation of bbot
+        # I.e. they are not "osinty" or intended to discover anything, they only compliment other modules
+
+        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
+        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
+        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
+
         # generate individual IP addresses from IP range
         if event.type == "IP_RANGE" and self.range_to_ip:
             net = ipaddress.ip_network(event.data)
@@ -89,28 +99,46 @@ class speculate(BaseInternalModule):
                     context=f"speculate converted range into individual IP_ADDRESS: {ip}",
                 )
 
+        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
+        if speculate_open_ports:
+            # don't act on unresolved DNS_NAMEs
+            usable_dns = False
+            if event.type == "DNS_NAME":
+                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
+                    usable_dns = True
+
+            if event.type == "IP_ADDRESS" or usable_dns:
+                for port in self.ports:
+                    await self.emit_event(
+                        self.helpers.make_netloc(event.data, port),
+                        "OPEN_TCP_PORT",
+                        parent=event,
+                        internal=True,
+                        context="speculated {event.type}: {event.data}",
+                    )
+
+        ### END ESSENTIAL SPECULATION ###
+        if self.essential_only:
+            return
+
         # parent domains
         if event.type.startswith("DNS_NAME"):
             parent = self.helpers.parent_domain(event.host_original)
             if parent != event.data:
                 await self.emit_event(
-                    parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}"
+                    parent, "DNS_NAME", parent=event, context="speculated parent {event.type}: {event.data}"
                 )
 
-        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
-        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
-        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
-
         # URL --> OPEN_TCP_PORT
-        if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
+        event_is_url = event.type == "URL"
+        if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
             # only speculate port from a URL if it wouldn't be speculated naturally from the host
             if event.host and (event.port not in self.ports or not speculate_open_ports):
                 await self.emit_event(
                     self.helpers.make_netloc(event.host, event.port),
                     "OPEN_TCP_PORT",
                     parent=event,
-                    internal=True,
-                    quick=(event.type == "URL"),
+                    internal=not event_is_url,  # if the URL is verified, the port is definitely open
                     context=f"speculated {{event.type}} from {event.type}: {{event.data}}",
                 )
 
@@ -144,25 +172,6 @@ class speculate(BaseInternalModule):
                     context="speculated {event.type}: {event.data}",
                 )
 
-        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
-        if speculate_open_ports:
-            # don't act on unresolved DNS_NAMEs
-            usable_dns = False
-            if event.type == "DNS_NAME":
-                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
-                    usable_dns = True
-
-            if event.type == "IP_ADDRESS" or usable_dns:
-                for port in self.ports:
-                    await self.emit_event(
-                        self.helpers.make_netloc(event.data, port),
-                        "OPEN_TCP_PORT",
-                        parent=event,
-                        internal=True,
-                        quick=True,
-                        context="speculated {event.type}: {event.data}",
-                    )
-
         # ORG_STUB from TLD, SOCIAL, AZURE_TENANT
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0:

bbot/modules/internetdb.py

@@ -48,6 +48,9 @@ class internetdb(BaseModule):
         "show_open_ports": "Display OPEN_TCP_PORT events in output, even if they didn't lead to an interesting discovery"
     }
 
+    # we get lots of 404s, that's normal
+    _api_failure_abort_threshold = 9999999999
+
     _qsize = 500
 
     base_url = "https://internetdb.shodan.io"
@@ -64,7 +67,7 @@ class internetdb(BaseModule):
         if ip is None:
             return
         url = f"{self.base_url}/{ip}"
-        r = await self.request_with_fail_count(url)
+        r = await self.api_request(url)
         if r is None:
             self.debug(f"No response for {event.data}")
             return
@@ -113,7 +116,6 @@ class internetdb(BaseModule):
                 "OPEN_TCP_PORT",
                 parent=event,
                 internal=(not self.show_open_ports),
-                quick=True,
                 context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}',
             )
         vulns = data.get("vulns", [])

bbot/modules/ip2location.py

@@ -32,12 +32,10 @@ class IP2Location(BaseModule):
 
     async def ping(self):
         url = self.build_url("8.8.8.8")
-        r = await self.request_with_fail_count(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
+        await super().ping(url)
 
     def build_url(self, data):
-        url = f"{self.base_url}/?key={self.api_key}&ip={data}&format=json&source=bbot"
+        url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot"
         if self.lang:
             url = f"{url}&lang={self.lang}"
         return url
@@ -45,7 +43,7 @@ class IP2Location(BaseModule):
     async def handle_event(self, event):
         try:
             url = self.build_url(event.data)
-            result = await self.request_with_fail_count(url)
+            result = await self.api_request(url)
             if result:
                 geo_data = result.json()
                 if not geo_data:

bbot/modules/ipneighbor.py

@@ -31,7 +31,7 @@ class ipneighbor(BaseModule):
         netmask = main_ip.max_prefixlen - min(main_ip.max_prefixlen, self.num_bits)
         network = ipaddress.ip_network(f"{main_ip}/{netmask}", strict=False)
         subnet_hash = hash(network)
-        if not subnet_hash in self.processed:
+        if subnet_hash not in self.processed:
             self.processed.add(subnet_hash)
             for ip in network:
                 if ip != main_ip: