bbot 2.2.0.5279rc0__py3-none-any.whl → 2.2.0.5309rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.2.0.5279rc"
+__version__ = "v2.2.0.5309rc"
 
 from .scanner import Scanner, Preset

bbot/cli.py

@@ -174,7 +174,7 @@ async def _main():
             if sys.stdin.isatty():
 
                 # warn if any targets belong directly to a cloud provider
-                for event in scan.target.events:
+                for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":
                         cloudcheck_result = scan.helpers.cloudcheck(event.host)
                         if cloudcheck_result:

bbot/core/engine.py

@@ -641,7 +641,7 @@ class EngineServer(EngineBase):
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):
                     self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})")
-                    for task in tasks:
+                    for task in list(tasks):
                         task.cancel()
                         self._await_cancelled_task(task)
                 else:
@@ -683,5 +683,5 @@ class EngineServer(EngineBase):
         for client_id in list(self.tasks):
             await self.cancel_task(client_id)
         for client_id, tasks in self.child_tasks.items():
-            for task in tasks:
+            for task in list(tasks):
                 await self._await_cancelled_task(task)

bbot/core/event/base.py

@@ -341,6 +341,21 @@ class BaseEvent:
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -1114,8 +1129,7 @@ class DnsEvent(BaseEvent):
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1689,6 +1703,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 

bbot/core/helpers/bloom.py

@@ -64,8 +64,15 @@ class BloomFilter:
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

bbot/core/helpers/dns/helpers.py

@@ -1,6 +1,6 @@
 import logging
 
-from bbot.core.helpers.regexes import dns_name_regex
+from bbot.core.helpers.regexes import dns_name_extraction_regex
 from bbot.core.helpers.misc import clean_dns_record, smart_decode
 
 log = logging.getLogger("bbot.core.helpers.dns")
@@ -198,7 +198,7 @@ def extract_targets(record):
     elif rdtype == "TXT":
         for s in record.strings:
             s = smart_decode(s)
-            for match in dns_name_regex.finditer(s):
+            for match in dns_name_extraction_regex.finditer(s):
                 start, end = match.span()
                 host = s[start:end]
                 add_result(rdtype, host)

bbot/core/helpers/helper.py

@@ -12,10 +12,11 @@ from .diff import HttpCompare
 from .regex import RegexHelper
 from .wordcloud import WordCloud
 from .interactsh import Interactsh
-from ...scanner.target import Target
 from .depsinstaller import DepsInstaller
 from .async_helpers import get_event_loop
 
+from bbot.scanner.target import BaseTarget
+
 log = logging.getLogger("bbot.core.helpers")
 
 
@@ -155,8 +156,8 @@ class ConfigAwareHelper:
         _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events, **kwargs):
-        return Target(*events, **kwargs)
+    def make_target(self, *targets, **kwargs):
+        return BaseTarget(*targets, scan=self.scan, **kwargs)
 
     @property
     def config(self):

bbot/core/helpers/misc.py

@@ -586,17 +586,18 @@ def is_dns_name(d, include_local=True):
     if include_local:
         if bbot_regexes.hostname_regex.match(d):
             return True
-    if bbot_regexes.dns_name_regex.match(d):
+    if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False
 
 
-def is_ip(d, version=None):
+def is_ip(d, version=None, include_network=False):
     """
     Checks if the given string or object represents a valid IP address.
 
     Args:
         d (str or ipaddress.IPvXAddress): The IP address to check.
+        include_network (bool, optional): Whether to include network types (IPv4Network or IPv6Network). Defaults to False.
         version (int, optional): The IP version to validate (4 or 6). Default is None.
 
     Returns:
@@ -612,12 +613,17 @@ def is_ip(d, version=None):
         >>> is_ip('evilcorp.com')
         False
     """
+    ip = None
     try:
         ip = ipaddress.ip_address(d)
-        if version is None or ip.version == version:
-            return True
     except Exception:
-        pass
+        if include_network:
+            try:
+                ip = ipaddress.ip_network(d, strict=False)
+            except Exception:
+                pass
+    if ip is not None and (version is None or ip.version == version):
+        return True
     return False
 
 

bbot/core/helpers/regexes.py

@@ -40,7 +40,8 @@ ip_range_regexes = list(re.compile(r, re.I) for r in _ip_range_regexes)
 
 # dns names with periods
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
-dns_name_regex = re.compile(_dns_name_regex, re.I)
+dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
+dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
 # dns names without periods
 _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"

bbot/core/helpers/web/web.py

@@ -58,7 +58,7 @@ class WebHelper(EngineClient):
         self.ssl_verify = self.config.get("ssl_verify", False)
         engine_debug = self.config.get("engine", {}).get("debug", False)
         super().__init__(
-            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.radix_only},
+            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.minimal},
             debug=engine_debug,
         )
 

bbot/modules/anubisdb.py

@@ -38,7 +38,7 @@ class anubisdb(subdomain_enum):
             return True, "DNS name is unresolved"
         return await super().abort_if(event)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json:

bbot/modules/baddns.py

@@ -116,7 +116,7 @@ class baddns(BaseModule):
                                 context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
                         else:
-                            self.warning(f"Got unrecognized confidence level: {r['confidence']}")
+                            self.warning(f"Got unrecognized confidence level: {r_dict['confidence']}")
 
                         found_domains = r_dict.get("found_domains", None)
                         if found_domains:

bbot/modules/bevigil.py

@@ -60,14 +60,14 @@ class bevigil(subdomain_enum_apikey):
         url = f"{self.base_url}/{self.helpers.quote(query)}/urls/"
         return await self.api_request(url)
 
-    def parse_subdomains(self, r, query=None):
+    async def parse_subdomains(self, r, query=None):
         results = set()
         subdomains = r.json().get("subdomains")
         if subdomains:
             results.update(subdomains)
         return results
 
-    def parse_urls(self, r, query=None):
+    async def parse_urls(self, r, query=None):
         results = set()
         urls = r.json().get("urls")
         if urls:

bbot/modules/binaryedge.py

@@ -37,6 +37,6 @@ class binaryedge(subdomain_enum_apikey):
         url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         return j.get("events", [])

bbot/modules/bufferoverrun.py

@@ -33,7 +33,7 @@ class BufferOverrun(subdomain_enum_apikey):
         url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):
@@ -44,5 +44,4 @@ class BufferOverrun(subdomain_enum_apikey):
                     subdomain = parts[4].strip()
                     if subdomain and subdomain.endswith(f".{query}"):
                         subdomains_set.add(subdomain)
-            for subdomain in subdomains_set:
-                yield subdomain
+        return subdomains_set

bbot/modules/builtwith.py

@@ -62,7 +62,7 @@ class builtwith(subdomain_enum_apikey):
         url = f"{self.base_url}/redirect1/api.json?KEY={{api_key}}&LOOKUP={query}"
         return await self.api_request(url)
 
-    def parse_domains(self, r, query):
+    async def parse_domains(self, r, query):
         """
         This method returns a set of subdomains.
         Each subdomain is an "FQDN" that was reported in the "Detailed Technology Profile" page on builtwith.com
@@ -92,7 +92,7 @@ class builtwith(subdomain_enum_apikey):
                     self.verbose(f"No results for {query}: {error}")
         return results_set
 
-    def parse_redirects(self, r, query):
+    async def parse_redirects(self, r, query):
         """
         This method creates a set.
         Each entry in the set is either an Inbound or Outbound Redirect reported in the "Redirect Profile" page on builtwith.com

bbot/modules/c99.py

@@ -26,7 +26,8 @@ class c99(subdomain_enum_apikey):
         url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             subdomains = j.get("subdomains", [])
@@ -34,4 +35,5 @@ class c99(subdomain_enum_apikey):
                 for s in subdomains:
                     subdomain = s.get("subdomain", "")
                     if subdomain:
-                        yield subdomain
+                        results.add(subdomain)
+        return results

bbot/modules/certspotter.py

@@ -17,9 +17,11 @@ class certspotter(subdomain_enum):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
         return self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         json = r.json()
         if json:
             for r in json:
                 for dns_name in r.get("dns_names", []):
-                    yield dns_name.lstrip(".*").rstrip(".")
+                    results.add(dns_name.lstrip(".*").rstrip("."))
+        return results

bbot/modules/chaos.py

@@ -26,7 +26,8 @@ class chaos(subdomain_enum_apikey):
         url = f"{self.base_url}/{domain}/subdomains"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):
@@ -39,4 +40,5 @@ class chaos(subdomain_enum_apikey):
                 for s in subdomains_set:
                     full_subdomain = f"{s}.{domain}"
                     if full_subdomain and full_subdomain.endswith(f".{query}"):
-                        yield full_subdomain
+                        results.add(full_subdomain)
+        return results

bbot/modules/columbus.py

@@ -17,7 +17,7 @@ class columbus(subdomain_enum):
         url = f"{self.base_url}/{self.helpers.quote(query)}?days=365"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

bbot/modules/crt.py

@@ -23,7 +23,8 @@ class crt(subdomain_enum):
         url = self.helpers.add_get_params(self.base_url, params).geturl()
         return await self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         for cert_info in j:
             if not type(cert_info) == dict:
@@ -35,4 +36,5 @@ class crt(subdomain_enum):
                     domain = cert_info.get("name_value")
                     if domain:
                         for d in domain.splitlines():
-                            yield d.lower()
+                            results.add(d.lower())
+        return results

bbot/modules/digitorus.py

@@ -19,7 +19,7 @@ class digitorus(subdomain_enum):
         url = f"{self.base_url}/{self.helpers.quote(query)}"
         return await self.helpers.request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         content = getattr(r, "text", "")
         extract_regex = re.compile(r"[\w.-]+\." + query, re.I)

bbot/modules/dnscaa.py

@@ -2,7 +2,7 @@
 #
 # Checks for and parses CAA DNS TXT records for IODEF reporting destination email addresses and/or URL's.
 #
-# NOTE: when the target domain is initially resolved basic "dns_name_regex" matched targets will be extracted so we do not perform that again here.
+# NOTE: when the target domain is initially resolved basic "dns_name_extraction_regex" matched targets will be extracted so we do not perform that again here.
 #
 # Example CAA records,
 #   0 iodef "mailto:dnsadmin@example.com"
@@ -23,7 +23,7 @@ from bbot.modules.base import BaseModule
 
 import re
 
-from bbot.core.helpers.regexes import dns_name_regex, email_regex, url_regexes
+from bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes
 
 # Handle '0 iodef "mailto:support@hcaptcha.com"'
 # Handle '1 iodef "https://some.host.tld/caa;"'
@@ -109,7 +109,7 @@ class dnscaa(BaseModule):
 
                     elif caa_match.group("property").lower().startswith("issue"):
                         if self._dns_names:
-                            for match in dns_name_regex.finditer(caa_match.group("text")):
+                            for match in dns_name_extraction_regex.finditer(caa_match.group("text")):
                                 start, end = match.span()
                                 name = caa_match.group("text")[start:end]
 

bbot/modules/fullhunt.py

@@ -35,5 +35,5 @@ class fullhunt(subdomain_enum_apikey):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         return r.json().get("hosts", [])

bbot/modules/hackertarget.py

@@ -18,12 +18,14 @@ class hackertarget(subdomain_enum):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for line in r.text.splitlines():
             host = line.split(",")[0]
             try:
                 self.helpers.validators.validate_host(host)
-                yield host
+                results.add(host)
             except ValueError:
                 self.debug(f"Error validating API result: {line}")
                 continue
+        return results

bbot/modules/internal/excavate.py

@@ -527,9 +527,8 @@ class excavate(BaseInternalModule, BaseInterceptModule):
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
             for identifier in yara_results.keys():
                 for csp_str in yara_results[identifier]:
-                    domains = await self.helpers.re.findall(bbot_regexes.dns_name_regex, csp_str)
-                    unique_domains = set(domains)
-                    for domain in unique_domains:
+                    domains = await self.excavate.scan.extract_in_scope_hostnames(csp_str)
+                    for domain in domains:
                         await self.report(domain, event, yara_rule_settings, discovery_context, event_type="DNS_NAME")
 
     class EmailExtractor(ExcavateRule):

bbot/modules/internal/speculate.py

@@ -65,7 +65,7 @@ class speculate(BaseInternalModule):
         if not self.portscanner_enabled:
             self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
 
-        target_len = len(self.scan.target)
+        target_len = len(self.scan.target.seeds)
         if target_len > self.config.get("max_hosts", 65536):
             if not self.portscanner_enabled:
                 self.hugewarning(

bbot/modules/leakix.py

@@ -35,10 +35,12 @@ class leakix(subdomain_enum_apikey):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
+        results = set()
         json = r.json()
         if json:
             for entry in json:
                 subdomain = entry.get("subdomain", "")
                 if subdomain:
-                    yield subdomain
+                    results.add(subdomain)
+        return results

bbot/modules/myssl.py

@@ -17,7 +17,7 @@ class myssl(subdomain_enum):
         url = f"{self.base_url}?domain={self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, dict):

bbot/modules/otx.py

@@ -17,10 +17,12 @@ class otx(subdomain_enum):
         url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
         return self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for entry in j.get("passive_dns", []):
                 subdomain = entry.get("hostname", "")
                 if subdomain:
-                    yield subdomain
+                    results.add(subdomain)
+        return results

bbot/modules/passivetotal.py

@@ -39,6 +39,8 @@ class passivetotal(subdomain_enum_apikey):
         url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for subdomain in r.json().get("subdomains", []):
-            yield f"{subdomain}.{query}"
+            results.add(f"{subdomain}.{query}")
+        return results

bbot/modules/rapiddns.py

@@ -18,11 +18,6 @@ class rapiddns(subdomain_enum):
         response = await self.api_request(url, timeout=self.http_timeout + 10)
         return response
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/securitytrails.py

@@ -26,8 +26,10 @@ class securitytrails(subdomain_enum_apikey):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for host in j.get("subdomains", []):
-                yield f"{host}.{query}"
+                results.add(f"{host}.{query}")
+        return results

bbot/modules/shodan_dns.py

@@ -22,5 +22,5 @@ class shodan_dns(shodan):
     def make_url(self, query):
         return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
 
-    def parse_results(self, json, query):
+    async def parse_results(self, json, query):
         return [f"{sub}.{query}" for sub in json.get("subdomains", [])]

bbot/modules/subdomaincenter.py

@@ -33,7 +33,7 @@ class subdomaincenter(subdomain_enum):
                 break
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

bbot/modules/templates/subdomain_enum.py

@@ -106,7 +106,7 @@ class subdomain_enum(BaseModule):
                 break
         return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for hostname in json:
@@ -123,7 +123,7 @@ class subdomain_enum(BaseModule):
                 self.info(f'Query "{query}" failed (no response)')
                 return []
             try:
-                results = list(parse_fn(response, query))
+                results = list(await parse_fn(response, query))
             except Exception as e:
                 if response:
                     self.info(
@@ -144,7 +144,7 @@ class subdomain_enum(BaseModule):
         agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)
         try:
             async for response in agen:
-                subdomains = self.parse_results(response, query)
+                subdomains = await self.parse_results(response, query)
                 self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"')
                 if not subdomains:
                     break

bbot/modules/trickest.py

@@ -36,7 +36,7 @@ class Trickest(subdomain_enum_apikey):
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
         return url
 
-    def parse_results(self, j, query):
+    async def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

bbot/modules/virustotal.py

@@ -24,11 +24,6 @@ class virustotal(subdomain_enum_apikey):
         kwargs["headers"]["x-apikey"] = self.api_key
         return url, kwargs
 
-    def parse_results(self, r, query):
-        results = set()
+    async def parse_results(self, r, query):
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)

bbot/modules/zoomeye.py

@@ -60,7 +60,7 @@ class zoomeye(subdomain_enum_apikey):
         agen = self.api_page_iter(url)
         try:
             async for j in agen:
-                r = list(self.parse_results(j))
+                r = list(await self.parse_results(j))
                 if r:
                     results.update(set(r))
                 if not r or i >= (self.max_pages - 1):
@@ -70,6 +70,8 @@ class zoomeye(subdomain_enum_apikey):
             agen.aclose()
         return results
 
-    def parse_results(self, r):
+    async def parse_results(self, r):
+        results = set()
         for entry in r.get("list", []):
-            yield entry["name"]
+            results.add(entry["name"])
+        return results

bbot/presets/spider.yml

@@ -3,6 +3,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row