bbot 2.6.0.6840rc0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.6.0.6840rc"
+__version__ = "v2.7.2.7424rc"
 
 from .scanner import Scanner, Preset
 

bbot/cli.py

@@ -7,7 +7,7 @@ import multiprocessing
 from bbot.errors import *
 from bbot import __version__
 from bbot.logger import log_to_stderr
-from bbot.core.helpers.misc import chain_lists
+from bbot.core.helpers.misc import chain_lists, rm_rf
 
 
 if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
 
         # --install-all-deps
         if options.install_all_deps:
-            all_modules = list(preset.module_loader.preloaded())
-            scan.helpers.depsinstaller.force_deps = True
-            succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            if failed:
-                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+            preloaded_modules = preset.module_loader.preloaded()
+            scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
+            output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
+            log.verbose("Creating dummy scan with all modules + output modules for deps installation")
+            dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
+            dummy_scan.helpers.depsinstaller.force_deps = True
+            log.info("Installing module dependencies")
+            await dummy_scan.load_modules()
+            log.verbose("Running module setups")
+            succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
+            # remove any leftovers from the dummy scan
+            rm_rf(dummy_scan.home, ignore_errors=True)
+            rm_rf(dummy_scan.temp_dir, ignore_errors=True)
+            if succeeded:
+                log.success(
+                    f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
+                )
+            if soft_failed or hard_failed:
+                failed = soft_failed + hard_failed
+                log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
                 return False
-            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
             return True
 
         scan_name = str(scan.name)
@@ -201,7 +215,7 @@ async def _main():
                 if not scan.preset.strict_scope:
                     for event in scan.target.seeds.event_seeds:
                         if event.type == "DNS_NAME":
-                            cloudcheck_result = scan.helpers.cloudcheck(event.host)
+                            cloudcheck_result = await scan.helpers.cloudcheck.lookup(event.host)
                             if cloudcheck_result:
                                 scan.hugewarning(
                                     f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!'

bbot/core/engine.py

@@ -636,7 +636,7 @@ class EngineServer(EngineBase):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/event/__init__.py

@@ -1,3 +1,3 @@
-from .base import make_event, is_event, event_from_json
+from .base import make_event, update_event, is_event, event_from_json
 
-__all__ = ["make_event", "is_event", "event_from_json"]
+__all__ = ["make_event", "update_event", "is_event", "event_from_json"]

bbot/core/event/base.py

@@ -789,26 +789,32 @@ class BaseEvent:
 
     def __contains__(self, other):
         """
-        Allows events to be compared using the "in" operator:
-        E.g.:
-            if some_event in other_event:
-                ...
+        Membership checks for Events.
+
+        Supports:
+            - some_event in other_event   (event vs event)
+            - "host:port" in other_event  (string coerced to an event)
         """
-        try:
-            other = make_event(other, dummy=True)
-        except ValidationError:
-            return False
+        # Fast path: already an Event
+        if is_event(other):
+            other_event = other
+        else:
+            try:
+                other_event = make_event(other, dummy=True)
+            except ValidationError:
+                return False
+
         # if hashes match
-        if other == self:
+        if other_event == self:
             return True
-        # if hosts match
-        if self.host and other.host:
-            if self.host == other.host:
+        # if hosts match (including subnet / domain containment)
+        if self.host and other_event.host:
+            if self.host == other_event.host:
                 return True
             # hostnames and IPs
             radixtarget = RadixTarget()
             radixtarget.insert(self.host)
-            return bool(radixtarget.search(other.host))
+            return bool(radixtarget.search(other_event.host))
         return False
 
     def json(self, mode="json", siem_friendly=False):
@@ -996,10 +1002,14 @@ class BaseEvent:
         return self.priority > getattr(other, "priority", (0,))
 
     def __eq__(self, other):
-        try:
-            other = make_event(other, dummy=True)
-        except ValidationError:
-            return False
+        """
+        Event equality is **only** defined between Event instances.
+
+        Equality is based on the event hash (derived from its id). Comparisons to
+        non-Event types raise a ValueError to make incorrect comparisons explicit.
+        """
+        if not is_event(other):
+            raise ValueError("Event equality is only defined between Event instances")
         return hash(self) == hash(other)
 
     def __hash__(self):
@@ -1748,6 +1758,55 @@ class MOBILE_APP(DictEvent):
         return self.data["url"]
 
 
+def update_event(
+    event,
+    parent=None,
+    context=None,
+    module=None,
+    scan=None,
+    tags=None,
+    internal=None,
+):
+    """
+    Updates an existing event object with additional metadata.
+
+    Parameters:
+        event (BaseEvent): The event object to update.
+        parent (BaseEvent, optional): New parent event.
+        context (str, optional): Discovery context to set.
+        module (str or BaseModule, optional): Module that discovered the event.
+        scan (Scan, optional): BBOT Scan object associated with the event.
+        tags (Union[str, List[str]], optional): Tags to merge into the event.
+        internal (Any, optional): Marks the event as internal if True.
+
+    Returns:
+        BaseEvent: The updated event object.
+    """
+    if not is_event(event):
+        raise ValidationError(f"update_event() expects an Event, got {type(event)}")
+
+    # allow tags to be either a string or an array
+    if not tags:
+        tags = []
+    elif isinstance(tags, str):
+        tags = [tags]
+    tags = set(tags)
+
+    if scan is not None and not event.scan:
+        event.scan = scan
+    if module is not None:
+        event.module = module
+    if parent is not None:
+        event.parent = parent
+    if context is not None:
+        event.discovery_context = context
+    if internal is True:
+        event.internal = True
+    if tags:
+        event.tags = tags.union(event.tags)
+    return event
+
+
 def make_event(
     data,
     event_type=None,
@@ -1761,14 +1820,13 @@ def make_event(
     internal=None,
 ):
     """
-    Creates and returns a new event object or modifies an existing one.
+    Creates and returns a new event object.
 
-    This function serves as a factory for creating new event objects, either by generating a new `Event`
-    object or by updating an existing event with additional metadata. If `data` is already an event,
-    it updates the event based on the additional parameters provided.
+    This function serves as a factory for creating new event objects from raw data.
+    If you need to modify an existing event, use ``update_event()`` instead.
 
     Parameters:
-        data (Union[str, dict, BaseEvent]): The primary data for the event or an existing event object.
+        data (Union[str, dict]): The primary data for the event.
         event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided.
         parent (BaseEvent, optional): Parent event leading to this event's discovery.
         context (str, optional): Description of circumstances leading to event's discovery.
@@ -1781,32 +1839,20 @@ def make_event(
         internal (Any, optional): Makes the event internal if set to True. Defaults to None.
 
     Returns:
-        BaseEvent: A new or updated event object.
+        BaseEvent: A new event object.
 
     Raises:
         ValidationError: Raised when there's an error in event data or type sanitization.
-
-    Examples:
-        If inside a module, e.g. from within its `handle_event()`:
-        >>> self.make_event("1.2.3.4", parent=event)
-        IP_ADDRESS("1.2.3.4", module=portscan, tags={'ipv4', 'distance-1'})
-
-        If you're outside a module but you have a scan object:
-        >>> scan.make_event("1.2.3.4", parent=scan.root_event)
-        IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4', 'distance-1'})
-
-        If you're outside a scan and just messing around:
-        >>> from bbot.core.event.base import make_event
-        >>> make_event("1.2.3.4", dummy=True)
-        IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4'})
-
-    Note:
-        When working within a module's `handle_event()`, use the instance method
-        `self.make_event()` instead of calling this function directly.
     """
     if not data:
         raise ValidationError("No data provided")
 
+    # do not allow passing an existing event here – use update_event() instead
+    if is_event(data):
+        raise ValidationError(
+            "make_event() does not accept an existing event object. Use update_event(event, ...) to modify an event."
+        )
+
     # allow tags to be either a string or an array
     if not tags:
         tags = []
@@ -1814,76 +1860,58 @@ def make_event(
         tags = [tags]
     tags = set(tags)
 
-    # if data is already an event, update it with the user's kwargs
-    if is_event(data):
-        event = copy(data)
-        if scan is not None and not event.scan:
-            event.scan = scan
-        if module is not None:
-            event.module = module
-        if parent is not None:
-            event.parent = parent
-        if context is not None:
-            event.discovery_context = context
-        if internal is True:
-            event.internal = True
-        if tags:
-            event.tags = tags.union(event.tags)
-        event_type = data.type
-        return event
-    else:
-        # if event_type is not provided, autodetect it
-        if event_type is None:
-            event_seed = EventSeed(data)
-            event_type = event_seed.type
-            data = event_seed.data
-            if not dummy:
-                log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
-
-        event_type = str(event_type).strip().upper()
-
-        # Catch these common whoopsies
-        if event_type in ("DNS_NAME", "IP_ADDRESS"):
-            # DNS_NAME <--> EMAIL_ADDRESS confusion
-            if validators.soft_validate(data, "email"):
-                event_type = "EMAIL_ADDRESS"
-            else:
-                # DNS_NAME <--> IP_ADDRESS confusion
-                try:
-                    data = validators.validate_host(data)
-                except Exception as e:
-                    log.trace(traceback.format_exc())
-                    raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}')
-                data_is_ip = is_ip(data)
-                if event_type == "DNS_NAME" and data_is_ip:
-                    event_type = "IP_ADDRESS"
-                elif event_type == "IP_ADDRESS" and not data_is_ip:
-                    event_type = "DNS_NAME"
-        # USERNAME <--> EMAIL_ADDRESS confusion
-        if event_type == "USERNAME" and validators.soft_validate(data, "email"):
+    # if event_type is not provided, autodetect it
+    if event_type is None:
+        event_seed = EventSeed(data)
+        event_type = event_seed.type
+        data = event_seed.data
+        if not dummy:
+            log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
+
+    event_type = str(event_type).strip().upper()
+
+    # Catch these common whoopsies
+    if event_type in ("DNS_NAME", "IP_ADDRESS"):
+        # DNS_NAME <--> EMAIL_ADDRESS confusion
+        if validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
-            tags.add("affiliate")
-        # Convert single-host IP_RANGE to IP_ADDRESS
-        if event_type == "IP_RANGE":
-            with suppress(Exception):
-                net = ipaddress.ip_network(data, strict=False)
-                if net.prefixlen == net.max_prefixlen:
-                    event_type = "IP_ADDRESS"
-                    data = net.network_address
-
-        event_class = globals().get(event_type, DefaultEvent)
-        return event_class(
-            data,
-            event_type=event_type,
-            parent=parent,
-            context=context,
-            module=module,
-            scan=scan,
-            tags=tags,
-            confidence=confidence,
-            _dummy=dummy,
-            _internal=internal,
-        )
+        else:
+            # DNS_NAME <--> IP_ADDRESS confusion
+            try:
+                data = validators.validate_host(data)
+            except Exception as e:
+                log.trace(traceback.format_exc())
+                raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}')
+            data_is_ip = is_ip(data)
+            if event_type == "DNS_NAME" and data_is_ip:
+                event_type = "IP_ADDRESS"
+            elif event_type == "IP_ADDRESS" and not data_is_ip:
+                event_type = "DNS_NAME"
+    # USERNAME <--> EMAIL_ADDRESS confusion
+    if event_type == "USERNAME" and validators.soft_validate(data, "email"):
+        event_type = "EMAIL_ADDRESS"
+        tags.add("affiliate")
+    # Convert single-host IP_RANGE to IP_ADDRESS
+    if event_type == "IP_RANGE":
+        with suppress(Exception):
+            net = ipaddress.ip_network(data, strict=False)
+            if net.prefixlen == net.max_prefixlen:
+                event_type = "IP_ADDRESS"
+                data = net.network_address
+
+    event_class = globals().get(event_type, DefaultEvent)
+    return event_class(
+        data,
+        event_type=event_type,
+        parent=parent,
+        context=context,
+        module=module,
+        scan=scan,
+        tags=tags,
+        confidence=confidence,
+        _dummy=dummy,
+        _internal=internal,
+    )
 
 
 def event_from_json(j, siem_friendly=False):

bbot/core/flags.py

@@ -6,6 +6,7 @@ flag_descriptions = {
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ class BloomFilter:
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""

bbot/core/helpers/depsinstaller/installer.py

@@ -2,6 +2,8 @@ import os
 import sys
 import stat
 import json
+import mmh3
+import orjson
 import shutil
 import getpass
 import logging
@@ -14,6 +16,7 @@ from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
+from bbot import __version__
 from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
@@ -68,7 +71,7 @@ class DepsInstaller:
             },
         ],
         # to compile just about any tool, we need the openssl dev headers
-        "openssl": [
+        "openssl_dev_headers": [
             {
                 "name": "Install OpenSSL library and development headers (Debian/Ubuntu)",
                 "package": {"name": ["libssl-dev", "openssl"], "state": "present"},
@@ -172,6 +175,7 @@ class DepsInstaller:
                     + self.venv
                     + str(self.parent_helper.bbot_home)
                     + os.uname()[1]
+                    + str(__version__)
                 ).hexdigest()
                 success = self.setup_status.get(module_hash, None)
                 dependencies = list(chain(*preloaded["deps"].values()))
@@ -439,6 +443,13 @@ class DepsInstaller:
                     log.warning("Incorrect password")
 
     async def install_core_deps(self):
+        # skip if we've already successfully installed core deps for this definition
+        core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS)))
+        core_deps_cache_file = self.parent_helper.cache_dir / core_deps_hash
+        if core_deps_cache_file.exists():
+            log.debug("Skipping core dependency installation (cache hit)")
+            return
+
         to_install = set()
         to_install_friendly = set()
         playbook = []
@@ -454,6 +465,7 @@ class DepsInstaller:
                 else:
                     playbook.extend(package_name_or_playbook)
         # install ansible community.general collection
+        overall_success = True
         if not self.setup_status.get("ansible:community.general", False):
             log.info("Installing Ansible Community General Collection")
             try:
@@ -465,6 +477,7 @@ class DepsInstaller:
                 log.warning(
                     f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}"
                 )
+                overall_success = False
         # construct ansible playbook
         if to_install:
             playbook.append(
@@ -478,7 +491,13 @@ class DepsInstaller:
         if playbook:
             log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()
-            self.ansible_run(tasks=playbook)
+            success, _ = self.ansible_run(tasks=playbook)
+            overall_success &= success
+
+        # mark cache only if everything succeeded (or nothing needed doing)
+        if overall_success:
+            with suppress(Exception):
+                core_deps_cache_file.touch()
 
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup:

bbot/core/helpers/dns/dns.py

@@ -38,7 +38,6 @@ class DNSHelper(EngineClient):
         _wildcard_cache (dict): Cache for wildcard detection results.
         _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.
         resolver_file (Path): File containing system's current resolver nameservers.
-        filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True.
 
     Args:
         parent_helper: The parent helper object with configuration details and utilities.

bbot/core/helpers/dns/engine.py

@@ -86,8 +86,6 @@ class DNSEngine(EngineServer):
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
-        self.filter_bad_ptrs = self.dns_config.get("filter_ptrs", True)
-
     async def resolve(self, query, **kwargs):
         """Resolve DNS names and IP addresses to their corresponding results.
 

bbot/core/helpers/files.py

@@ -9,7 +9,7 @@ from .misc import rm_at_exit
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):

bbot/core/helpers/git.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def sanitize_git_repo(repo_folder: Path):
+    # sanitizing the git config is infeasible since there are too many different ways to do evil things
+    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    config_file = repo_folder / ".git" / "config"
+    if config_file.exists():
+        config_file.rename(repo_folder / "git_config_original")
+    # move the index file
+    index_file = repo_folder / ".git" / "index"
+    if index_file.exists():
+        index_file.rename(repo_folder / "git_index_original")
+    # move the hooks folder
+    hooks_folder = repo_folder / ".git" / "hooks"
+    if hooks_folder.exists():
+        hooks_folder.rename(repo_folder / "git_hooks_original")

bbot/core/helpers/helper.py

@@ -89,6 +89,7 @@ class ConfigAwareHelper:
         self.yara = YaraHelper(self)
         self._dns = None
         self._web = None
+        self._cloudcheck = None
         self.config_aware_validators = self.validators.Validators(self)
         self.depsinstaller = DepsInstaller(self)
         self.word_cloud = WordCloud(self)
@@ -107,12 +108,12 @@ class ConfigAwareHelper:
         return self._web
 
     @property
-    def cloud(self):
-        if self._cloud is None:
-            from cloudcheck import cloud_providers
+    def cloudcheck(self):
+        if self._cloudcheck is None:
+            from cloudcheck import CloudCheck
 
-            self._cloud = cloud_providers
-        return self._cloud
+            self._cloudcheck = CloudCheck()
+        return self._cloudcheck
 
     def bloom_filter(self, size):
         from .bloom import BloomFilter

bbot/core/helpers/misc.py

@@ -17,6 +17,7 @@ from unidecode import unidecode  # noqa F401
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
+from .git import *  # noqa F401
 from .url import *  # noqa F401
 from ... import errors
 from . import regexes as bbot_regexes
@@ -216,26 +217,29 @@ def split_host_port(d):
     host = None
     port = None
     scheme = None
+
+    # first, try to parse as an IP address
     if is_ip(d):
         return make_ip_type(d), port
 
+    # if not an IP address, try to parse as a host:port
     match = bbot_regexes.split_host_port_regex.match(d)
     if match is None:
-        raise ValueError(f'split_port() failed to parse "{d}"')
+        raise ValueError(f'split_host_port() failed to parse "{d}"')
     scheme = match.group("scheme")
     netloc = match.group("netloc")
     if netloc is None:
-        raise ValueError(f'split_port() failed to parse "{d}"')
+        raise ValueError(f'split_host_port() failed to parse "{d}"')
 
     match = bbot_regexes.extract_open_port_regex.match(netloc)
     if match is None:
-        raise ValueError(f'split_port() failed to parse netloc "{netloc}" (original value: {d})')
+        raise ValueError(f'split_host_port() failed to parse netloc "{netloc}" (original value: {d})')
 
     host = match.group(2)
     if host is None:
         host = match.group(1)
     if host is None:
-        raise ValueError(f'split_port() failed to locate host in netloc "{netloc}" (original value: {d})')
+        raise ValueError(f'split_host_port() failed to locate host in netloc "{netloc}" (original value: {d})')
 
     port = match.group(3)
     if port is None and scheme is not None:
@@ -2288,25 +2292,6 @@ def is_file(f):
     return False
 
 
-def cloudcheck(ip):
-    """
-    Check whether an IP address belongs to a cloud provider and returns the provider name, type, and subnet.
-
-    Args:
-        ip (str): The IP address to check.
-
-    Returns:
-        tuple: A tuple containing provider name (str), provider type (str), and subnet (IPv4Network).
-
-    Examples:
-        >>> cloudcheck("168.62.20.37")
-        ('Azure', 'cloud', IPv4Network('168.62.0.0/19'))
-    """
-    import cloudcheck as _cloudcheck
-
-    return _cloudcheck.check(ip)
-
-
 def is_async_function(f):
     """
     Check if a given function is an asynchronous function.

bbot/core/helpers/ntlm.py

@@ -17,11 +17,9 @@ class StrStruct(object):
         self.alloc = alloc
         self.offset = offset
         self.raw = raw[offset : offset + length]
-        self.utf16 = False
 
         if len(self.raw) >= 2 and self.raw[1] == "\0":
             self.string = self.raw.decode("utf-16")
-            self.utf16 = True
         else:
             self.string = self.raw