bbot 2.5.0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.5.0"
+__version__ = "v2.7.2.7424rc"
 
 from .scanner import Scanner, Preset
 

bbot/cli.py

@@ -7,7 +7,7 @@ import multiprocessing
 from bbot.errors import *
 from bbot import __version__
 from bbot.logger import log_to_stderr
-from bbot.core.helpers.misc import chain_lists
+from bbot.core.helpers.misc import chain_lists, rm_rf
 
 
 if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
 
         # --install-all-deps
         if options.install_all_deps:
-            all_modules = list(preset.module_loader.preloaded())
-            scan.helpers.depsinstaller.force_deps = True
-            succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            if failed:
-                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+            preloaded_modules = preset.module_loader.preloaded()
+            scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
+            output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
+            log.verbose("Creating dummy scan with all modules + output modules for deps installation")
+            dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
+            dummy_scan.helpers.depsinstaller.force_deps = True
+            log.info("Installing module dependencies")
+            await dummy_scan.load_modules()
+            log.verbose("Running module setups")
+            succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
+            # remove any leftovers from the dummy scan
+            rm_rf(dummy_scan.home, ignore_errors=True)
+            rm_rf(dummy_scan.temp_dir, ignore_errors=True)
+            if succeeded:
+                log.success(
+                    f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
+                )
+            if soft_failed or hard_failed:
+                failed = soft_failed + hard_failed
+                log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
                 return False
-            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
             return True
 
         scan_name = str(scan.name)
@@ -201,7 +215,7 @@ async def _main():
                 if not scan.preset.strict_scope:
                     for event in scan.target.seeds.event_seeds:
                         if event.type == "DNS_NAME":
-                            cloudcheck_result = scan.helpers.cloudcheck(event.host)
+                            cloudcheck_result = await scan.helpers.cloudcheck.lookup(event.host)
                             if cloudcheck_result:
                                 scan.hugewarning(
                                     f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!'

bbot/core/engine.py

@@ -636,7 +636,7 @@ class EngineServer(EngineBase):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/event/__init__.py

@@ -1,3 +1,3 @@
-from .base import make_event, is_event, event_from_json
+from .base import make_event, update_event, is_event, event_from_json
 
-__all__ = ["make_event", "is_event", "event_from_json"]
+__all__ = ["make_event", "update_event", "is_event", "event_from_json"]

bbot/core/event/base.py

@@ -789,26 +789,32 @@ class BaseEvent:
 
     def __contains__(self, other):
         """
-        Allows events to be compared using the "in" operator:
-        E.g.:
-            if some_event in other_event:
-                ...
+        Membership checks for Events.
+
+        Supports:
+            - some_event in other_event   (event vs event)
+            - "host:port" in other_event  (string coerced to an event)
         """
-        try:
-            other = make_event(other, dummy=True)
-        except ValidationError:
-            return False
+        # Fast path: already an Event
+        if is_event(other):
+            other_event = other
+        else:
+            try:
+                other_event = make_event(other, dummy=True)
+            except ValidationError:
+                return False
+
         # if hashes match
-        if other == self:
+        if other_event == self:
             return True
-        # if hosts match
-        if self.host and other.host:
-            if self.host == other.host:
+        # if hosts match (including subnet / domain containment)
+        if self.host and other_event.host:
+            if self.host == other_event.host:
                 return True
             # hostnames and IPs
             radixtarget = RadixTarget()
             radixtarget.insert(self.host)
-            return bool(radixtarget.search(other.host))
+            return bool(radixtarget.search(other_event.host))
         return False
 
     def json(self, mode="json", siem_friendly=False):
@@ -996,10 +1002,14 @@ class BaseEvent:
         return self.priority > getattr(other, "priority", (0,))
 
     def __eq__(self, other):
-        try:
-            other = make_event(other, dummy=True)
-        except ValidationError:
-            return False
+        """
+        Event equality is **only** defined between Event instances.
+
+        Equality is based on the event hash (derived from its id). Comparisons to
+        non-Event types raise a ValueError to make incorrect comparisons explicit.
+        """
+        if not is_event(other):
+            raise ValueError("Event equality is only defined between Event instances")
         return hash(self) == hash(other)
 
     def __hash__(self):
@@ -1748,6 +1758,55 @@ class MOBILE_APP(DictEvent):
         return self.data["url"]
 
 
+def update_event(
+    event,
+    parent=None,
+    context=None,
+    module=None,
+    scan=None,
+    tags=None,
+    internal=None,
+):
+    """
+    Updates an existing event object with additional metadata.
+
+    Parameters:
+        event (BaseEvent): The event object to update.
+        parent (BaseEvent, optional): New parent event.
+        context (str, optional): Discovery context to set.
+        module (str or BaseModule, optional): Module that discovered the event.
+        scan (Scan, optional): BBOT Scan object associated with the event.
+        tags (Union[str, List[str]], optional): Tags to merge into the event.
+        internal (Any, optional): Marks the event as internal if True.
+
+    Returns:
+        BaseEvent: The updated event object.
+    """
+    if not is_event(event):
+        raise ValidationError(f"update_event() expects an Event, got {type(event)}")
+
+    # allow tags to be either a string or an array
+    if not tags:
+        tags = []
+    elif isinstance(tags, str):
+        tags = [tags]
+    tags = set(tags)
+
+    if scan is not None and not event.scan:
+        event.scan = scan
+    if module is not None:
+        event.module = module
+    if parent is not None:
+        event.parent = parent
+    if context is not None:
+        event.discovery_context = context
+    if internal is True:
+        event.internal = True
+    if tags:
+        event.tags = tags.union(event.tags)
+    return event
+
+
 def make_event(
     data,
     event_type=None,
@@ -1761,14 +1820,13 @@ def make_event(
     internal=None,
 ):
     """
-    Creates and returns a new event object or modifies an existing one.
+    Creates and returns a new event object.
 
-    This function serves as a factory for creating new event objects, either by generating a new `Event`
-    object or by updating an existing event with additional metadata. If `data` is already an event,
-    it updates the event based on the additional parameters provided.
+    This function serves as a factory for creating new event objects from raw data.
+    If you need to modify an existing event, use ``update_event()`` instead.
 
     Parameters:
-        data (Union[str, dict, BaseEvent]): The primary data for the event or an existing event object.
+        data (Union[str, dict]): The primary data for the event.
         event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided.
         parent (BaseEvent, optional): Parent event leading to this event's discovery.
         context (str, optional): Description of circumstances leading to event's discovery.
@@ -1781,32 +1839,20 @@ def make_event(
         internal (Any, optional): Makes the event internal if set to True. Defaults to None.
 
     Returns:
-        BaseEvent: A new or updated event object.
+        BaseEvent: A new event object.
 
     Raises:
         ValidationError: Raised when there's an error in event data or type sanitization.
-
-    Examples:
-        If inside a module, e.g. from within its `handle_event()`:
-        >>> self.make_event("1.2.3.4", parent=event)
-        IP_ADDRESS("1.2.3.4", module=portscan, tags={'ipv4', 'distance-1'})
-
-        If you're outside a module but you have a scan object:
-        >>> scan.make_event("1.2.3.4", parent=scan.root_event)
-        IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4', 'distance-1'})
-
-        If you're outside a scan and just messing around:
-        >>> from bbot.core.event.base import make_event
-        >>> make_event("1.2.3.4", dummy=True)
-        IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4'})
-
-    Note:
-        When working within a module's `handle_event()`, use the instance method
-        `self.make_event()` instead of calling this function directly.
     """
     if not data:
         raise ValidationError("No data provided")
 
+    # do not allow passing an existing event here – use update_event() instead
+    if is_event(data):
+        raise ValidationError(
+            "make_event() does not accept an existing event object. Use update_event(event, ...) to modify an event."
+        )
+
     # allow tags to be either a string or an array
     if not tags:
         tags = []
@@ -1814,76 +1860,58 @@ def make_event(
         tags = [tags]
     tags = set(tags)
 
-    # if data is already an event, update it with the user's kwargs
-    if is_event(data):
-        event = copy(data)
-        if scan is not None and not event.scan:
-            event.scan = scan
-        if module is not None:
-            event.module = module
-        if parent is not None:
-            event.parent = parent
-        if context is not None:
-            event.discovery_context = context
-        if internal is True:
-            event.internal = True
-        if tags:
-            event.tags = tags.union(event.tags)
-        event_type = data.type
-        return event
-    else:
-        # if event_type is not provided, autodetect it
-        if event_type is None:
-            event_seed = EventSeed(data)
-            event_type = event_seed.type
-            data = event_seed.data
-            if not dummy:
-                log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
-
-        event_type = str(event_type).strip().upper()
-
-        # Catch these common whoopsies
-        if event_type in ("DNS_NAME", "IP_ADDRESS"):
-            # DNS_NAME <--> EMAIL_ADDRESS confusion
-            if validators.soft_validate(data, "email"):
-                event_type = "EMAIL_ADDRESS"
-            else:
-                # DNS_NAME <--> IP_ADDRESS confusion
-                try:
-                    data = validators.validate_host(data)
-                except Exception as e:
-                    log.trace(traceback.format_exc())
-                    raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}')
-                data_is_ip = is_ip(data)
-                if event_type == "DNS_NAME" and data_is_ip:
-                    event_type = "IP_ADDRESS"
-                elif event_type == "IP_ADDRESS" and not data_is_ip:
-                    event_type = "DNS_NAME"
-        # USERNAME <--> EMAIL_ADDRESS confusion
-        if event_type == "USERNAME" and validators.soft_validate(data, "email"):
+    # if event_type is not provided, autodetect it
+    if event_type is None:
+        event_seed = EventSeed(data)
+        event_type = event_seed.type
+        data = event_seed.data
+        if not dummy:
+            log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
+
+    event_type = str(event_type).strip().upper()
+
+    # Catch these common whoopsies
+    if event_type in ("DNS_NAME", "IP_ADDRESS"):
+        # DNS_NAME <--> EMAIL_ADDRESS confusion
+        if validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
-            tags.add("affiliate")
-        # Convert single-host IP_RANGE to IP_ADDRESS
-        if event_type == "IP_RANGE":
-            with suppress(Exception):
-                net = ipaddress.ip_network(data, strict=False)
-                if net.prefixlen == net.max_prefixlen:
-                    event_type = "IP_ADDRESS"
-                    data = net.network_address
-
-        event_class = globals().get(event_type, DefaultEvent)
-        return event_class(
-            data,
-            event_type=event_type,
-            parent=parent,
-            context=context,
-            module=module,
-            scan=scan,
-            tags=tags,
-            confidence=confidence,
-            _dummy=dummy,
-            _internal=internal,
-        )
+        else:
+            # DNS_NAME <--> IP_ADDRESS confusion
+            try:
+                data = validators.validate_host(data)
+            except Exception as e:
+                log.trace(traceback.format_exc())
+                raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}')
+            data_is_ip = is_ip(data)
+            if event_type == "DNS_NAME" and data_is_ip:
+                event_type = "IP_ADDRESS"
+            elif event_type == "IP_ADDRESS" and not data_is_ip:
+                event_type = "DNS_NAME"
+    # USERNAME <--> EMAIL_ADDRESS confusion
+    if event_type == "USERNAME" and validators.soft_validate(data, "email"):
+        event_type = "EMAIL_ADDRESS"
+        tags.add("affiliate")
+    # Convert single-host IP_RANGE to IP_ADDRESS
+    if event_type == "IP_RANGE":
+        with suppress(Exception):
+            net = ipaddress.ip_network(data, strict=False)
+            if net.prefixlen == net.max_prefixlen:
+                event_type = "IP_ADDRESS"
+                data = net.network_address
+
+    event_class = globals().get(event_type, DefaultEvent)
+    return event_class(
+        data,
+        event_type=event_type,
+        parent=parent,
+        context=context,
+        module=module,
+        scan=scan,
+        tags=tags,
+        confidence=confidence,
+        _dummy=dummy,
+        _internal=internal,
+    )
 
 
 def event_from_json(j, siem_friendly=False):

bbot/core/flags.py

@@ -6,6 +6,7 @@ flag_descriptions = {
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ class BloomFilter:
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""

bbot/core/helpers/command.py

@@ -123,7 +123,7 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                     proc.send_signal(SIGINT)
                     raise
                 except ValueError as e:
-                    command_str = " ".join([str(c) for c in command])
+                    command_str = " ".join(command)
                     log.warning(f"Error executing command {command_str}: {e}")
                     log.trace(traceback.format_exc())
                     continue
@@ -185,7 +185,9 @@ async def _spawn_proc(self, *command, **kwargs):
     try:
         command, kwargs = self._prepare_command_kwargs(command, kwargs)
     except SubprocessError as e:
-        log.warning(e)
+        command_str = " ".join([str(s) for s in command])
+        log.warning(f"Error running command: '{command_str}': {e}")
+        log.trace(traceback.format_exc())
         return None, None, None
     _input = kwargs.pop("input", None)
     if _input is not None:
@@ -279,6 +281,7 @@ def _prepare_command_kwargs(self, command, kwargs):
 
     if len(command) == 1 and isinstance(command[0], (list, tuple)):
         command = command[0]
+
     command = [str(s) for s in command]
 
     if not command:

bbot/core/helpers/depsinstaller/installer.py

@@ -2,6 +2,8 @@ import os
 import sys
 import stat
 import json
+import mmh3
+import orjson
 import shutil
 import getpass
 import logging
@@ -14,6 +16,7 @@ from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
+from bbot import __version__
 from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
@@ -32,6 +35,20 @@ class DepsInstaller:
         "bash": "bash",
         "which": "which",
         "tar": "tar",
+        "xz": [
+            {
+                "name": "Install xz-utils (Debian)",
+                "package": {"name": ["xz-utils"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Debian'",
+            },
+            {
+                "name": "Install xz (Non-Debian)",
+                "package": {"name": ["xz"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] != 'Debian'",
+            },
+        ],
         # debian why are you like this
         "7z": [
             {
@@ -53,6 +70,44 @@ class DepsInstaller:
                 "when": "ansible_facts['distribution'] == 'Fedora'",
             },
         ],
+        # to compile just about any tool, we need the openssl dev headers
+        "openssl_dev_headers": [
+            {
+                "name": "Install OpenSSL library and development headers (Debian/Ubuntu)",
+                "package": {"name": ["libssl-dev", "openssl"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Debian'",
+                "ignore_errors": True,
+            },
+            {
+                "name": "Install OpenSSL library and development headers (RedHat/CentOS/Fedora)",
+                "package": {"name": ["openssl", "openssl-devel"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'RedHat' or ansible_facts['os_family'] == 'Suse' ",
+                "ignore_errors": True,
+            },
+            {
+                "name": "Install OpenSSL library and development headers (Arch)",
+                "package": {"name": ["openssl"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Archlinux'",
+                "ignore_errors": True,
+            },
+            {
+                "name": "Install OpenSSL library and development headers (Alpine)",
+                "package": {"name": ["openssl", "openssl-dev"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Alpine'",
+                "ignore_errors": True,
+            },
+            {
+                "name": "Install OpenSSL library and development headers (FreeBSD)",
+                "package": {"name": ["openssl"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'FreeBSD'",
+                "ignore_errors": True,
+            },
+        ],
     }
 
     def __init__(self, parent_helper):
@@ -120,6 +175,7 @@ class DepsInstaller:
                     + self.venv
                     + str(self.parent_helper.bbot_home)
                     + os.uname()[1]
+                    + str(__version__)
                 ).hexdigest()
                 success = self.setup_status.get(module_hash, None)
                 dependencies = list(chain(*preloaded["deps"].values()))
@@ -171,11 +227,6 @@ class DepsInstaller:
         success = True
         preloaded = self.all_modules_preloaded[module]
 
-        # ansible tasks
-        ansible_tasks = preloaded["deps"]["ansible"]
-        if ansible_tasks:
-            success &= self.tasks(module, ansible_tasks)
-
         # apt
         deps_apt = preloaded["deps"]["apt"]
         if deps_apt:
@@ -196,7 +247,7 @@ class DepsInstaller:
         deps_common = preloaded["deps"]["common"]
         if deps_common:
             for dep_common in deps_common:
-                if self.setup_status.get(dep_common, False) is True:
+                if self.setup_status.get(dep_common, False) is True and self.deps_behavior != "force_install":
                     log.debug(
                         f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
                     )
@@ -206,6 +257,11 @@ class DepsInstaller:
                 self.setup_status[dep_common] = result
                 success &= result
 
+        # ansible tasks
+        ansible_tasks = preloaded["deps"]["ansible"]
+        if ansible_tasks:
+            success &= self.tasks(module, ansible_tasks)
+
         return success
 
     async def pip_install(self, packages, constraints=None):
@@ -387,6 +443,13 @@ class DepsInstaller:
                     log.warning("Incorrect password")
 
     async def install_core_deps(self):
+        # skip if we've already successfully installed core deps for this definition
+        core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS)))
+        core_deps_cache_file = self.parent_helper.cache_dir / core_deps_hash
+        if core_deps_cache_file.exists():
+            log.debug("Skipping core dependency installation (cache hit)")
+            return
+
         to_install = set()
         to_install_friendly = set()
         playbook = []
@@ -402,6 +465,7 @@ class DepsInstaller:
                 else:
                     playbook.extend(package_name_or_playbook)
         # install ansible community.general collection
+        overall_success = True
         if not self.setup_status.get("ansible:community.general", False):
             log.info("Installing Ansible Community General Collection")
             try:
@@ -413,6 +477,7 @@ class DepsInstaller:
                 log.warning(
                     f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}"
                 )
+                overall_success = False
         # construct ansible playbook
         if to_install:
             playbook.append(
@@ -426,7 +491,13 @@ class DepsInstaller:
         if playbook:
             log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()
-            self.ansible_run(tasks=playbook)
+            success, _ = self.ansible_run(tasks=playbook)
+            overall_success &= success
+
+        # mark cache only if everything succeeded (or nothing needed doing)
+        if overall_success:
+            with suppress(Exception):
+                core_deps_cache_file.touch()
 
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup:

bbot/core/helpers/dns/dns.py

@@ -38,7 +38,6 @@ class DNSHelper(EngineClient):
         _wildcard_cache (dict): Cache for wildcard detection results.
         _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.
         resolver_file (Path): File containing system's current resolver nameservers.
-        filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True.
 
     Args:
         parent_helper: The parent helper object with configuration details and utilities.

bbot/core/helpers/dns/engine.py

@@ -86,8 +86,6 @@ class DNSEngine(EngineServer):
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
-        self.filter_bad_ptrs = self.dns_config.get("filter_ptrs", True)
-
     async def resolve(self, query, **kwargs):
         """Resolve DNS names and IP addresses to their corresponding results.
 

bbot/core/helpers/files.py

@@ -9,7 +9,7 @@ from .misc import rm_at_exit
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):