bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/core/event/base.py

@@ -1,5 +1,6 @@
 import io
 import re
+import uuid
 import json
 import base64
 import logging
@@ -24,6 +25,7 @@ from bbot.core.helpers import (
     is_domain,
     is_subdomain,
     is_ip,
+    is_ip_type,
     is_ptr,
     is_uri,
     url_depth,
@@ -58,9 +60,10 @@ class BaseEvent:
 
     Attributes:
         type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`.
-        id (str): A unique identifier for the event.
+        id (str): An identifier for the event (event type + sha1 hash of data). NOT universally unique.
+        uuid (UUID): A universally unique identifier for the event.
         data (str or dict): The main data for the event, e.g., a URL or IP address.
-        data_graph (str): Representation of `self.data` for Neo4j graph nodes.
+        data_graph (str): Representation of `self.data` for graph nodes (e.g. Neo4j).
         data_human (str): Representation of `self.data` for human output.
         data_id (str): Representation of `self.data` used to calculate the event's ID (and ultimately its hash, which is used for deduplication)
         data_json (str): Representation of `self.data` to be used in JSON serialization.
@@ -75,6 +78,7 @@ class BaseEvent:
         resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names.
         parent (BaseEvent): The parent event that led to the discovery of this event.
         parent_id (str): The `id` attribute of the parent event.
+        parent_uuid (str): The `uuid` attribute of the parent event.
         tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`.
         module (BaseModule): The module that discovered the event.
         module_sequence (str): The sequence of modules that participated in the discovery.
@@ -154,7 +158,7 @@ class BaseEvent:
         Raises:
             ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
-
+        self._uuid = uuid.uuid4()
         self._id = None
         self._hash = None
         self._data = None
@@ -166,12 +170,13 @@ class BaseEvent:
         self._parent = None
         self._priority = None
         self._parent_id = None
+        self._parent_uuid = None
         self._host_original = None
         self._scope_distance = None
         self._module_priority = None
         self._resolved_hosts = set()
-        self.dns_children = dict()
-        self.raw_dns_records = dict()
+        self.dns_children = {}
+        self.raw_dns_records = {}
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
@@ -198,7 +203,7 @@ class BaseEvent:
         # self.scan holds the instantiated scan object (for helpers, etc.)
         self.scan = scan
         if (not self.scan) and (not self._dummy):
-            raise ValidationError(f"Must specify scan")
+            raise ValidationError("Must specify scan")
         # self.scans holds a list of scan IDs from scans that encountered this event
         self.scans = []
         if scans is not None:
@@ -217,7 +222,7 @@ class BaseEvent:
 
         self.parent = parent
         if (not self.parent) and (not self._dummy):
-            raise ValidationError(f"Must specify event parent")
+            raise ValidationError("Must specify event parent")
 
         if tags is not None:
             for tag in tags:
@@ -296,9 +301,9 @@ class BaseEvent:
         The purpose of internal events is to enable speculative/explorative discovery without cluttering
         the console with irrelevant or uninteresting events.
         """
-        if not value in (True, False):
+        if value not in (True, False):
             raise ValueError(f'"internal" must be boolean, not {type(value)}')
-        if value == True:
+        if value is True:
             self.add_tag("internal")
         else:
             self.remove_tag("internal")
@@ -336,6 +341,21 @@ class BaseEvent:
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -348,6 +368,12 @@ class BaseEvent:
                 return 80
         return self._port
 
+    @property
+    def netloc(self):
+        if self.host and is_ip_type(self.host, network=False):
+            return make_netloc(self.host, self.port)
+        return None
+
     @property
     def host_stem(self):
         """
@@ -393,7 +419,7 @@ class BaseEvent:
         parent_chain = []
         if self.parent is not None and self.parent is not self:
             parent_chain = self.parent.parent_chain
-        return parent_chain + [self.id]
+        return parent_chain + [str(self.uuid)]
 
     @property
     def words(self):
@@ -436,11 +462,6 @@ class BaseEvent:
         no_host_information = not bool(self.host)
         return self._always_emit or always_emit_tags or no_host_information
 
-    @property
-    def quick_emit(self):
-        no_host_information = not bool(self.host)
-        return self._quick_emit or no_host_information
-
     @property
     def id(self):
         """
@@ -450,6 +471,13 @@ class BaseEvent:
             self._id = f"{self.type}:{self.data_hash.hex()}"
         return self._id
 
+    @property
+    def uuid(self):
+        """
+        A universally unique identifier for the event
+        """
+        return f"{self.type}:{self._uuid}"
+
     @property
     def data_hash(self):
         """
@@ -490,12 +518,13 @@ class BaseEvent:
             for t in list(self.tags):
                 if t.startswith("distance-"):
                     self.remove_tag(t)
-            if scope_distance == 0:
-                self.add_tag("in-scope")
-                self.remove_tag("affiliate")
-            else:
-                self.remove_tag("in-scope")
-                self.add_tag(f"distance-{new_scope_distance}")
+            if self.host:
+                if scope_distance == 0:
+                    self.add_tag("in-scope")
+                    self.remove_tag("affiliate")
+                else:
+                    self.remove_tag("in-scope")
+                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
@@ -567,6 +596,13 @@ class BaseEvent:
             return parent_id
         return self._parent_id
 
+    @property
+    def parent_uuid(self):
+        parent_uuid = getattr(self.get_parent(), "uuid", None)
+        if parent_uuid is not None:
+            return parent_uuid
+        return self._parent_uuid
+
     @property
     def validators(self):
         """
@@ -733,12 +769,12 @@ class BaseEvent:
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
         """
+        j = {}
         # type, ID, scope description
-        j = dict()
-        for i in ("type", "id", "scope_description"):
+        for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
             if v:
-                j.update({i: v})
+                j.update({i: str(v)})
         # event data
         data_attr = getattr(self, f"data_{mode}", None)
         if data_attr is not None:
@@ -754,6 +790,8 @@ class BaseEvent:
             j["host"] = str(self.host)
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
             j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        if isinstance(self.port, int):
+            j["port"] = self.port
         # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
@@ -769,6 +807,9 @@ class BaseEvent:
         parent_id = self.parent_id
         if parent_id:
             j["parent"] = parent_id
+        parent_uuid = self.parent_uuid
+        if parent_uuid:
+            j["parent_uuid"] = parent_uuid
         # tags
         if self.tags:
             j.update({"tags": list(self.tags)})
@@ -972,18 +1013,20 @@ class ClosestHostEvent(DictHostEvent):
         if not self.host:
             for parent in self.get_parents(include_self=True):
                 # inherit closest URL
-                if not "url" in self.data:
+                if "url" not in self.data:
                     parent_url = getattr(parent, "parsed_url", None)
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if "path" not in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -993,20 +1036,21 @@ class ClosestHostEvent(DictHostEvent):
 class DictPathEvent(DictEvent):
     def sanitize_data(self, data):
         new_data = dict(data)
+        new_data["path"] = str(new_data["path"])
         file_blobs = getattr(self.scan, "_file_blobs", False)
         folder_blobs = getattr(self.scan, "_folder_blobs", False)
         blob = None
         try:
-            data_path = Path(data["path"])
-            if data_path.is_file():
+            self._data_path = Path(data["path"])
+            if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
-                    with open(data_path, "rb") as file:
+                    with open(self._data_path, "rb") as file:
                         blob = file.read()
-            elif data_path.is_dir():
+            elif self._data_path.is_dir():
                 self.add_tag("folder")
                 if folder_blobs:
-                    blob = self._tar_directory(data_path)
+                    blob = self._tar_directory(self._data_path)
         except KeyError:
             pass
         if blob:
@@ -1085,8 +1129,7 @@ class DnsEvent(BaseEvent):
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1137,7 +1180,6 @@ class URL_UNVERIFIED(BaseEvent):
         self.num_redirects = getattr(self.parent, "num_redirects", 0)
 
     def _data_id(self):
-
         data = super()._data_id()
 
         # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)
@@ -1185,7 +1227,7 @@ class URL_UNVERIFIED(BaseEvent):
 
     def add_tag(self, tag):
         host_same_as_parent = self.parent and self.host == self.parent.host
-        if tag == "spider-danger" and host_same_as_parent and not "spider-danger" in self.tags:
+        if tag == "spider-danger" and host_same_as_parent and "spider-danger" not in self.tags:
             # increment the web spider distance
             if self.type == "URL_UNVERIFIED":
                 self.web_spider_distance += 1
@@ -1207,7 +1249,7 @@ class URL_UNVERIFIED(BaseEvent):
 
     def _words(self):
         first_elem = self.parsed_url.path.lstrip("/").split("/")[0]
-        if not "." in first_elem:
+        if "." not in first_elem:
             return extract_words(first_elem)
         return set()
 
@@ -1224,7 +1266,6 @@ class URL_UNVERIFIED(BaseEvent):
 
 
 class URL(URL_UNVERIFIED):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -1236,7 +1277,7 @@ class URL(URL_UNVERIFIED):
     @property
     def resolved_hosts(self):
         # TODO: remove this when we rip out httpx
-        return set(".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-"))
+        return {".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")}
 
     @property
     def pretty_string(self):
@@ -1266,7 +1307,6 @@ class URL_HINT(URL_UNVERIFIED):
 
 
 class WEB_PARAMETER(DictHostEvent):
-
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")
@@ -1515,7 +1555,25 @@ class WAF(DictHostEvent):
 
 
 class FILESYSTEM(DictPathEvent):
-    pass
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self._data_path.is_file():
+            # detect type of file content using magic
+            from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+            extension, mime_type, description, confidence = get_magic_info(self.data["path"])
+            self.data["magic_extension"] = extension
+            self.data["magic_mime_type"] = mime_type
+            self.data["magic_description"] = description
+            self.data["magic_confidence"] = confidence
+            # detection compression
+            compression = get_compression(mime_type)
+            if compression:
+                self.add_tag("compressed")
+                self.add_tag(f"{compression}-archive")
+                self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
 
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
@@ -1523,6 +1581,13 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     _always_emit_tags = ["target"]
 
 
+class MOBILE_APP(DictEvent):
+    _always_emit = True
+
+    def _pretty_string(self):
+        return self.data["url"]
+
+
 def make_event(
     data,
     event_type=None,
@@ -1589,23 +1654,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
-        if internal == True:
-            data.internal = True
+            event.discovery_context = context
+        if internal is True:
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.tags = tags.union(event.tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)
@@ -1635,6 +1700,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 
@@ -1691,6 +1763,9 @@ def event_from_json(j, siem_friendly=False):
             data = j["data"]
         kwargs["data"] = data
         event = make_event(**kwargs)
+        event_uuid = j.get("uuid", None)
+        if event_uuid is not None:
+            event._uuid = uuid.UUID(event_uuid.split(":")[-1])
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
@@ -1700,6 +1775,10 @@ def event_from_json(j, siem_friendly=False):
         parent_id = j.get("parent", None)
         if parent_id is not None:
             event._parent_id = parent_id
+        parent_uuid = j.get("parent_uuid", None)
+        if parent_uuid is not None:
+            parent_type, parent_uuid = parent_uuid.split(":", 1)
+            event._parent_uuid = parent_type + ":" + str(uuid.UUID(parent_uuid))
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")

bbot/core/helpers/bloom.py

@@ -5,14 +5,14 @@ import mmap
 
 class BloomFilter:
     """
-    Simple bloom filter implementation capable of rougly 400K lookups/s.
+    Simple bloom filter implementation capable of roughly 400K lookups/s.
 
     BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track
     of which mutations have been tried so far.
 
     A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
     A python hash is 36 bytes. So if you wanted to store these in a set, this would take up
-    36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
+    36 * 10M * 2 (key+value) == 720 megabytes. So we save roughly 7 times the space.
     """
 
     def __init__(self, size=8000000):
@@ -64,8 +64,15 @@ class BloomFilter:
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

bbot/core/helpers/command.py

@@ -210,9 +210,10 @@ async def _write_proc_line(proc, chunk):
         return True
     except Exception as e:
         proc_args = [str(s) for s in getattr(proc, "args", [])]
-        command = " ".join(proc_args)
-        log.warning(f"Error writing line to stdin for command: {command}: {e}")
-        log.trace(traceback.format_exc())
+        command = " ".join(proc_args).strip()
+        if command:
+            log.warning(f"Error writing line to stdin for command: {command}: {e}")
+            log.trace(traceback.format_exc())
         return False
 
 
@@ -268,11 +269,11 @@ def _prepare_command_kwargs(self, command, kwargs):
         (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
     """
     # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
-    if not "limit" in kwargs:
+    if "limit" not in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100
-    if not "stdout" in kwargs:
+    if "stdout" not in kwargs:
         kwargs["stdout"] = asyncio.subprocess.PIPE
-    if not "stderr" in kwargs:
+    if "stderr" not in kwargs:
         kwargs["stderr"] = asyncio.subprocess.PIPE
     sudo = kwargs.pop("sudo", False)
 
@@ -285,7 +286,7 @@ def _prepare_command_kwargs(self, command, kwargs):
 
     # use full path of binary, if not already specified
     binary = command[0]
-    if not "/" in binary:
+    if "/" not in binary:
         binary_full_path = which(binary)
         if binary_full_path is None:
             raise SubprocessError(f'Command "{binary}" was not found')

bbot/core/helpers/depsinstaller/installer.py

@@ -14,7 +14,7 @@ from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform, rm_at_exit
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -44,7 +44,13 @@ class DepsInstaller:
         self.parent_helper.mkdir(self.command_status)
         self.setup_status = self.read_setup_status()
 
-        self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
+        # make sure we're using a minimal git config
+        self.minimal_git_config = self.data_dir / "minimal_git.config"
+        self.minimal_git_config.touch()
+        os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
+
+        self.deps_config = self.parent_helper.config.get("deps", {})
+        self.deps_behavior = self.deps_config.get("behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
         if sys.prefix != sys.base_prefix:
@@ -91,11 +97,11 @@ class DepsInstaller:
                         or self.deps_behavior == "force_install"
                     ):
                         if not notified:
-                            log.hugeinfo(f"Installing module dependencies. Please be patient, this may take a while.")
+                            log.hugeinfo("Installing module dependencies. Please be patient, this may take a while.")
                             notified = True
                         log.verbose(f'Installing dependencies for module "{m}"')
                         # get sudo access if we need it
-                        if preloaded.get("sudo", False) == True:
+                        if preloaded.get("sudo", False) is True:
                             self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
                         success = await self.install_module(m)
                         self.setup_status[module_hash] = success
@@ -153,7 +159,7 @@ class DepsInstaller:
         deps_common = preloaded["deps"]["common"]
         if deps_common:
             for dep_common in deps_common:
-                if self.setup_status.get(dep_common, False) == True:
+                if self.setup_status.get(dep_common, False) is True:
                     log.debug(
                         f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
                     )
@@ -171,10 +177,13 @@ class DepsInstaller:
 
         command = [sys.executable, "-m", "pip", "install", "--upgrade"] + packages
 
-        if constraints:
-            constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
-            command.append("--constraint")
-            command.append(constraints_tempfile)
+        # if no custom constraints are provided, use the constraints of the currently installed version of bbot
+        if constraints is not None:
+            constraints = get_python_constraints()
+
+        constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
+        command.append("--constraint")
+        command.append(constraints_tempfile)
 
         process = None
         try:
@@ -235,7 +244,7 @@ class DepsInstaller:
         if success:
             log.info(f"Successfully ran {len(commands):,} shell commands")
         else:
-            log.warning(f"Failed to run shell dependencies")
+            log.warning("Failed to run shell dependencies")
         return success
 
     def tasks(self, module, tasks):
@@ -248,7 +257,7 @@ class DepsInstaller:
         return success
 
     def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
-        _ansible_args = {"ansible_connection": "local"}
+        _ansible_args = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable}
         if ansible_args is not None:
             _ansible_args.update(ansible_args)
         module_args = None
@@ -301,7 +310,7 @@ class DepsInstaller:
         return success, err
 
     def read_setup_status(self):
-        setup_status = dict()
+        setup_status = {}
         if self.setup_status_cache.is_file():
             with open(self.setup_status_cache) as f:
                 with suppress(Exception):
@@ -342,7 +351,16 @@ class DepsInstaller:
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
         # command: package_name
-        core_deps = {"unzip": "unzip", "curl": "curl"}
+        core_deps = {
+            "unzip": "unzip",
+            "zipinfo": "unzip",
+            "curl": "curl",
+            "git": "git",
+            "make": "make",
+            "gcc": "gcc",
+            "bash": "bash",
+            "which": "which",
+        }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):
                 to_install.add(package_name)