bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/core/event/base.py

@@ -1,5 +1,6 @@
 import io
 import re
+import uuid
 import json
 import base64
 import logging
@@ -24,6 +25,7 @@ from bbot.core.helpers import (
     is_domain,
     is_subdomain,
     is_ip,
+    is_ip_type,
     is_ptr,
     is_uri,
     url_depth,
@@ -58,9 +60,10 @@ class BaseEvent:
 
     Attributes:
         type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`.
-        id (str): A unique identifier for the event.
+        id (str): An identifier for the event (event type + sha1 hash of data). NOT universally unique.
+        uuid (UUID): A universally unique identifier for the event.
         data (str or dict): The main data for the event, e.g., a URL or IP address.
-        data_graph (str): Representation of `self.data` for Neo4j graph nodes.
+        data_graph (str): Representation of `self.data` for graph nodes (e.g. Neo4j).
         data_human (str): Representation of `self.data` for human output.
         data_id (str): Representation of `self.data` used to calculate the event's ID (and ultimately its hash, which is used for deduplication)
         data_json (str): Representation of `self.data` to be used in JSON serialization.
@@ -75,6 +78,7 @@ class BaseEvent:
         resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names.
         parent (BaseEvent): The parent event that led to the discovery of this event.
         parent_id (str): The `id` attribute of the parent event.
+        parent_uuid (str): The `uuid` attribute of the parent event.
         tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`.
         module (BaseModule): The module that discovered the event.
         module_sequence (str): The sequence of modules that participated in the discovery.
@@ -154,7 +158,7 @@ class BaseEvent:
         Raises:
             ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
-
+        self._uuid = uuid.uuid4()
         self._id = None
         self._hash = None
         self._data = None
@@ -166,11 +170,13 @@ class BaseEvent:
         self._parent = None
         self._priority = None
         self._parent_id = None
+        self._parent_uuid = None
         self._host_original = None
         self._scope_distance = None
         self._module_priority = None
         self._resolved_hosts = set()
-        self.dns_children = dict()
+        self.dns_children = {}
+        self.raw_dns_records = {}
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
@@ -197,7 +203,7 @@ class BaseEvent:
         # self.scan holds the instantiated scan object (for helpers, etc.)
         self.scan = scan
         if (not self.scan) and (not self._dummy):
-            raise ValidationError(f"Must specify scan")
+            raise ValidationError("Must specify scan")
         # self.scans holds a list of scan IDs from scans that encountered this event
         self.scans = []
         if scans is not None:
@@ -216,7 +222,7 @@ class BaseEvent:
 
         self.parent = parent
         if (not self.parent) and (not self._dummy):
-            raise ValidationError(f"Must specify event parent")
+            raise ValidationError("Must specify event parent")
 
         if tags is not None:
             for tag in tags:
@@ -295,9 +301,9 @@ class BaseEvent:
         The purpose of internal events is to enable speculative/explorative discovery without cluttering
         the console with irrelevant or uninteresting events.
         """
-        if not value in (True, False):
+        if value not in (True, False):
             raise ValueError(f'"internal" must be boolean, not {type(value)}')
-        if value == True:
+        if value is True:
             self.add_tag("internal")
         else:
             self.remove_tag("internal")
@@ -335,6 +341,21 @@ class BaseEvent:
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -347,6 +368,12 @@ class BaseEvent:
                 return 80
         return self._port
 
+    @property
+    def netloc(self):
+        if self.host and is_ip_type(self.host, network=False):
+            return make_netloc(self.host, self.port)
+        return None
+
     @property
     def host_stem(self):
         """
@@ -379,10 +406,20 @@ class BaseEvent:
         """
         This event's full discovery context, including those of all its parents
         """
-        parent_path = []
+        discovery_path = []
+        if self.parent is not None and self.parent is not self:
+            discovery_path = self.parent.discovery_path
+        return discovery_path + [self.discovery_context]
+
+    @property
+    def parent_chain(self):
+        """
+        This event's full discovery context, including those of all its parents
+        """
+        parent_chain = []
         if self.parent is not None and self.parent is not self:
-            parent_path = self.parent.discovery_path
-        return parent_path + [[self.id, self.discovery_context]]
+            parent_chain = self.parent.parent_chain
+        return parent_chain + [str(self.uuid)]
 
     @property
     def words(self):
@@ -425,11 +462,6 @@ class BaseEvent:
         no_host_information = not bool(self.host)
         return self._always_emit or always_emit_tags or no_host_information
 
-    @property
-    def quick_emit(self):
-        no_host_information = not bool(self.host)
-        return self._quick_emit or no_host_information
-
     @property
     def id(self):
         """
@@ -439,6 +471,13 @@ class BaseEvent:
             self._id = f"{self.type}:{self.data_hash.hex()}"
         return self._id
 
+    @property
+    def uuid(self):
+        """
+        A universally unique identifier for the event
+        """
+        return f"{self.type}:{self._uuid}"
+
     @property
     def data_hash(self):
         """
@@ -479,12 +518,13 @@ class BaseEvent:
             for t in list(self.tags):
                 if t.startswith("distance-"):
                     self.remove_tag(t)
-            if scope_distance == 0:
-                self.add_tag("in-scope")
-                self.remove_tag("affiliate")
-            else:
-                self.remove_tag("in-scope")
-                self.add_tag(f"distance-{new_scope_distance}")
+            if self.host:
+                if scope_distance == 0:
+                    self.add_tag("in-scope")
+                    self.remove_tag("affiliate")
+                else:
+                    self.remove_tag("in-scope")
+                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
@@ -556,6 +596,13 @@ class BaseEvent:
             return parent_id
         return self._parent_id
 
+    @property
+    def parent_uuid(self):
+        parent_uuid = getattr(self.get_parent(), "uuid", None)
+        if parent_uuid is not None:
+            return parent_uuid
+        return self._parent_uuid
+
     @property
     def validators(self):
         """
@@ -722,12 +769,12 @@ class BaseEvent:
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
         """
+        j = {}
         # type, ID, scope description
-        j = dict()
-        for i in ("type", "id", "scope_description"):
+        for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
             if v:
-                j.update({i: v})
+                j.update({i: str(v)})
         # event data
         data_attr = getattr(self, f"data_{mode}", None)
         if data_attr is not None:
@@ -743,6 +790,8 @@ class BaseEvent:
             j["host"] = str(self.host)
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
             j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        if isinstance(self.port, int):
+            j["port"] = self.port
         # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
@@ -758,6 +807,9 @@ class BaseEvent:
         parent_id = self.parent_id
         if parent_id:
             j["parent"] = parent_id
+        parent_uuid = self.parent_uuid
+        if parent_uuid:
+            j["parent_uuid"] = parent_uuid
         # tags
         if self.tags:
             j.update({"tags": list(self.tags)})
@@ -770,6 +822,7 @@ class BaseEvent:
         # discovery context
         j["discovery_context"] = self.discovery_context
         j["discovery_path"] = self.discovery_path
+        j["parent_chain"] = self.parent_chain
 
         # normalize non-primitive python objects
         for k, v in list(j.items()):
@@ -909,6 +962,10 @@ class SCAN(BaseEvent):
     def discovery_path(self):
         return []
 
+    @property
+    def parent_chain(self):
+        return []
+
 
 class FINISHED(BaseEvent):
     """
@@ -956,18 +1013,20 @@ class ClosestHostEvent(DictHostEvent):
         if not self.host:
             for parent in self.get_parents(include_self=True):
                 # inherit closest URL
-                if not "url" in self.data:
+                if "url" not in self.data:
                     parent_url = getattr(parent, "parsed_url", None)
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if "path" not in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -977,20 +1036,21 @@ class ClosestHostEvent(DictHostEvent):
 class DictPathEvent(DictEvent):
     def sanitize_data(self, data):
         new_data = dict(data)
+        new_data["path"] = str(new_data["path"])
         file_blobs = getattr(self.scan, "_file_blobs", False)
         folder_blobs = getattr(self.scan, "_folder_blobs", False)
         blob = None
         try:
-            data_path = Path(data["path"])
-            if data_path.is_file():
+            self._data_path = Path(data["path"])
+            if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
-                    with open(data_path, "rb") as file:
+                    with open(self._data_path, "rb") as file:
                         blob = file.read()
-            elif data_path.is_dir():
+            elif self._data_path.is_dir():
                 self.add_tag("folder")
                 if folder_blobs:
-                    blob = self._tar_directory(data_path)
+                    blob = self._tar_directory(self._data_path)
         except KeyError:
             pass
         if blob:
@@ -1053,13 +1113,23 @@ class DnsEvent(BaseEvent):
             if parent_module_type == "DNS":
                 self.dns_resolve_distance += 1
         # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
+        # tag subdomain / domain
+        if is_subdomain(self.host):
+            self.add_tag("subdomain")
+        elif is_domain(self.host):
+            self.add_tag("domain")
+        # tag private IP
+        try:
+            if self.host.is_private:
+                self.add_tag("private-ip")
+        except AttributeError:
+            pass
 
 
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1069,13 +1139,6 @@ class IP_RANGE(DnsEvent):
 
 
 class DNS_NAME(DnsEvent):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if is_subdomain(self.data):
-            self.add_tag("subdomain")
-        elif is_domain(self.data):
-            self.add_tag("domain")
-
     def sanitize_data(self, data):
         return validators.validate_host(data)
 
@@ -1117,7 +1180,6 @@ class URL_UNVERIFIED(BaseEvent):
         self.num_redirects = getattr(self.parent, "num_redirects", 0)
 
     def _data_id(self):
-
         data = super()._data_id()
 
         # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)
@@ -1165,7 +1227,7 @@ class URL_UNVERIFIED(BaseEvent):
 
     def add_tag(self, tag):
         host_same_as_parent = self.parent and self.host == self.parent.host
-        if tag == "spider-danger" and host_same_as_parent and not "spider-danger" in self.tags:
+        if tag == "spider-danger" and host_same_as_parent and "spider-danger" not in self.tags:
             # increment the web spider distance
             if self.type == "URL_UNVERIFIED":
                 self.web_spider_distance += 1
@@ -1187,7 +1249,7 @@ class URL_UNVERIFIED(BaseEvent):
 
     def _words(self):
         first_elem = self.parsed_url.path.lstrip("/").split("/")[0]
-        if not "." in first_elem:
+        if "." not in first_elem:
             return extract_words(first_elem)
         return set()
 
@@ -1204,7 +1266,6 @@ class URL_UNVERIFIED(BaseEvent):
 
 
 class URL(URL_UNVERIFIED):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -1216,7 +1277,7 @@ class URL(URL_UNVERIFIED):
     @property
     def resolved_hosts(self):
         # TODO: remove this when we rip out httpx
-        return set(".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-"))
+        return {".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")}
 
     @property
     def pretty_string(self):
@@ -1246,7 +1307,6 @@ class URL_HINT(URL_UNVERIFIED):
 
 
 class WEB_PARAMETER(DictHostEvent):
-
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")
@@ -1495,14 +1555,39 @@ class WAF(DictHostEvent):
 
 
 class FILESYSTEM(DictPathEvent):
-    pass
-
-
-class RAW_DNS_RECORD(DictHostEvent):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self._data_path.is_file():
+            # detect type of file content using magic
+            from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+            extension, mime_type, description, confidence = get_magic_info(self.data["path"])
+            self.data["magic_extension"] = extension
+            self.data["magic_mime_type"] = mime_type
+            self.data["magic_description"] = description
+            self.data["magic_confidence"] = confidence
+            # detection compression
+            compression = get_compression(mime_type)
+            if compression:
+                self.add_tag("compressed")
+                self.add_tag(f"{compression}-archive")
+                self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
+
+
+class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     # don't emit raw DNS records for affiliates
     _always_emit_tags = ["target"]
 
 
+class MOBILE_APP(DictEvent):
+    _always_emit = True
+
+    def _pretty_string(self):
+        return self.data["url"]
+
+
 def make_event(
     data,
     event_type=None,
@@ -1569,23 +1654,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
-        if internal == True:
-            data.internal = True
+            event.discovery_context = context
+        if internal is True:
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.tags = tags.union(event.tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)
@@ -1615,6 +1700,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 
@@ -1671,6 +1763,9 @@ def event_from_json(j, siem_friendly=False):
             data = j["data"]
         kwargs["data"] = data
         event = make_event(**kwargs)
+        event_uuid = j.get("uuid", None)
+        if event_uuid is not None:
+            event._uuid = uuid.UUID(event_uuid.split(":")[-1])
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
@@ -1680,6 +1775,10 @@ def event_from_json(j, siem_friendly=False):
         parent_id = j.get("parent", None)
         if parent_id is not None:
             event._parent_id = parent_id
+        parent_uuid = j.get("parent_uuid", None)
+        if parent_uuid is not None:
+            parent_type, parent_uuid = parent_uuid.split(":", 1)
+            event._parent_uuid = parent_type + ":" + str(uuid.UUID(parent_uuid))
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")

bbot/core/helpers/bloom.py

@@ -5,14 +5,14 @@ import mmap
 
 class BloomFilter:
     """
-    Simple bloom filter implementation capable of rougly 400K lookups/s.
+    Simple bloom filter implementation capable of roughly 400K lookups/s.
 
     BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track
     of which mutations have been tried so far.
 
     A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
     A python hash is 36 bytes. So if you wanted to store these in a set, this would take up
-    36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
+    36 * 10M * 2 (key+value) == 720 megabytes. So we save roughly 7 times the space.
     """
 
     def __init__(self, size=8000000):
@@ -64,8 +64,15 @@ class BloomFilter:
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

bbot/core/helpers/command.py

@@ -210,9 +210,10 @@ async def _write_proc_line(proc, chunk):
         return True
     except Exception as e:
         proc_args = [str(s) for s in getattr(proc, "args", [])]
-        command = " ".join(proc_args)
-        log.warning(f"Error writing line to stdin for command: {command}: {e}")
-        log.trace(traceback.format_exc())
+        command = " ".join(proc_args).strip()
+        if command:
+            log.warning(f"Error writing line to stdin for command: {command}: {e}")
+            log.trace(traceback.format_exc())
         return False
 
 
@@ -268,11 +269,11 @@ def _prepare_command_kwargs(self, command, kwargs):
         (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
     """
     # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
-    if not "limit" in kwargs:
+    if "limit" not in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100
-    if not "stdout" in kwargs:
+    if "stdout" not in kwargs:
         kwargs["stdout"] = asyncio.subprocess.PIPE
-    if not "stderr" in kwargs:
+    if "stderr" not in kwargs:
         kwargs["stderr"] = asyncio.subprocess.PIPE
     sudo = kwargs.pop("sudo", False)
 
@@ -285,7 +286,7 @@ def _prepare_command_kwargs(self, command, kwargs):
 
     # use full path of binary, if not already specified
     binary = command[0]
-    if not "/" in binary:
+    if "/" not in binary:
         binary_full_path = which(binary)
         if binary_full_path is None:
             raise SubprocessError(f'Command "{binary}" was not found')
@@ -295,7 +296,7 @@ def _prepare_command_kwargs(self, command, kwargs):
     if sudo and os.geteuid() != 0:
         self.depsinstaller.ensure_root()
         env["SUDO_ASKPASS"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve())
-        env["BBOT_SUDO_PASS"] = self.depsinstaller._sudo_password
+        env["BBOT_SUDO_PASS"] = self.depsinstaller.encrypted_sudo_pw
         kwargs["env"] = env
 
         PATH = os.environ.get("PATH", "")