bbot 2.1.2.5156rc0__tar.gz → 2.2.0.5263rc0__tar.gz

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.2.5156rc0
+Version: 2.2.0.5263rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
@@ -30,6 +30,7 @@ Requires-Dist: lxml (>=4.9.2,<6.0.0)
 Requires-Dist: mmh3 (>=4.1,<6.0)
 Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
 Requires-Dist: psutil (>=5.9.4,<7.0.0)
+Requires-Dist: puremagic (>=1.28,<2.0)
 Requires-Dist: pycryptodome (>=3.17,<4.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pyjwt (>=2.7.0,<3.0.0)
@@ -410,6 +411,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
     - **Modules**
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
+        - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/README.md

@@ -357,6 +357,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
     - **Modules**
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
+        - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.2.5156rc"
+__version__ = "v2.2.0.5263rc"
 
 from .scanner import Scanner, Preset

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/config/logger.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import atexit
 import logging
@@ -9,6 +10,7 @@ from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
+from ..multiprocess import SHARED_INTERPRETER_STATE
 
 
 debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
@@ -65,8 +67,9 @@ class BBOTLogger:
 
         self.listener = None
 
-        self.process_name = multiprocessing.current_process().name
-        if self.process_name == "MainProcess":
+        # if we haven't set up logging yet, do it now
+        if not "_BBOT_LOGGING_SETUP" in os.environ:
+            os.environ["_BBOT_LOGGING_SETUP"] = "1"
             self.queue = multiprocessing.Queue()
             self.setup_queue_handler()
             # Start the QueueListener
@@ -113,7 +116,7 @@ class BBOTLogger:
 
         self.core_logger.setLevel(log_level)
         # disable asyncio logging for child processes
-        if self.process_name != "MainProcess":
+        if not SHARED_INTERPRETER_STATE.is_main_process:
             logging.getLogger("asyncio").setLevel(logging.ERROR)
 
     def addLoggingLevel(self, levelName, levelNum, methodName=None):

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/core.py

@@ -6,6 +6,7 @@ from contextlib import suppress
 from omegaconf import OmegaConf
 
 from bbot.errors import BBOTError
+from .multiprocess import SHARED_INTERPRETER_STATE
 
 
 DEFAULT_CONFIG = None
@@ -41,9 +42,23 @@ class BBOTCore:
         self.logger
         self.log = logging.getLogger("bbot.core")
 
+        self._prep_multiprocessing()
+
+    def _prep_multiprocessing(self):
         import multiprocessing
+        from .helpers.process import BBOTProcess
+
+        if SHARED_INTERPRETER_STATE.is_main_process:
+            # if this is the main bbot process, set the logger and queue for the first time
+            from functools import partialmethod
 
-        self.process_name = multiprocessing.current_process().name
+            BBOTProcess.__init__ = partialmethod(
+                BBOTProcess.__init__, log_level=self.logger.log_level, log_queue=self.logger.queue
+            )
+
+        # this makes our process class the default for process pools, etc.
+        mp_context = multiprocessing.get_context("spawn")
+        mp_context.Process = BBOTProcess
 
     @property
     def home(self):
@@ -187,12 +202,14 @@ class BBOTCore:
         if os.environ.get("BBOT_TESTING", "") == "True":
             process = self.create_thread(*args, **kwargs)
         else:
-            if self.process_name == "MainProcess":
+            if SHARED_INTERPRETER_STATE.is_scan_process:
                 from .helpers.process import BBOTProcess
 
                 process = BBOTProcess(*args, **kwargs)
             else:
-                raise BBOTError(f"Tried to start server from process {self.process_name}")
+                import multiprocessing
+
+                raise BBOTError(f"Tried to start server from process {multiprocessing.current_process().name}")
         process.daemon = True
         return process
 

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/engine.py

@@ -10,6 +10,7 @@ import traceback
 import contextlib
 import contextvars
 import zmq.asyncio
+import multiprocessing
 from pathlib import Path
 from concurrent.futures import CancelledError
 from contextlib import asynccontextmanager, suppress
@@ -17,6 +18,7 @@ from contextlib import asynccontextmanager, suppress
 from bbot.core import CORE
 from bbot.errors import BBOTEngineError
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
 from bbot.core.helpers.misc import rand_string, in_exception_chain
 
 
@@ -264,10 +266,8 @@ class EngineClient(EngineBase):
         return [s for s in self.CMDS if isinstance(s, str)]
 
     def start_server(self):
-        import multiprocessing
-
         process_name = multiprocessing.current_process().name
-        if process_name == "MainProcess":
+        if SHARED_INTERPRETER_STATE.is_scan_process:
             kwargs = dict(self.server_kwargs)
             # if we're in tests, we use a single event loop to avoid weird race conditions
             # this allows us to more easily mock http, etc.

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/event/base.py

@@ -503,12 +503,13 @@ class BaseEvent:
             for t in list(self.tags):
                 if t.startswith("distance-"):
                     self.remove_tag(t)
-            if scope_distance == 0:
-                self.add_tag("in-scope")
-                self.remove_tag("affiliate")
-            else:
-                self.remove_tag("in-scope")
-                self.add_tag(f"distance-{new_scope_distance}")
+            if self.host:
+                if scope_distance == 0:
+                    self.add_tag("in-scope")
+                    self.remove_tag("affiliate")
+                else:
+                    self.remove_tag("in-scope")
+                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
@@ -1002,13 +1003,15 @@ class ClosestHostEvent(DictHostEvent):
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -1018,20 +1021,21 @@ class ClosestHostEvent(DictHostEvent):
 class DictPathEvent(DictEvent):
     def sanitize_data(self, data):
         new_data = dict(data)
+        new_data["path"] = str(new_data["path"])
         file_blobs = getattr(self.scan, "_file_blobs", False)
         folder_blobs = getattr(self.scan, "_folder_blobs", False)
         blob = None
         try:
-            data_path = Path(data["path"])
-            if data_path.is_file():
+            self._data_path = Path(data["path"])
+            if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
-                    with open(data_path, "rb") as file:
+                    with open(self._data_path, "rb") as file:
                         blob = file.read()
-            elif data_path.is_dir():
+            elif self._data_path.is_dir():
                 self.add_tag("folder")
                 if folder_blobs:
-                    blob = self._tar_directory(data_path)
+                    blob = self._tar_directory(self._data_path)
         except KeyError:
             pass
         if blob:
@@ -1540,7 +1544,25 @@ class WAF(DictHostEvent):
 
 
 class FILESYSTEM(DictPathEvent):
-    pass
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self._data_path.is_file():
+            # detect type of file content using magic
+            from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+            extension, mime_type, description, confidence = get_magic_info(self.data["path"])
+            self.data["magic_extension"] = extension
+            self.data["magic_mime_type"] = mime_type
+            self.data["magic_description"] = description
+            self.data["magic_confidence"] = confidence
+            # detection compression
+            compression = get_compression(mime_type)
+            if compression:
+                self.add_tag("compressed")
+                self.add_tag(f"{compression}-archive")
+                self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
 
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
@@ -1621,23 +1643,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
+            event.discovery_context = context
         if internal == True:
-            data.internal = True
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.tags = tags.union(event.tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/helpers/command.py

@@ -210,9 +210,10 @@ async def _write_proc_line(proc, chunk):
         return True
     except Exception as e:
         proc_args = [str(s) for s in getattr(proc, "args", [])]
-        command = " ".join(proc_args)
-        log.warning(f"Error writing line to stdin for command: {command}: {e}")
-        log.trace(traceback.format_exc())
+        command = " ".join(proc_args).strip()
+        if command:
+            log.warning(f"Error writing line to stdin for command: {command}: {e}")
+            log.trace(traceback.format_exc())
         return False
 
 

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -44,6 +44,11 @@ class DepsInstaller:
         self.parent_helper.mkdir(self.command_status)
         self.setup_status = self.read_setup_status()
 
+        # make sure we're using a minimal git config
+        self.minimal_git_config = self.data_dir / "minimal_git.config"
+        self.minimal_git_config.touch()
+        os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
+
         self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
@@ -350,6 +355,7 @@ class DepsInstaller:
             "make": "make",
             "gcc": "gcc",
             "bash": "bash",
+            "which": "which",
         }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):

bbot-2.2.0.5263rc0/bbot/core/helpers/libmagic.py

@@ -0,0 +1,68 @@
+import puremagic
+
+
+def get_magic_info(file):
+
+    magic_detections = puremagic.magic_file(file)
+    if magic_detections:
+        magic_detections.sort(key=lambda x: x.confidence, reverse=True)
+        detection = magic_detections[0]
+        return detection.extension, detection.mime_type, detection.name, detection.confidence
+    return "", "", "", 0
+
+
+def get_compression(mime_type):
+    mime_type = mime_type.lower()
+    # from https://github.com/cdgriffith/puremagic/blob/master/puremagic/magic_data.json
+    compression_map = {
+        "application/gzip": "gzip",  # Gzip compressed file
+        "application/zip": "zip",  # Zip archive
+        "application/x-bzip2": "bzip2",  # Bzip2 compressed file
+        "application/x-xz": "xz",  # XZ compressed file
+        "application/x-7z-compressed": "7z",  # 7-Zip archive
+        "application/vnd.rar": "rar",  # RAR archive
+        "application/x-lzma": "lzma",  # LZMA compressed file
+        "application/x-compress": "compress",  # Unix compress file
+        "application/zstd": "zstd",  # Zstandard compressed file
+        "application/x-lz4": "lz4",  # LZ4 compressed file
+        "application/x-tar": "tar",  # Tar archive
+        "application/x-zip-compressed-fb2": "zip",  # Zip archive (FB2)
+        "application/epub+zip": "zip",  # EPUB book (Zip archive)
+        "application/pak": "pak",  # PAK archive
+        "application/x-lha": "lha",  # LHA archive
+        "application/arj": "arj",  # ARJ archive
+        "application/vnd.ms-cab-compressed": "cab",  # Microsoft Cabinet archive
+        "application/x-sit": "sit",  # StuffIt archive
+        "application/binhex": "binhex",  # BinHex encoded file
+        "application/x-lrzip": "lrzip",  # Long Range ZIP
+        "application/x-alz": "alz",  # ALZip archive
+        "application/x-tgz": "tgz",  # Gzip compressed Tar archive
+        "application/x-gzip": "gzip",  # Gzip compressed file
+        "application/x-lzip": "lzip",  # Lzip compressed file
+        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/vnd.comicbook+zip": "zip",  # Comic book archive (Zip)
+        "application/vnd.palm": "palm",  # Palm OS data
+        "application/fictionbook2+zip": "zip",  # FictionBook 2.0 (Zip)
+        "application/fictionbook3+zip": "zip",  # FictionBook 3.0 (Zip)
+        "application/x-cpio": "cpio",  # CPIO archive
+        "application/x-java-pack200": "pack200",  # Java Pack200 archive
+        "application/x-par2": "par2",  # PAR2 recovery file
+        "application/x-rar-compressed": "rar",  # RAR archive
+        "application/java-archive": "zip",  # Java Archive (JAR)
+        "application/x-webarchive": "zip",  # Web archive (Zip)
+        "application/vnd.android.package-archive": "zip",  # Android package (APK)
+        "application/x-itunes-ipa": "zip",  # iOS application archive (IPA)
+        "application/x-stuffit": "sit",  # StuffIt archive
+        "application/x-archive": "ar",  # Unix archive
+        "application/x-qpress": "qpress",  # Qpress archive
+        "application/x-xar": "xar",  # XAR archive
+        "application/x-ace": "ace",  # ACE archive
+        "application/x-zoo": "zoo",  # Zoo archive
+        "application/x-arc": "arc",  # ARC archive
+        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/vnd.comicbook-rar": "rar",  # Comic book archive (RAR)
+    }
+
+    return compression_map.get(mime_type, "")

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/helpers/process.py

@@ -1,15 +1,11 @@
 import logging
 import traceback
 import threading
-import multiprocessing
 from multiprocessing.context import SpawnProcess
 
 from .misc import in_exception_chain
 
 
-current_process = multiprocessing.current_process()
-
-
 class BBOTThread(threading.Thread):
 
     default_name = "default bbot thread"
@@ -57,17 +53,3 @@ class BBOTProcess(SpawnProcess):
             if not in_exception_chain(e, (KeyboardInterrupt,)):
                 log.warning(f"Error in {self.name}: {e}")
             log.trace(traceback.format_exc())
-
-
-if current_process.name == "MainProcess":
-    # if this is the main bbot process, set the logger and queue for the first time
-    from bbot.core import CORE
-    from functools import partialmethod
-
-    BBOTProcess.__init__ = partialmethod(
-        BBOTProcess.__init__, log_level=CORE.logger.log_level, log_queue=CORE.logger.queue
-    )
-
-# this makes our process class the default for process pools, etc.
-mp_context = multiprocessing.get_context("spawn")
-mp_context.Process = BBOTProcess

bbot-2.2.0.5263rc0/bbot/core/multiprocess.py

@@ -0,0 +1,58 @@
+import os
+import atexit
+from contextlib import suppress
+
+
+class SharedInterpreterState:
+    """
+    A class to track the primary BBOT process.
+
+    Used to prevent spawning multiple unwanted processes with multiprocessing.
+    """
+
+    def __init__(self):
+        self.main_process_var_name = "_BBOT_MAIN_PID"
+        self.scan_process_var_name = "_BBOT_SCAN_PID"
+        atexit.register(self.cleanup)
+
+    @property
+    def is_main_process(self):
+        is_main_process = self.main_pid == os.getpid()
+        return is_main_process
+
+    @property
+    def is_scan_process(self):
+        is_scan_process = os.getpid() == self.scan_pid
+        return is_scan_process
+
+    @property
+    def main_pid(self):
+        main_pid = int(os.environ.get(self.main_process_var_name, 0))
+        if main_pid == 0:
+            main_pid = os.getpid()
+            # if main PID is not set, set it to the current PID
+            os.environ[self.main_process_var_name] = str(main_pid)
+        return main_pid
+
+    @property
+    def scan_pid(self):
+        scan_pid = int(os.environ.get(self.scan_process_var_name, 0))
+        if scan_pid == 0:
+            scan_pid = os.getpid()
+            # if scan PID is not set, set it to the current PID
+            os.environ[self.scan_process_var_name] = str(scan_pid)
+        return scan_pid
+
+    def update_scan_pid(self):
+        os.environ[self.scan_process_var_name] = str(os.getpid())
+
+    def cleanup(self):
+        with suppress(Exception):
+            if self.is_main_process:
+                with suppress(KeyError):
+                    del os.environ[self.main_process_var_name]
+                with suppress(KeyError):
+                    del os.environ[self.scan_process_var_name]
+
+
+SHARED_INTERPRETER_STATE = SharedInterpreterState()

{bbot-2.1.2.5156rc0 → bbot-2.2.0.5263rc0}/bbot/core/shared_deps.py

@@ -79,13 +79,23 @@ DEP_CHROMIUM = [
         "ignore_errors": True,
     },
     {
-        "name": "Install Chromium dependencies (Debian)",
+        "name": "Install Chromium dependencies (Ubuntu 24.04)",
         "package": {
-            "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
+            "name": "libasound2t64,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
             "state": "present",
         },
         "become": True,
-        "when": "ansible_facts['os_family'] == 'Debian'",
+        "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04'",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install Chromium dependencies (Other Debian-based)",
+        "package": {
+            "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
+            "state": "present",
+        },
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian' and not (ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04')",
         "ignore_errors": True,
     },
     {
@@ -149,6 +159,39 @@ DEP_MASSCAN = [
     },
 ]
 
+DEP_JAVA = [
+    {
+        "name": "Check if Java is installed",
+        "command": "which java",
+        "register": "java_installed",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install latest JRE (Debian)",
+        "package": {"name": ["default-jre"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Arch)",
+        "package": {"name": ["jre-openjdk"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Archlinux' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Fedora)",
+        "package": {"name": ["which", "java-latest-openjdk-headless"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'RedHat' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Alpine)",
+        "package": {"name": ["openjdk11"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Alpine' and java_installed.rc != 0",
+    },
+]
+
 # shared module dependencies -- ffuf, massdns, chromium, etc.
 SHARED_DEPS = {}
 for var, val in list(locals().items()):