bbot 2.3.0.5504rc0__tar.gz → 2.3.0.5515rc0__tar.gz

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5504rc0
+Version: 2.3.0.5515rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5504rc"
+__version__ = "v2.3.0.5515rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/core/event/base.py

@@ -590,7 +590,7 @@ class BaseEvent:
                         if t in ("spider-danger", "spider-max"):
                             self.add_tag(t)
         elif not self._dummy:
-            log.warning(f"Tried to set invalid parent on {self}: (got: {parent})")
+            log.warning(f"Tried to set invalid parent on {self}: (got: {repr(parent)} ({type(parent)}))")
 
     @property
     def parent_id(self):
@@ -1045,6 +1045,9 @@ class DictPathEvent(DictEvent):
         blob = None
         try:
             self._data_path = Path(data["path"])
+            # prepend the scan's home dir if the path is relative
+            if not self._data_path.is_absolute():
+                self._data_path = self.scan.home / self._data_path
             if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -20,11 +20,43 @@ log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
 
 class DepsInstaller:
+    CORE_DEPS = {
+        # core BBOT dependencies in the format of binary: package_name
+        # each one will only be installed if the binary is not found
+        "unzip": "unzip",
+        "zipinfo": "unzip",
+        "curl": "curl",
+        "git": "git",
+        "make": "make",
+        "gcc": "gcc",
+        "bash": "bash",
+        "which": "which",
+        "unrar": "unrar-free",
+        "tar": "tar",
+        # debian why are you like this
+        "7z": [
+            {
+                "name": "Install 7zip (Debian)",
+                "package": {"name": ["p7zip-full"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Debian'",
+            },
+            {
+                "name": "Install 7zip (Non-Debian)",
+                "package": {"name": ["p7zip"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] != 'Debian'",
+            },
+        ],
+    }
+
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.preset = self.parent_helper.preset
         self.core = self.preset.core
 
+        self.os_platform = os_platform()
+
         # respect BBOT's http timeout
         self.web_config = self.parent_helper.config.get("web", {})
         http_timeout = self.web_config.get("http_timeout", 30)
@@ -202,28 +234,32 @@ class DepsInstaller:
         """
         Install packages with the OS's default package manager (apt, pacman, dnf, etc.)
         """
-        packages_str = ",".join(packages)
+        args, kwargs = self._make_apt_ansible_args(packages)
+        success, err = self.ansible_run(module="package", args=args, **kwargs)
+        if success:
+            log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"')
+        else:
+            log.warning(
+                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
+            )
+            for p in packages:
+                log.warning(f" - {p}")
+        return success
+
+    def _make_apt_ansible_args(self, packages):
+        packages_str = ",".join(sorted(packages))
         log.info(f"Installing the following OS packages: {packages_str}")
         args = {"name": packages_str, "state": "present"}  # , "update_cache": True, "cache_valid_time": 86400}
         kwargs = {}
         # don't sudo brew
-        if os_platform() != "darwin":
+        if self.os_platform != "darwin":
             kwargs = {
                 "ansible_args": {
                     "ansible_become": True,
                     "ansible_become_method": "sudo",
                 }
             }
-        success, err = self.ansible_run(module="package", args=args, **kwargs)
-        if success:
-            log.info(f'Successfully installed OS packages "{packages_str}"')
-        else:
-            log.warning(
-                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
-            )
-            for p in packages:
-                log.warning(f" - {p}")
-        return success
+        return args, kwargs
 
     def shell(self, module, commands):
         tasks = []
@@ -269,7 +305,7 @@ class DepsInstaller:
             for task in tasks:
                 if "package" in task:
                     # special case for macos
-                    if os_platform() == "darwin":
+                    if self.os_platform == "darwin":
                         # don't sudo brew
                         task["become"] = False
                         # brew doesn't support update_cache
@@ -292,8 +328,8 @@ class DepsInstaller:
             },
             module=module,
             module_args=module_args,
-            quiet=not self.ansible_debug,
-            verbosity=(3 if self.ansible_debug else 0),
+            quiet=True,
+            verbosity=0,
             cancel_callback=lambda: None,
         )
 
@@ -303,7 +339,7 @@ class DepsInstaller:
         err = ""
         for e in res.events:
             if self.ansible_debug and not success:
-                log.debug(json.dumps(e, indent=4))
+                log.debug(json.dumps(e, indent=2))
             if e["event"] == "runner_on_failed":
                 err = e["event_data"]["res"]["msg"]
                 break
@@ -347,26 +383,30 @@ class DepsInstaller:
 
     def install_core_deps(self):
         to_install = set()
+        to_install_friendly = set()
+        playbook = []
         self._install_sudo_askpass()
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
-        # command: package_name
-        core_deps = {
-            "unzip": "unzip",
-            "zipinfo": "unzip",
-            "curl": "curl",
-            "git": "git",
-            "make": "make",
-            "gcc": "gcc",
-            "bash": "bash",
-            "which": "which",
-        }
-        for command, package_name in core_deps.items():
+        for command, package_name_or_playbook in self.CORE_DEPS.items():
             if not self.parent_helper.which(command):
-                to_install.add(package_name)
+                to_install_friendly.add(command)
+                if isinstance(package_name_or_playbook, str):
+                    to_install.add(package_name_or_playbook)
+                else:
+                    playbook.extend(package_name_or_playbook)
         if to_install:
+            playbook.append(
+                {
+                    "name": "Install Core BBOT Dependencies",
+                    "package": {"name": list(to_install), "state": "present"},
+                    "become": True,
+                }
+            )
+        if playbook:
+            log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()
-            self.apt_install(list(to_install))
+            self.ansible_run(tasks=playbook)
 
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup:

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/modules/filedownload.py

@@ -179,7 +179,9 @@ class filedownload(BaseModule):
         if extension:
             filename = f"{filename}.{extension}"
             orig_filename = f"{orig_filename}.{extension}"
-        return orig_filename, self.download_dir / filename, base_url
+        file_destination = self.download_dir / filename
+        file_destination = self.helpers.truncate_filename(file_destination)
+        return orig_filename, file_destination, base_url
 
     async def report(self):
         if self.files_downloaded > 0:

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/modules/gowitness.py

@@ -142,6 +142,9 @@ class gowitness(BaseModule):
             url = screenshot["url"]
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
+            filename = filename.relative_to(self.scan.home)
+            # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
+            filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
             parent_event = event_dict[url]
             await self.emit_event(

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/test/test_step_2/module_tests/test_module_filedownload.py

@@ -60,3 +60,28 @@ trailer <</Root 1 0 R>>"""
         # we don't want html files
         html_files = list(download_dir.glob("*.html"))
         assert len(html_files) == 0, "HTML files were erroneously downloaded"
+
+
+class TestFileDownloadLongFilename(TestFileDownload):
+    async def setup_after_prep(self, module_test):
+        module_test.set_expect_requests(
+            {"uri": "/"},
+            {
+                "response_data": '<a href="/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity.txt"/>'
+            },
+        )
+        module_test.set_expect_requests(
+            {
+                "uri": "/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity.txt"
+            },
+            {
+                "response_data": "juicy stuff",
+            },
+        )
+
+    def check(self, module_test, events):
+        filesystem_events = [e for e in events if e.type == "FILESYSTEM"]
+        assert len(filesystem_events) == 1
+        file_path = Path(filesystem_events[0].data["path"])
+        assert file_path.is_file(), f"File not found at {file_path}"
+        assert file_path.read_text() == "juicy stuff", f"File at {file_path} does not contain the correct content"

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/bbot/test/test_step_2/module_tests/test_module_gowitness.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from .base import ModuleTestBase
 
 
@@ -102,3 +104,33 @@ class TestGoWitnessWithBlob(TestGowitness):
         webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"]
         assert webscreenshots, "failed to raise WEBSCREENSHOT events"
         assert all("blob" in e.data and e.data["blob"] for e in webscreenshots), "blob not found in WEBSCREENSHOT data"
+
+
+class TestGoWitnessLongFilename(TestGowitness):
+    """
+    Make sure long filenames are truncated properly
+    """
+
+    targets = [
+        "http://127.0.0.1:8888/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity"
+    ]
+    config_overrides = {"file_blobs": True}
+
+    async def setup_after_prep(self, module_test):
+        request_args = {
+            "uri": "/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity/blacklanternsecurity"
+        }
+        respond_args = {
+            "response_data": "<html><head><title>BBOT is life</title></head><body>BBOT is life</body></html>",
+            "headers": {"Server": "Apache/2.4.41 (Ubuntu)"},
+        }
+        module_test.set_expect_requests(request_args, respond_args)
+
+    def check(self, module_test, events):
+        webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"]
+        assert webscreenshots, "failed to raise WEBSCREENSHOT events"
+        assert len(webscreenshots) == 1
+        webscreenshot = webscreenshots[0]
+        filename = Path(webscreenshot.data["path"])
+        # sadly this file doesn't exist because gowitness doesn't truncate properly
+        assert not filename.exists()

{bbot-2.3.0.5504rc0 → bbot-2.3.0.5515rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.3.0.5504rc"
+version = "v2.3.0.5515rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",