bbot 2.1.0.5082rc0__tar.gz → 2.1.1__tar.gz

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.0.5082rc0
+Version: 2.1.1
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.0.5082rc"
+__version__ = "v2.1.1"
 
 from .scanner import Scanner, Preset

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/core/config/logger.py

@@ -5,6 +5,7 @@ from copy import copy
 import multiprocessing
 import logging.handlers
 from pathlib import Path
+from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
@@ -71,10 +72,36 @@ class BBOTLogger:
             # Start the QueueListener
             self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())
             self.listener.start()
-            atexit.register(self.listener.stop)
+            atexit.register(self.cleanup_logging)
 
         self.log_level = logging.INFO
 
+    def cleanup_logging(self):
+        # Close the queue handler
+        with suppress(Exception):
+            self.queue_handler.close()
+
+        # Clean root logger
+        root_logger = logging.getLogger()
+        for handler in list(root_logger.handlers):
+            with suppress(Exception):
+                root_logger.removeHandler(handler)
+            with suppress(Exception):
+                handler.close()
+
+        # Clean all other loggers
+        for logger in logging.Logger.manager.loggerDict.values():
+            if hasattr(logger, "handlers"):  # Logger, not PlaceHolder
+                for handler in list(logger.handlers):
+                    with suppress(Exception):
+                        logger.removeHandler(handler)
+                    with suppress(Exception):
+                        handler.close()
+
+        # Stop queue listener
+        with suppress(Exception):
+            self.listener.stop()
+
     def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
         if logging_queue is None:
             logging_queue = self.queue

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/core/helpers/depsinstaller/installer.py

@@ -342,7 +342,15 @@ class DepsInstaller:
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
         # command: package_name
-        core_deps = {"unzip": "unzip", "curl": "curl"}
+        core_deps = {
+            "unzip": "unzip",
+            "zipinfo": "unzip",
+            "curl": "curl",
+            "git": "git",
+            "make": "make",
+            "gcc": "gcc",
+            "bash": "bash",
+        }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):
                 to_install.add(package_name)

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/modules/gowitness.py

@@ -1,5 +1,5 @@
 import asyncio
-import sqlite3
+import aiosqlite
 import multiprocessing
 from pathlib import Path
 from contextlib import suppress
@@ -34,6 +34,7 @@ class gowitness(BaseModule):
         "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds",
     }
     deps_common = ["chromium"]
+    deps_pip = ["aiosqlite"]
     deps_ansible = [
         {
             "name": "Download gowitness",
@@ -72,7 +73,7 @@ class gowitness(BaseModule):
 
         # make sure we have a working chrome install
         chrome_test_pass = False
-        for binary in ("chrome", "chromium", custom_chrome_path):
+        for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path):
             binary_path = self.helpers.which(binary)
             if binary_path and Path(binary_path).is_file():
                 chrome_test_proc = await self.run_process([binary_path, "--version"])
@@ -136,7 +137,8 @@ class gowitness(BaseModule):
             return
 
         # emit web screenshots
-        for filename, screenshot in self.new_screenshots.items():
+        new_screenshots = await self.get_new_screenshots()
+        for filename, screenshot in new_screenshots.items():
             url = screenshot["url"]
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
@@ -150,7 +152,8 @@ class gowitness(BaseModule):
             )
 
         # emit URLs
-        for url, row in self.new_network_logs.items():
+        new_network_logs = await self.get_new_network_logs()
+        for url, row in new_network_logs.items():
             ip = row["ip"]
             status_code = row["status_code"]
             tags = [f"status-{status_code}", f"ip-{ip}", "spider-danger"]
@@ -168,7 +171,8 @@ class gowitness(BaseModule):
                 )
 
         # emit technologies
-        for _, row in self.new_technologies.items():
+        new_technologies = await self.get_new_technologies()
+        for _, row in new_technologies.items():
             parent_id = row["url_id"]
             parent_url = self.screenshots_taken[parent_id]
             parent_event = event_dict[parent_url]
@@ -207,59 +211,53 @@ class gowitness(BaseModule):
         command += ["--timeout", str(self.timeout)]
         return command
 
-    @property
-    def new_screenshots(self):
+    async def get_new_screenshots(self):
         screenshots = {}
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
                 con.text_factory = self.helpers.smart_decode
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM urls")
-                for row in res:
-                    row = dict(row)
-                    _id = row["id"]
-                    if _id not in self.screenshots_taken:
-                        self.screenshots_taken[_id] = row["url"]
-                        screenshots[_id] = row
+                async with con.execute("SELECT * FROM urls") as cur:
+                    async for row in cur:
+                        row = dict(row)
+                        _id = row["id"]
+                        if _id not in self.screenshots_taken:
+                            self.screenshots_taken[_id] = row["url"]
+                            screenshots[_id] = row
         return screenshots
 
-    @property
-    def new_network_logs(self):
+    async def get_new_network_logs(self):
         network_logs = dict()
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM network_logs")
-                for row in res:
-                    row = dict(row)
-                    url = row["final_url"]
-                    if url not in self.connections_logged:
-                        self.connections_logged.add(url)
-                        network_logs[url] = row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
+                async with con.execute("SELECT * FROM network_logs") as cur:
+                    async for row in cur:
+                        row = dict(row)
+                        url = row["final_url"]
+                        if url not in self.connections_logged:
+                            self.connections_logged.add(url)
+                            network_logs[url] = row
         return network_logs
 
-    @property
-    def new_technologies(self):
+    async def get_new_technologies(self):
         technologies = dict()
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM technologies")
-                for row in res:
-                    _id = row["id"]
-                    if _id not in self.technologies_found:
-                        self.technologies_found.add(_id)
-                        row = dict(row)
-                        technologies[_id] = row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
+                async with con.execute("SELECT * FROM technologies") as cur:
+                    async for row in cur:
+                        _id = row["id"]
+                        if _id not in self.technologies_found:
+                            self.technologies_found.add(_id)
+                            row = dict(row)
+                            technologies[_id] = row
         return technologies
 
-    def cur_execute(self, cur, query):
+    async def cur_execute(self, cur, query):
         try:
-            return cur.execute(query)
-        except sqlite3.OperationalError as e:
+            return await cur.execute(query)
+        except aiosqlite.OperationalError as e:
             self.warning(f"Error executing query: {query}: {e}")
             return []
 

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/modules/trufflehog.py

@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.82.11",
+        "version": "3.83.1",
         "config": "",
         "only_verified": True,
         "concurrency": 8,

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/modules/wpscan.py

@@ -33,7 +33,7 @@ class wpscan(BaseModule):
     deps_apt = ["curl", "make", "gcc"]
     deps_ansible = [
         {
-            "name": "Install Ruby Deps (Debian/Ubuntu)",
+            "name": "Install Ruby Deps (Debian)",
             "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"},
             "become": True,
             "when": "ansible_facts['os_family'] == 'Debian'",
@@ -48,7 +48,13 @@ class wpscan(BaseModule):
             "name": "Install Ruby Deps (Fedora)",
             "package": {"name": ["rubygems", "ruby-devel"], "state": "present"},
             "become": True,
-            "when": "ansible_facts['os_family'] == 'Fedora'",
+            "when": "ansible_facts['os_family'] == 'RedHat'",
+        },
+        {
+            "name": "Install Ruby Deps (Alpine)",
+            "package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Alpine'",
         },
         {
             "name": "Install wpscan gem",

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/bbot/test/conftest.py

@@ -13,17 +13,31 @@ from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async
 from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
+# silence stdout + trace
+root_logger = logging.getLogger()
+pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
+print(f"pytest_debug_file: {pytest_debug_file}")
+debug_handler = logging.FileHandler(pytest_debug_file)
+debug_handler.setLevel(logging.DEBUG)
+debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
+debug_handler.setFormatter(debug_format)
+root_logger.addHandler(debug_handler)
 
 test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
-if test_config.get("debug", False):
-    os.environ["BBOT_DEBUG"] = "True"
-    logging.getLogger("bbot").setLevel(logging.DEBUG)
-    CORE.logger.log_level = logging.DEBUG
-else:
-    # silence stdout + trace
-    root_logger = logging.getLogger()
-    for h in root_logger.handlers:
-        h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
+os.environ["BBOT_DEBUG"] = "True"
+CORE.logger.log_level = logging.DEBUG
+
+# silence all stderr output:
+stderr_handler = CORE.logger.log_handlers["stderr"]
+stderr_handler.setLevel(logging.CRITICAL)
+handlers = list(CORE.logger.listener.handlers)
+handlers.remove(stderr_handler)
+CORE.logger.listener.handlers = tuple(handlers)
+
+for h in root_logger.handlers:
+    h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
 
 CORE.merge_default(test_config)
 
@@ -33,6 +47,13 @@ def assert_all_responses_were_requested() -> bool:
     return False
 
 
+@pytest.fixture(autouse=True)
+def silence_live_logging():
+    for handler in logging.getLogger().handlers:
+        if type(handler).__name__ == "_LiveLoggingStreamHandler":
+            handler.setLevel(logging.CRITICAL)
+
+
 @pytest.fixture
 def bbot_httpserver():
     server = HTTPServer(host="127.0.0.1", port=8888, threaded=True)
@@ -202,20 +223,20 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):  # pragma: no
     errors = len(stats.get("error", []))
     failed = stats.get("failed", [])
 
-    print("\nTest Session Summary:")
-    print(f"Total tests run: {total_tests}")
-    print(
-        f"{GREEN}Passed: {passed}{RESET}, {RED}Failed: {len(failed)}{RESET}, {YELLOW}Skipped: {skipped}{RESET}, Errors: {errors}"
+    terminalreporter.write("\nTest Session Summary:")
+    terminalreporter.write(f"\nTotal tests run: {total_tests}")
+    terminalreporter.write(
+        f"\n{GREEN}Passed: {passed}{RESET}, {RED}Failed: {len(failed)}{RESET}, {YELLOW}Skipped: {skipped}{RESET}, Errors: {errors}"
     )
 
     if failed:
-        print(f"\n{RED}Detailed failed test report:{RESET}")
+        terminalreporter.write(f"\n{RED}Detailed failed test report:{RESET}")
         for item in failed:
             test_name = item.nodeid.split("::")[-1] if "::" in item.nodeid else item.nodeid
             file_and_line = f"{item.location[0]}:{item.location[1]}"  # File path and line number
-            print(f"{BLUE}Test Name: {test_name}{RESET} {CYAN}({file_and_line}){RESET}")
-            print(f"{RED}Location: {item.nodeid} at {item.location[0]}:{item.location[1]}{RESET}")
-            print(f"{RED}Failure details:\n{item.longreprtext}{RESET}")
+            terminalreporter.write(f"\n{BLUE}Test Name: {test_name}{RESET} {CYAN}({file_and_line}){RESET}")
+            terminalreporter.write(f"\n{RED}Location: {item.nodeid} at {item.location[0]}:{item.location[1]}{RESET}")
+            terminalreporter.write(f"\n{RED}Failure details:\n{item.longreprtext}{RESET}")
 
 
 # BELOW: debugging for frozen/hung tests

{bbot-2.1.0.5082rc0 → bbot-2.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.1.0.5082rc"
+version = "v2.1.1"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",
@@ -100,7 +100,7 @@ extend-exclude = "(test_step_1/test_manager_*)"
 [tool.poetry-dynamic-versioning]
 enable = false
 metadata = false
-format-jinja = 'v2.1.0{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
+format-jinja = 'v2.1.1{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
 
 [tool.poetry-dynamic-versioning.substitution]
 files = ["*/__init__.py"]