bbot 2.3.0.5354rc0__tar.gz → 2.3.0.5362rc0__tar.gz

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5354rc0
+Version: 2.3.0.5362rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
@@ -294,6 +294,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/README.md

@@ -240,6 +240,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5354rc"
+__version__ = "v2.3.0.5362rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -49,7 +49,8 @@ class DepsInstaller:
         self.minimal_git_config.touch()
         os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
 
-        self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
+        self.deps_config = self.parent_helper.config.get("deps", {})
+        self.deps_behavior = self.deps_config.get("behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
         if sys.prefix != sys.base_prefix:

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/db/sql/models.py

@@ -141,8 +141,8 @@ class Target(BBOTBaseModel, table=True):
     seeds: List = Field(default=[], sa_type=JSON)
     whitelist: List = Field(default=None, sa_type=JSON)
     blacklist: List = Field(default=[], sa_type=JSON)
-    hash: str = Field(sa_column=Column("hash", String, unique=True, primary_key=True, index=True))
-    scope_hash: str = Field(sa_column=Column("scope_hash", String, index=True))
-    seed_hash: str = Field(sa_column=Column("seed_hashhash", String, index=True))
-    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String, index=True))
-    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String, index=True))
+    hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
+    scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True))
+    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True))
+    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True))

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/defaults.yml

@@ -112,6 +112,13 @@ engine:
 deps:
   ffuf:
     version: "2.1.0"
+  # How to handle installation of module dependencies
+  # Choices are:
+  #  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
+  #  - retry_failed - try again to install failed dependencies
+  #  - ignore_failed - run the scan regardless of what happens with dependency installation
+  #  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
+  behavior: abort_on_failure
 
 ### ADVANCED OPTIONS ###
 
@@ -129,14 +136,6 @@ dnsresolve: True
 # Cloud provider tagging
 cloudcheck: True
 
-# How to handle installation of module dependencies
-# Choices are:
-#  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
-#  - retry_failed - try again to install failed dependencies
-#  - ignore_failed - run the scan regardless of what happens with dependency installation
-#  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
-deps_behavior: abort_on_failure
-
 # Strip querystring from URLs by default
 url_querystring_remove: True
 # When query string is retained, by default collapse parameter values down to a single value per parameter

bbot-2.3.0.5362rc0/bbot/modules/output/mysql.py

@@ -0,0 +1,51 @@
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class MySQL(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a MySQL database"}
+    options = {
+        "username": "root",
+        "password": "bbotislife",
+        "host": "localhost",
+        "port": 3306,
+        "database": "bbot",
+    }
+    options_desc = {
+        "username": "The username to connect to MySQL",
+        "password": "The password to connect to MySQL",
+        "host": "The server running MySQL",
+        "port": "The port to connect to MySQL",
+        "database": "The database name to connect to",
+    }
+    deps_pip = ["sqlmodel", "aiomysql"]
+    protocol = "mysql+aiomysql"
+
+    async def create_database(self):
+        from sqlalchemy import text
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        # Create the engine for the initial connection to the server
+        initial_engine = create_async_engine(self.connection_string().rsplit("/", 1)[0])
+
+        async with initial_engine.connect() as conn:
+            # Check if the database exists
+            result = await conn.execute(text(f"SHOW DATABASES LIKE '{self.database}'"))
+            database_exists = result.scalar() is not None
+
+            # Create the database if it does not exist
+            if not database_exists:
+                # Use aiomysql directly to create the database
+                import aiomysql
+
+                raw_conn = await aiomysql.connect(
+                    user=self.username,
+                    password=self.password,
+                    host=self.host,
+                    port=self.port,
+                )
+                try:
+                    async with raw_conn.cursor() as cursor:
+                        await cursor.execute(f"CREATE DATABASE {self.database}")
+                finally:
+                    await raw_conn.ensure_closed()

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/modules/templates/sql.py

@@ -1,3 +1,4 @@
+from contextlib import suppress
 from sqlmodel import SQLModel
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
@@ -88,3 +89,7 @@ class SQLTemplate(BaseOutputModule):
         if self.database:
             connection_string += f"/{self.database}"
         return connection_string
+
+    async def cleanup(self):
+        with suppress(Exception):
+            await self.engine.dispose()

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/bbot/test/test.conf

@@ -36,6 +36,8 @@ dns:
     - example.com
     - evilcorp.com
     - one
+deps:
+  behavior: retry_failed
 engine:
   debug: true
 agent_url: ws://127.0.0.1:8765

bbot-2.3.0.5362rc0/bbot/test/test_step_2/module_tests/test_module_mysql.py

@@ -0,0 +1,76 @@
+import asyncio
+import time
+
+from .base import ModuleTestBase
+
+
+class TestMySQL(ModuleTestBase):
+    targets = ["evilcorp.com"]
+    skip_distro_tests = True
+
+    async def setup_before_prep(self, module_test):
+        process = await asyncio.create_subprocess_exec(
+            "docker",
+            "run",
+            "--name",
+            "bbot-test-mysql",
+            "--rm",
+            "-e",
+            "MYSQL_ROOT_PASSWORD=bbotislife",
+            "-e",
+            "MYSQL_DATABASE=bbot",
+            "-p",
+            "3306:3306",
+            "-d",
+            "mysql",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+
+        import aiomysql
+
+        # wait for the container to start
+        start_time = time.time()
+        while True:
+            try:
+                conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+                conn.close()
+                break
+            except Exception as e:
+                if time.time() - start_time > 60:  # timeout after 60 seconds
+                    self.log.error("MySQL server did not start in time.")
+                    raise e
+                await asyncio.sleep(1)
+
+        if process.returncode != 0:
+            self.log.error(f"Failed to start MySQL server: {stderr.decode()}")
+
+    async def check(self, module_test, events):
+        import aiomysql
+
+        # Connect to the MySQL database
+        conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+
+        try:
+            async with conn.cursor() as cur:
+                await cur.execute("SELECT * FROM event")
+                events = await cur.fetchall()
+                assert len(events) == 3, "No events found in MySQL database"
+
+                await cur.execute("SELECT * FROM scan")
+                scans = await cur.fetchall()
+                assert len(scans) == 1, "No scans found in MySQL database"
+
+                await cur.execute("SELECT * FROM target")
+                targets = await cur.fetchall()
+                assert len(targets) == 1, "No targets found in MySQL database"
+        finally:
+            conn.close()
+            process = await asyncio.create_subprocess_exec(
+                "docker", "stop", "bbot-test-mysql", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                raise Exception(f"Failed to stop MySQL server: {stderr.decode()}")

{bbot-2.3.0.5354rc0 → bbot-2.3.0.5362rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.3.0.5354rc"
+version = "v2.3.0.5362rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",