bbot 2.3.0.5336rc0__tar.gz → 2.3.0.5362rc0__tar.gz

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5336rc0
+Version: 2.3.0.5362rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
@@ -249,10 +249,10 @@ flags:
 
 ```bash
 # everything everywhere all at once
-bbot -t evilcorp.com -p kitchen-sink
+bbot -t evilcorp.com -p kitchen-sink --allow-deadly
 
 # roughly equivalent to:
-bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots
+bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly
 ```
 
 <!-- BBOT KITCHEN-SINK PRESET EXPANDABLE -->
@@ -294,6 +294,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/README.md

@@ -195,10 +195,10 @@ flags:
 
 ```bash
 # everything everywhere all at once
-bbot -t evilcorp.com -p kitchen-sink
+bbot -t evilcorp.com -p kitchen-sink --allow-deadly
 
 # roughly equivalent to:
-bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots
+bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly
 ```
 
 <!-- BBOT KITCHEN-SINK PRESET EXPANDABLE -->
@@ -240,6 +240,24 @@ Click the graph below to explore the [inner workings](https://www.blacklanternse
 
 [![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)
 
+## Output Modules
+
+- [Neo4j](docs/scanning/output.md#neo4j)
+- [Teams](docs/scanning/output.md#teams)
+- [Discord](docs/scanning/output.md#discord)
+- [Slack](docs/scanning/output.md#slack)
+- [Postgres](docs/scanning/output.md#postgres)
+- [MySQL](docs/scanning/output.md#mysql)
+- [SQLite](docs/scanning/output.md#sqlite)
+- [Splunk](docs/scanning/output.md#splunk)
+- [Elasticsearch](docs/scanning/output.md#elasticsearch)
+- [CSV](docs/scanning/output.md#csv)
+- [JSON](docs/scanning/output.md#json)
+- [HTTP](docs/scanning/output.md#http)
+- [Websocket](docs/scanning/output.md#websocket)
+
+...and [more](docs/scanning/output.md)!
+
 ## BBOT as a Python Library
 
 #### Synchronous

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5336rc"
+__version__ = "v2.3.0.5362rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -49,7 +49,8 @@ class DepsInstaller:
         self.minimal_git_config.touch()
         os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
 
-        self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
+        self.deps_config = self.parent_helper.config.get("deps", {})
+        self.deps_behavior = self.deps_config.get("behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
         if sys.prefix != sys.base_prefix:

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/db/sql/models.py

@@ -141,8 +141,8 @@ class Target(BBOTBaseModel, table=True):
     seeds: List = Field(default=[], sa_type=JSON)
     whitelist: List = Field(default=None, sa_type=JSON)
     blacklist: List = Field(default=[], sa_type=JSON)
-    hash: str = Field(sa_column=Column("hash", String, unique=True, primary_key=True, index=True))
-    scope_hash: str = Field(sa_column=Column("scope_hash", String, index=True))
-    seed_hash: str = Field(sa_column=Column("seed_hashhash", String, index=True))
-    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String, index=True))
-    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String, index=True))
+    hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
+    scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True))
+    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True))
+    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True))

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/defaults.yml

@@ -112,6 +112,13 @@ engine:
 deps:
   ffuf:
     version: "2.1.0"
+  # How to handle installation of module dependencies
+  # Choices are:
+  #  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
+  #  - retry_failed - try again to install failed dependencies
+  #  - ignore_failed - run the scan regardless of what happens with dependency installation
+  #  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
+  behavior: abort_on_failure
 
 ### ADVANCED OPTIONS ###
 
@@ -129,14 +136,6 @@ dnsresolve: True
 # Cloud provider tagging
 cloudcheck: True
 
-# How to handle installation of module dependencies
-# Choices are:
-#  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
-#  - retry_failed - try again to install failed dependencies
-#  - ignore_failed - run the scan regardless of what happens with dependency installation
-#  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
-deps_behavior: abort_on_failure
-
 # Strip querystring from URLs by default
 url_querystring_remove: True
 # When query string is retained, by default collapse parameter values down to a single value per parameter

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/modules/internal/cloudcheck.py

@@ -1,3 +1,5 @@
+from contextlib import suppress
+
 from bbot.modules.base import BaseInterceptModule
 
 
@@ -28,15 +30,28 @@ class CloudCheck(BaseInterceptModule):
         if self.dummy_modules is None:
             self.make_dummy_modules()
         # cloud tagging by hosts
-        hosts_to_check = set(str(s) for s in event.resolved_hosts)
-        # we use the original host, since storage buckets hostnames might be collapsed to _wildcard
-        hosts_to_check.add(str(event.host_original))
-        for host in hosts_to_check:
+        hosts_to_check = set(event.resolved_hosts)
+        with suppress(KeyError):
+            hosts_to_check.remove(event.host_original)
+        hosts_to_check = [event.host_original] + list(hosts_to_check)
+
+        for i, host in enumerate(hosts_to_check):
+            host_is_ip = self.helpers.is_ip(host)
             for provider, provider_type, subnet in self.helpers.cloudcheck(host):
                 if provider:
                     event.add_tag(f"{provider_type}-{provider}")
+                    if host_is_ip:
+                        event.add_tag(f"{provider_type}-ip")
+                    else:
+                        # if the original hostname is a cloud domain, tag it as such
+                        if i == 0:
+                            event.add_tag(f"{provider_type}-domain")
+                        # any children are tagged as CNAMEs
+                        else:
+                            event.add_tag(f"{provider_type}-cname")
 
         found = set()
+        str_hosts_to_check = [str(host) for host in hosts_to_check]
         # look for cloud assets in hosts, http responses
         # loop through each provider
         for provider in self.helpers.cloud.providers.values():
@@ -54,7 +69,7 @@ class CloudCheck(BaseInterceptModule):
                     if event.type == "HTTP_RESPONSE":
                         matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
                     elif event.type.startswith("DNS_NAME"):
-                        for host in hosts_to_check:
+                        for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:
                                 matches.append(match.groups())

bbot-2.3.0.5362rc0/bbot/modules/output/mysql.py

@@ -0,0 +1,51 @@
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class MySQL(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a MySQL database"}
+    options = {
+        "username": "root",
+        "password": "bbotislife",
+        "host": "localhost",
+        "port": 3306,
+        "database": "bbot",
+    }
+    options_desc = {
+        "username": "The username to connect to MySQL",
+        "password": "The password to connect to MySQL",
+        "host": "The server running MySQL",
+        "port": "The port to connect to MySQL",
+        "database": "The database name to connect to",
+    }
+    deps_pip = ["sqlmodel", "aiomysql"]
+    protocol = "mysql+aiomysql"
+
+    async def create_database(self):
+        from sqlalchemy import text
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        # Create the engine for the initial connection to the server
+        initial_engine = create_async_engine(self.connection_string().rsplit("/", 1)[0])
+
+        async with initial_engine.connect() as conn:
+            # Check if the database exists
+            result = await conn.execute(text(f"SHOW DATABASES LIKE '{self.database}'"))
+            database_exists = result.scalar() is not None
+
+            # Create the database if it does not exist
+            if not database_exists:
+                # Use aiomysql directly to create the database
+                import aiomysql
+
+                raw_conn = await aiomysql.connect(
+                    user=self.username,
+                    password=self.password,
+                    host=self.host,
+                    port=self.port,
+                )
+                try:
+                    async with raw_conn.cursor() as cursor:
+                        await cursor.execute(f"CREATE DATABASE {self.database}")
+                finally:
+                    await raw_conn.ensure_closed()

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/modules/portscan.py

@@ -6,6 +6,9 @@ from radixtarget import RadixTarget
 from bbot.modules.base import BaseModule
 
 
+# TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN.
+
+
 class portscan(BaseModule):
     flags = ["active", "portscan", "safe"]
     watched_events = ["IP_ADDRESS", "IP_RANGE", "DNS_NAME"]
@@ -27,6 +30,8 @@ class portscan(BaseModule):
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
+        "cdn_tags": "cdn-",
+        "allowed_cdn_ports": None,
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -39,6 +44,8 @@ class portscan(BaseModule):
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
+        "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,cloud'",
+        "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs",
     }
     deps_common = ["masscan"]
     batch_size = 1000000
@@ -60,7 +67,15 @@ class portscan(BaseModule):
             try:
                 self.helpers.parse_port_string(self.ports)
             except ValueError as e:
-                return False, f"Error parsing ports: {e}"
+                return False, f"Error parsing ports '{self.ports}': {e}"
+        self.cdn_tags = [t.strip() for t in self.config.get("cdn_tags", "").split(",")]
+        self.allowed_cdn_ports = self.config.get("allowed_cdn_ports", None)
+        if self.allowed_cdn_ports is not None:
+            try:
+                self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(",")]
+            except Exception as e:
+                return False, f"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}"
+
         # whether we've finished scanning our original scan targets
         self.scanned_initial_targets = False
         # keeps track of individual scanned IPs and their open ports
@@ -227,9 +242,20 @@ class portscan(BaseModule):
             parent=parent_event,
             context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}",
         )
-        await self.emit_event(event)
+
+        await self.emit_event(event, abort_if=self.abort_if)
         return event
 
+    def abort_if(self, event):
+        if self.allowed_cdn_ports is not None:
+            # if the host is a CDN
+            for cdn_tag in self.cdn_tags:
+                if any(t.startswith(str(cdn_tag)) for t in event.tags):
+                    # and if its port isn't in the list of allowed CDN ports
+                    if event.port not in self.allowed_cdn_ports:
+                        return True, "event is a CDN and port is not in the allowed list"
+        return False
+
     def parse_json_line(self, line):
         try:
             j = json.loads(line)

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/modules/templates/sql.py

@@ -1,3 +1,4 @@
+from contextlib import suppress
 from sqlmodel import SQLModel
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
@@ -88,3 +89,7 @@ class SQLTemplate(BaseOutputModule):
         if self.database:
             connection_string += f"/{self.database}"
         return connection_string
+
+    async def cleanup(self):
+        with suppress(Exception):
+            await self.engine.dispose()

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/modules/templates/subdomain_enum.py

@@ -169,7 +169,7 @@ class subdomain_enum(BaseModule):
         if any(t.startswith("cloud-") for t in event.tags):
             is_cloud = True
         # reject if it's a cloud resource and not in our target
-        if is_cloud and event not in self.scan.target:
+        if is_cloud and event not in self.scan.target.whitelist:
             return False, "Event is a cloud resource and not a direct target"
         # optionally reject events with wildcards / errors
         if self.reject_wildcards:

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/test/test.conf

@@ -36,6 +36,8 @@ dns:
     - example.com
     - evilcorp.com
     - one
+deps:
+  behavior: retry_failed
 engine:
   debug: true
 agent_url: ws://127.0.0.1:8765

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py

@@ -51,6 +51,10 @@ class TestCloudCheck(ModuleTestBase):
             await module.handle_event(event)
             assert "cloud-amazon" in event.tags, f"{event} was not properly cloud-tagged"
 
+        assert "cloud-domain" in aws_event1.tags
+        assert "cloud-ip" in other_event2.tags
+        assert "cloud-cname" in other_event3.tags
+
         for event in (aws_event3, other_event1):
             await module.handle_event(event)
             assert "cloud-amazon" not in event.tags, f"{event} was improperly cloud-tagged"

bbot-2.3.0.5362rc0/bbot/test/test_step_2/module_tests/test_module_mysql.py

@@ -0,0 +1,76 @@
+import asyncio
+import time
+
+from .base import ModuleTestBase
+
+
+class TestMySQL(ModuleTestBase):
+    targets = ["evilcorp.com"]
+    skip_distro_tests = True
+
+    async def setup_before_prep(self, module_test):
+        process = await asyncio.create_subprocess_exec(
+            "docker",
+            "run",
+            "--name",
+            "bbot-test-mysql",
+            "--rm",
+            "-e",
+            "MYSQL_ROOT_PASSWORD=bbotislife",
+            "-e",
+            "MYSQL_DATABASE=bbot",
+            "-p",
+            "3306:3306",
+            "-d",
+            "mysql",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+
+        import aiomysql
+
+        # wait for the container to start
+        start_time = time.time()
+        while True:
+            try:
+                conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+                conn.close()
+                break
+            except Exception as e:
+                if time.time() - start_time > 60:  # timeout after 60 seconds
+                    self.log.error("MySQL server did not start in time.")
+                    raise e
+                await asyncio.sleep(1)
+
+        if process.returncode != 0:
+            self.log.error(f"Failed to start MySQL server: {stderr.decode()}")
+
+    async def check(self, module_test, events):
+        import aiomysql
+
+        # Connect to the MySQL database
+        conn = await aiomysql.connect(user="root", password="bbotislife", db="bbot", host="localhost")
+
+        try:
+            async with conn.cursor() as cur:
+                await cur.execute("SELECT * FROM event")
+                events = await cur.fetchall()
+                assert len(events) == 3, "No events found in MySQL database"
+
+                await cur.execute("SELECT * FROM scan")
+                scans = await cur.fetchall()
+                assert len(scans) == 1, "No scans found in MySQL database"
+
+                await cur.execute("SELECT * FROM target")
+                targets = await cur.fetchall()
+                assert len(targets) == 1, "No targets found in MySQL database"
+        finally:
+            conn.close()
+            process = await asyncio.create_subprocess_exec(
+                "docker", "stop", "bbot-test-mysql", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                raise Exception(f"Failed to stop MySQL server: {stderr.decode()}")

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/bbot/test/test_step_2/module_tests/test_module_portscan.py

@@ -109,10 +109,12 @@ class TestPortscan(ModuleTestBase):
                 if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module"
             ]
         )
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3
+        # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first
+        # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port.
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 4
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"])

{bbot-2.3.0.5336rc0 → bbot-2.3.0.5362rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.3.0.5336rc"
+version = "v2.3.0.5362rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",