bbot 2.1.2.5173rc0__py3-none-any.whl → 2.1.2.5180rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.2.5173rc"
+__version__ = "v2.1.2.5180rc"
 
 from .scanner import Scanner, Preset

bbot/db/sql/models.py

@@ -0,0 +1,147 @@
+# This file contains SQLModel (Pydantic + SQLAlchemy) models for BBOT events, scans, and targets.
+# Used by the SQL output modules, but portable for outside use.
+
+import json
+import logging
+from datetime import datetime
+from pydantic import ConfigDict
+from typing import List, Optional
+from typing_extensions import Annotated
+from pydantic.functional_validators import AfterValidator
+from sqlmodel import inspect, Column, Field, SQLModel, JSON, String, DateTime as SQLADateTime
+
+
+log = logging.getLogger("bbot_server.models")
+
+
+def naive_datetime_validator(d: datetime):
+    """
+    Converts all dates into UTC, then drops timezone information.
+
+    This is needed to prevent inconsistencies in sqlite, because it is timezone-naive.
+    """
+    # drop timezone info
+    return d.replace(tzinfo=None)
+
+
+NaiveUTC = Annotated[datetime, AfterValidator(naive_datetime_validator)]
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        # handle datetime
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+class BBOTBaseModel(SQLModel):
+    model_config = ConfigDict(extra="ignore")
+
+    def __init__(self, *args, **kwargs):
+        self._validated = None
+        super().__init__(*args, **kwargs)
+
+    @property
+    def validated(self):
+        try:
+            if self._validated is None:
+                self._validated = self.__class__.model_validate(self)
+            return self._validated
+        except AttributeError:
+            return self
+
+    def to_json(self, **kwargs):
+        return json.dumps(self.validated.model_dump(), sort_keys=True, cls=CustomJSONEncoder, **kwargs)
+
+    @classmethod
+    def _pk_column_names(cls):
+        return [column.name for column in inspect(cls).primary_key]
+
+    def __hash__(self):
+        return hash(self.to_json())
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+
+### EVENT ###
+
+
+class Event(BBOTBaseModel, table=True):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        data = self._get_data(self.data, self.type)
+        self.data = {self.type: data}
+        if self.host:
+            self.reverse_host = self.host[::-1]
+
+    def get_data(self):
+        return self._get_data(self.data, self.type)
+
+    @staticmethod
+    def _get_data(data, type):
+        # handle SIEM-friendly format
+        if isinstance(data, dict) and list(data) == [type]:
+            return data[type]
+        return data
+
+    uuid: str = Field(
+        primary_key=True,
+        index=True,
+        nullable=False,
+    )
+    id: str = Field(index=True)
+    type: str = Field(index=True)
+    scope_description: str
+    data: dict = Field(sa_type=JSON)
+    host: Optional[str]
+    port: Optional[int]
+    netloc: Optional[str]
+    # store the host in reversed form for efficient lookups by domain
+    reverse_host: Optional[str] = Field(default="", exclude=True, index=True)
+    resolved_hosts: List = Field(default=[], sa_type=JSON)
+    dns_children: dict = Field(default={}, sa_type=JSON)
+    web_spider_distance: int = 10
+    scope_distance: int = Field(default=10, index=True)
+    scan: str = Field(index=True)
+    timestamp: NaiveUTC = Field(index=True)
+    parent: str = Field(index=True)
+    tags: List = Field(default=[], sa_type=JSON)
+    module: str = Field(index=True)
+    module_sequence: str
+    discovery_context: str = ""
+    discovery_path: List[str] = Field(default=[], sa_type=JSON)
+    parent_chain: List[str] = Field(default=[], sa_type=JSON)
+
+
+### SCAN ###
+
+
+class Scan(BBOTBaseModel, table=True):
+    id: str = Field(primary_key=True)
+    name: str
+    status: str
+    started_at: NaiveUTC = Field(index=True)
+    finished_at: Optional[NaiveUTC] = Field(default=None, sa_column=Column(SQLADateTime, nullable=True, index=True))
+    duration_seconds: Optional[float] = Field(default=None)
+    duration: Optional[str] = Field(default=None)
+    target: dict = Field(sa_type=JSON)
+    preset: dict = Field(sa_type=JSON)
+
+
+### TARGET ###
+
+
+class Target(BBOTBaseModel, table=True):
+    name: str = "Default Target"
+    strict_scope: bool = False
+    seeds: List = Field(default=[], sa_type=JSON)
+    whitelist: List = Field(default=None, sa_type=JSON)
+    blacklist: List = Field(default=[], sa_type=JSON)
+    hash: str = Field(sa_column=Column("hash", String, unique=True, primary_key=True, index=True))
+    scope_hash: str = Field(sa_column=Column("scope_hash", String, index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hashhash", String, index=True))
+    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String, index=True))
+    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String, index=True))

bbot/modules/output/sqlite.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class SQLite(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a SQLite database"}
+    options = {
+        "database": "",
+    }
+    options_desc = {
+        "database": "The path to the sqlite database file",
+    }
+    deps_pip = ["sqlmodel", "sqlalchemy-utils", "aiosqlite"]
+
+    async def setup(self):
+        db_file = self.config.get("database", "")
+        if not db_file:
+            db_file = self.scan.home / "output.sqlite"
+        db_file = Path(db_file)
+        if not db_file.is_absolute():
+            db_file = self.scan.home / db_file
+        self.db_file = db_file
+        self.db_file.parent.mkdir(parents=True, exist_ok=True)
+        return await super().setup()
+
+    def connection_string(self, mask_password=False):
+        return f"sqlite+aiosqlite:///{self.db_file}"

bbot/modules/templates/sql.py

@@ -0,0 +1,89 @@
+from sqlmodel import SQLModel
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+from sqlalchemy_utils.functions import database_exists, create_database
+
+from bbot.db.sql.models import Event, Scan, Target
+from bbot.modules.output.base import BaseOutputModule
+
+
+class SQLTemplate(BaseOutputModule):
+    meta = {"description": "SQL output module template"}
+    options = {
+        "protocol": "",
+        "database": "bbot",
+        "username": "",
+        "password": "",
+        "host": "127.0.0.1",
+        "port": 0,
+    }
+    options_desc = {
+        "protocol": "The protocol to use to connect to the database",
+        "database": "The database to use",
+        "username": "The username to use to connect to the database",
+        "password": "The password to use to connect to the database",
+        "host": "The host to use to connect to the database",
+        "port": "The port to use to connect to the database",
+    }
+
+    async def setup(self):
+        self.database = self.config.get("database", "bbot")
+        self.username = self.config.get("username", "")
+        self.password = self.config.get("password", "")
+        self.host = self.config.get("host", "127.0.0.1")
+        self.port = self.config.get("port", 0)
+
+        self.log.info(f"Connecting to {self.connection_string(mask_password=True)}")
+
+        self.engine = create_async_engine(self.connection_string())
+        # Create a session factory bound to the engine
+        self.async_session = sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)
+        await self.init_database()
+        return True
+
+    async def handle_event(self, event):
+        event_obj = Event(**event.json()).validated
+
+        async with self.async_session() as session:
+            async with session.begin():
+                # insert event
+                session.add(event_obj)
+
+                # if it's a SCAN event, create/update the scan and target
+                if event_obj.type == "SCAN":
+                    event_data = event_obj.get_data()
+                    if not isinstance(event_data, dict):
+                        raise ValueError(f"Invalid data for SCAN event: {event_data}")
+                    scan = Scan(**event_data).validated
+                    await session.merge(scan)  # Insert or update scan
+
+                    target_data = event_data.get("target", {})
+                    if not isinstance(target_data, dict):
+                        raise ValueError(f"Invalid target for SCAN event: {target_data}")
+                    target = Target(**target_data).validated
+                    await session.merge(target)  # Insert or update target
+
+                await session.commit()
+
+    async def init_database(self):
+        async with self.engine.begin() as conn:
+            # Check if the database exists using the connection's engine URL
+            if not await conn.run_sync(lambda sync_conn: database_exists(sync_conn.engine.url)):
+                await conn.run_sync(lambda sync_conn: create_database(sync_conn.engine.url))
+            # Create all tables
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+    def connection_string(self, mask_password=False):
+        connection_string = f"{self.protocol}://"
+        if self.username:
+            password = self.password
+            if mask_password:
+                password = "****"
+            connection_string += f"{self.username}:{password}"
+        if self.host:
+            connection_string += f"@{self.host}"
+            if self.port:
+                connection_string += f":{self.port}"
+        if self.database:
+            connection_string += f"/{self.database}"
+        return connection_string

bbot/scanner/scanner.py

@@ -161,7 +161,7 @@ class Scanner:
                 tries += 1
         else:
             scan_name = str(self.preset.scan_name)
-        self.name = scan_name
+        self.name = scan_name.replace("/", "_")
 
         # make sure the preset has a description
         if not self.preset.description:

bbot/test/test_step_2/module_tests/test_module_sqlite.py

@@ -0,0 +1,18 @@
+import sqlite3
+from .base import ModuleTestBase
+
+
+class TestSQLite(ModuleTestBase):
+    targets = ["evilcorp.com"]
+
+    def check(self, module_test, events):
+        sqlite_output_file = module_test.scan.home / "output.sqlite"
+        assert sqlite_output_file.exists(), "SQLite output file not found"
+        with sqlite3.connect(sqlite_output_file) as db:
+            cursor = db.cursor()
+            cursor.execute("SELECT * FROM event")
+            assert len(cursor.fetchall()) > 0, "No events found in SQLite database"
+            cursor.execute("SELECT * FROM scan")
+            assert len(cursor.fetchall()) > 0, "No scans found in SQLite database"
+            cursor.execute("SELECT * FROM target")
+            assert len(cursor.fetchall()) > 0, "No targets found in SQLite database"

{bbot-2.1.2.5173rc0.dist-info → bbot-2.1.2.5180rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.2.5173rc0
+Version: 2.1.2.5180rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.1.2.5173rc0.dist-info → bbot-2.1.2.5180rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=iK8WXit-jx4ZqGMOaH9TkpxZ6WlAKQrfE2Y-x0ZMPXY,130
+bbot/__init__.py,sha256=7P-sFinr0Bm0Rs1mjS4V3YlX2Bn0VY0mSdU-Wy9dDJY,130
 bbot/cli.py,sha256=7S3a4eB-Dl8yodc5WC-927Z30CNlLl9EXimGvIVypJo,10434
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -46,6 +46,7 @@ bbot/core/helpers/web/web.py,sha256=K7BOts1c1bRjU5rpluD94jClwchmBMZQk8FZI1ljS94,
 bbot/core/helpers/wordcloud.py,sha256=WdQwboCNcCxcUdLuB6MMMDQBL4ZshFM_f6GW7nUZEBQ,19819
 bbot/core/modules.py,sha256=OOUSncr-EM6bJBrI3iH5wvfnpTXKQ-A8OL8UMvkL0CU,31432
 bbot/core/shared_deps.py,sha256=IZgYbeJy20ToUNa8TnNAgzaKRK_c09W6rl-uxEhudd0,5187
+bbot/db/sql/models.py,sha256=AXefz4nEtpV2p19d6rLaEcCQNnqXODpldA-P5uSUntg,4729
 bbot/defaults.yml,sha256=_3sNH-2TWPaQHZ6ozBA1UKWLB7HuHK8vjZ534mb8cO4,6042
 bbot/errors.py,sha256=xwQcD26nU9oc7-o0kv5jmEDTInmi8_W8eKAgQZZxdVM,953
 bbot/logger.py,sha256=rLcLzNDvfR8rFj7_tZ-f5QB3Z8T0RVroact3W0ogjpA,1408
@@ -143,6 +144,7 @@ bbot/modules/output/neo4j.py,sha256=u950eUwu8YMql_WaBA38TN2bUhx7xnZdIIvYfR3xVcY,
 bbot/modules/output/python.py,sha256=RvK2KN-Zp0Vy_1zGSNioE5eeL5hIh6Z_riFtaTymyIM,270
 bbot/modules/output/slack.py,sha256=Ir_z11VYBdXDx8DwntWCv33Ic43vO1UIbxcp9gj0vvk,1181
 bbot/modules/output/splunk.py,sha256=TjTCUmDwRwKOFKBJw-Xbjku64U77OauHjtR56gyaAPs,1952
+bbot/modules/output/sqlite.py,sha256=kR5JSYbenCVrFfwMyHtOmuRyGdBK-xTEfVZMUeedYs8,908
 bbot/modules/output/stdout.py,sha256=kRVlliUcQ6aZeweZvTsl12IqVbQdnPIhTO0Bid-ILaI,3024
 bbot/modules/output/subdomains.py,sha256=3KZz4vD0itmqpo56uCyk43Z_zN1Q0Q_nyXjdnEublPA,1515
 bbot/modules/output/teams.py,sha256=I2d52LqDC7e_oboLHoYBaRK6UpN0nkJ0uRnBXrhZf9E,4628
@@ -178,6 +180,7 @@ bbot/modules/templates/bucket.py,sha256=x-c_iAeMILux6wRm0xkUUJkc2P69hYWS6DxqD7g5
 bbot/modules/templates/github.py,sha256=ENnDWpzzmZBsTisDx6Cg9V_NwJKyVyPIOpGAPktigdI,1455
 bbot/modules/templates/postman.py,sha256=oxwVusW2EdNotVX7xnnxCTnWtj3xNPbfs8aff9s4phs,614
 bbot/modules/templates/shodan.py,sha256=BfI0mNPbqkykGmjMtARhmCGKmk1uq7yTlZoPgzzJ040,1175
+bbot/modules/templates/sql.py,sha256=7HRp4gBA7NipZeUL5OACiWkHf4jYJTX1j30Yv50d4eQ,3748
 bbot/modules/templates/subdomain_enum.py,sha256=lT5MZF66OuzsyFFrj20wKlsZflzL9MOkPjDIbN3o65o,8375
 bbot/modules/templates/webhook.py,sha256=MYhKWrNYrsfM0a4PR6yVotudLyyCwgmy2eI-l9LvpBs,3706
 bbot/modules/trickest.py,sha256=HfAzjnawxXd9ypi3gumDHqImE5-C7uwNugo8d_b9HT0,1544
@@ -215,7 +218,7 @@ bbot/scanner/preset/conditions.py,sha256=hFL9cSIWGEsv2TfM5UGurf0c91cyaM8egb5IngB
 bbot/scanner/preset/environ.py,sha256=-wbFk1YHpU8IJLKVw23Q3btQTICeX0iulURo7D673L0,4732
 bbot/scanner/preset/path.py,sha256=p9tZC7XcgZv2jXpbEJAg1lU2b4ZLX5COFnCxEUOXz2g,2234
 bbot/scanner/preset/preset.py,sha256=-HH_nlr4VaXmKCooXMG5av39gOUdCVOO_y9Bhgbt_u4,40180
-bbot/scanner/scanner.py,sha256=62DKCjgV1uLxNAwpxjvE5h1uzQCxG-nzBxp1PBCSVKc,53674
+bbot/scanner/scanner.py,sha256=n0jpHQ9tXqrPJTwg8DndMgNPas0NbNzV0FeoQDbQgJE,53692
 bbot/scanner/stats.py,sha256=re93sArKXZSiD0Owgqk2J3Kdvfm3RL4Y9Qy_VOcaVk8,3623
 bbot/scanner/target.py,sha256=X25gpgRv5HmqQjGADiSe6b8744yOkRhAGAvKKYbXnSI,19886
 bbot/scripts/docs.py,sha256=kg2CzovmUVGJx9hBZjAjUdE1hXeIwC7Ry3CyrnE8GL8,10782
@@ -360,6 +363,7 @@ bbot/test/test_step_2/module_tests/test_module_smuggler.py,sha256=J23iNHxJFbZ0vd
 bbot/test/test_step_2/module_tests/test_module_social.py,sha256=McjY8g97HbZm5xnarW924pGytwwGGnzA6MfD0HmBPMU,2050
 bbot/test/test_step_2/module_tests/test_module_speculate.py,sha256=UvlSWOAtxl6jS63m4JXFRUrBHuCZZzRCsTRgdbljSeQ,3131
 bbot/test/test_step_2/module_tests/test_module_splunk.py,sha256=zCHVlgTVlFiXwhcYBeGb9Fy2RlmvVodpl4pq3j-8jJw,1859
+bbot/test/test_step_2/module_tests/test_module_sqlite.py,sha256=9Q-V4wwezgJN0vdBKhkWYePuHO5zOdw98schSFhfj1c,793
 bbot/test/test_step_2/module_tests/test_module_sslcert.py,sha256=XeiV9eQZNnA5oALCVnP7bEs3m9kMaVwEtg-hvYfgi3Y,711
 bbot/test/test_step_2/module_tests/test_module_stdout.py,sha256=JbvSUCygrG3Tq225i9dabqeUFxBHUq44TeuxH-tEWpc,3250
 bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py,sha256=sNWlk4_dDtFhMtGGr7VLnMukJkVwcEGpCYViEcqrVEw,610
@@ -396,8 +400,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ruUQwVfia1_m2u
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=VciB-ww0y-O8Ii1wpTR6rJzGDiC2r-dhVsIJApS1ZYU,3309
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.1.2.5173rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.1.2.5173rc0.dist-info/METADATA,sha256=d1BvjvRLvQF4l-rmOfTs8unWVCHLSe0iVIP4j92pzSU,17003
-bbot-2.1.2.5173rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-bbot-2.1.2.5173rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.1.2.5173rc0.dist-info/RECORD,,
+bbot-2.1.2.5180rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.1.2.5180rc0.dist-info/METADATA,sha256=NsqTqHYM8LWSlsyChMjEOjd5DjtY3F6DjeEULRur55M,17003
+bbot-2.1.2.5180rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+bbot-2.1.2.5180rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.1.2.5180rc0.dist-info/RECORD,,