bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/db/sql/models.py

@@ -0,0 +1,147 @@
+# This file contains SQLModel (Pydantic + SQLAlchemy) models for BBOT events, scans, and targets.
+# Used by the SQL output modules, but portable for outside use.
+
+import json
+import logging
+from pydantic import ConfigDict
+from typing import List, Optional
+from datetime import datetime, timezone
+from typing_extensions import Annotated
+from pydantic.functional_validators import AfterValidator
+from sqlmodel import inspect, Column, Field, SQLModel, JSON, String, DateTime as SQLADateTime
+
+
+log = logging.getLogger("bbot_server.models")
+
+
+def naive_datetime_validator(d: datetime):
+    """
+    Converts all dates into UTC, then drops timezone information.
+
+    This is needed to prevent inconsistencies in sqlite, because it is timezone-naive.
+    """
+    # drop timezone info
+    return d.replace(tzinfo=None)
+
+
+NaiveUTC = Annotated[datetime, AfterValidator(naive_datetime_validator)]
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        # handle datetime
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+class BBOTBaseModel(SQLModel):
+    model_config = ConfigDict(extra="ignore")
+
+    def __init__(self, *args, **kwargs):
+        self._validated = None
+        super().__init__(*args, **kwargs)
+
+    @property
+    def validated(self):
+        try:
+            if self._validated is None:
+                self._validated = self.__class__.model_validate(self)
+            return self._validated
+        except AttributeError:
+            return self
+
+    def to_json(self, **kwargs):
+        return json.dumps(self.validated.model_dump(), sort_keys=True, cls=CustomJSONEncoder, **kwargs)
+
+    @classmethod
+    def _pk_column_names(cls):
+        return [column.name for column in inspect(cls).primary_key]
+
+    def __hash__(self):
+        return hash(self.to_json())
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+
+### EVENT ###
+
+
+class Event(BBOTBaseModel, table=True):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        data = self._get_data(self.data, self.type)
+        self.data = {self.type: data}
+        if self.host:
+            self.reverse_host = self.host[::-1]
+
+    def get_data(self):
+        return self._get_data(self.data, self.type)
+
+    @staticmethod
+    def _get_data(data, type):
+        # handle SIEM-friendly format
+        if isinstance(data, dict) and list(data) == [type]:
+            return data[type]
+        return data
+
+    uuid: str = Field(
+        primary_key=True,
+        index=True,
+        nullable=False,
+    )
+    id: str = Field(index=True)
+    type: str = Field(index=True)
+    scope_description: str
+    data: dict = Field(sa_type=JSON)
+    host: Optional[str]
+    port: Optional[int]
+    netloc: Optional[str]
+    # store the host in reversed form for efficient lookups by domain
+    reverse_host: Optional[str] = Field(default="", exclude=True, index=True)
+    resolved_hosts: List = Field(default=[], sa_type=JSON)
+    dns_children: dict = Field(default={}, sa_type=JSON)
+    web_spider_distance: int = 10
+    scope_distance: int = Field(default=10, index=True)
+    scan: str = Field(index=True)
+    timestamp: NaiveUTC = Field(index=True)
+    parent: str = Field(index=True)
+    tags: List = Field(default=[], sa_type=JSON)
+    module: str = Field(index=True)
+    module_sequence: str
+    discovery_context: str = ""
+    discovery_path: List[str] = Field(default=[], sa_type=JSON)
+    parent_chain: List[str] = Field(default=[], sa_type=JSON)
+    inserted_at: NaiveUTC = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+### SCAN ###
+
+
+class Scan(BBOTBaseModel, table=True):
+    id: str = Field(primary_key=True)
+    name: str
+    status: str
+    started_at: NaiveUTC = Field(index=True)
+    finished_at: Optional[NaiveUTC] = Field(default=None, sa_column=Column(SQLADateTime, nullable=True, index=True))
+    duration_seconds: Optional[float] = Field(default=None)
+    duration: Optional[str] = Field(default=None)
+    target: dict = Field(sa_type=JSON)
+    preset: dict = Field(sa_type=JSON)
+
+
+### TARGET ###
+
+
+class Target(BBOTBaseModel, table=True):
+    name: str = "Default Target"
+    strict_scope: bool = False
+    seeds: List = Field(default=[], sa_type=JSON)
+    whitelist: List = Field(default=None, sa_type=JSON)
+    blacklist: List = Field(default=[], sa_type=JSON)
+    hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
+    scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True))
+    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True))
+    blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True))

bbot/defaults.yml

@@ -14,6 +14,9 @@ folder_blobs: false
 ### SCOPE ###
 
 scope:
+  # strict scope means only exact DNS names are considered in-scope
+  # subdomains are not included unless they are explicitly provided in the target list
+  strict: false
   # Filter by scope distance which events are displayed in the output
   # 0 == show only in-scope events (affiliates are always shown)
   # 1 == show all events up to distance-1 (1 hop from target)
@@ -30,10 +33,13 @@ dns:
   # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records
   minimal: false
   # How many instances of the dns module to run concurrently
-  threads: 20
+  threads: 25
   # How many concurrent DNS resolvers to use when brute-forcing
   # (under the hood this is passed through directly to massdns -s)
   brute_threads: 1000
+  # nameservers to use for DNS brute-forcing
+  # default is updated weekly and contains ~10K high-quality public servers
+  brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt
   # How far away from the main target to explore via DNS resolution (independent of scope.search_distance)
   # This is safe to change
   search_distance: 1
@@ -106,11 +112,18 @@ engine:
 deps:
   ffuf:
     version: "2.1.0"
+  # How to handle installation of module dependencies
+  # Choices are:
+  #  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
+  #  - retry_failed - try again to install failed dependencies
+  #  - ignore_failed - run the scan regardless of what happens with dependency installation
+  #  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
+  behavior: abort_on_failure
 
 ### ADVANCED OPTIONS ###
 
 # Load BBOT modules from these custom paths
-module_paths: []
+module_dirs: []
 
 # Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc.
 speculate: True
@@ -123,14 +136,6 @@ dnsresolve: True
 # Cloud provider tagging
 cloudcheck: True
 
-# How to handle installation of module dependencies
-# Choices are:
-#  - abort_on_failure (default) - if a module dependency fails to install, abort the scan
-#  - retry_failed - try again to install failed dependencies
-#  - ignore_failed - run the scan regardless of what happens with dependency installation
-#  - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)
-deps_behavior: abort_on_failure
-
 # Strip querystring from URLs by default
 url_querystring_remove: True
 # When query string is retained, by default collapse parameter values down to a single value per parameter

bbot/errors.py

@@ -38,14 +38,6 @@ class WordlistError(BBOTError):
     pass
 
 
-class DNSError(BBOTError):
-    pass
-
-
-class DNSWildcardBreak(DNSError):
-    pass
-
-
 class CurlError(BBOTError):
     pass
 

bbot/modules/anubisdb.py

@@ -20,7 +20,7 @@ class anubisdb(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url)
+        return await self.api_request(url)
 
     def abort_if_pre(self, hostname):
         """
@@ -38,7 +38,7 @@ class anubisdb(subdomain_enum):
             return True, "DNS name is unresolved"
         return await super().abort_if(event)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json:

bbot/modules/apkpure.py

@@ -0,0 +1,63 @@
+import re
+from pathlib import Path
+from bbot.modules.base import BaseModule
+
+
+class apkpure(BaseModule):
+    watched_events = ["MOBILE_APP"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "safe", "code-enum"]
+    meta = {
+        "description": "Download android applications from apkpure.com",
+        "created_date": "2024-10-11",
+        "author": "@domwhewell-sage",
+    }
+    options = {"output_folder": ""}
+    options_desc = {"output_folder": "Folder to download apk's to"}
+
+    async def setup(self):
+        output_folder = self.config.get("output_folder")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "apk_files"
+        else:
+            self.output_dir = self.scan.home / "apk_files"
+        self.helpers.mkdir(self.output_dir)
+        return await super().setup()
+
+    async def filter_event(self, event):
+        if event.type == "MOBILE_APP":
+            if "android" not in event.tags:
+                return False, "event is not an android app"
+        return True
+
+    async def handle_event(self, event):
+        app_id = event.data.get("id", "")
+        path = await self.download_apk(app_id)
+        if path:
+            await self.emit_event(
+                {"path": str(path)},
+                "FILESYSTEM",
+                tags=["apk", "file"],
+                parent=event,
+                context=f'{{module}} downloaded the apk "{app_id}" to: {path}',
+            )
+
+    async def download_apk(self, app_id):
+        path = None
+        url = f"https://d.apkpure.com/b/XAPK/{app_id}?version=latest"
+        self.helpers.mkdir(self.output_dir / app_id)
+        response = await self.helpers.request(url, allow_redirects=True)
+        if response:
+            attachment = response.headers.get("Content-Disposition", "")
+            if "filename" in attachment:
+                match = re.search(r'filename="?([^"]+)"?', attachment)
+                if match:
+                    filename = match.group(1)
+                    extension = filename.split(".")[-1]
+                    content = response.content
+                    file_destination = self.output_dir / app_id / f"{app_id}.{extension}"
+                    with open(file_destination, "wb") as f:
+                        f.write(content)
+                    self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}')
+                    path = file_destination
+        return path

bbot/modules/azure_tenant.py

@@ -102,7 +102,7 @@ class azure_tenant(BaseModule):
         status_code = getattr(r, "status_code", 0)
         if status_code not in (200, 421):
             self.verbose(f'Error retrieving azure_tenant domains for "{domain}" (status code: {status_code})')
-            return set(), dict()
+            return set(), {}
         found_domains = list(set(await self.helpers.re.findall(self.d_xml_regex, r.text)))
         domains = set()
 
@@ -116,7 +116,7 @@ class azure_tenant(BaseModule):
             self.scan.word_cloud.absorb_word(d)
 
         r = await openid_task
-        openid_config = dict()
+        openid_config = {}
         with suppress(Exception):
             openid_config = r.json()
 

bbot/modules/baddns.py

@@ -15,26 +15,26 @@ class baddns(BaseModule):
         "created_date": "2024-01-18",
         "author": "@liquidsec",
     }
-    options = {"custom_nameservers": [], "only_high_confidence": False, "enable_references": False}
+    options = {"custom_nameservers": [], "only_high_confidence": False, "enabled_submodules": []}
     options_desc = {
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
         "only_high_confidence": "Do not emit low-confidence or generic detections",
-        "enable_references": "Enable the references module (off by default)",
+        "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.1.815"]
+    deps_pip = ["baddns~=1.4.13"]
 
     def select_modules(self):
-
-        module_list = ["CNAME", "NS", "MX", "TXT"]
-        if self.config.get("enable_references", False):
-            module_list.append("references")
-
-        selected_modules = []
+        selected_submodules = []
         for m in get_all_modules():
-            if m.name in module_list:
-                selected_modules.append(m)
-        return selected_modules
+            if m.name in self.enabled_submodules:
+                selected_submodules.append(m)
+        return selected_submodules
+
+    def set_modules(self):
+        self.enabled_submodules = self.config.get("enabled_submodules", [])
+        if self.enabled_submodules == []:
+            self.enabled_submodules = ["CNAME", "MX", "TXT"]
 
     async def setup(self):
         self.preset.core.logger.include_logger(logging.getLogger("baddns"))
@@ -43,10 +43,18 @@ class baddns(BaseModule):
             self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers)
         self.only_high_confidence = self.config.get("only_high_confidence", False)
         self.signatures = load_signatures()
+        self.set_modules()
+        all_submodules_list = [m.name for m in get_all_modules()]
+        for m in self.enabled_submodules:
+            if m not in all_submodules_list:
+                self.hugewarning(
+                    f"Selected BadDNS submodule [{m}] does not exist. Available submodules: [{','.join(all_submodules_list)}]"
+                )
+                return False
+        self.debug(f"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]")
         return True
 
     async def handle_event(self, event):
-
         tasks = []
         for ModuleClass in self.select_modules():
             kwargs = {
@@ -62,11 +70,18 @@ class baddns(BaseModule):
                 kwargs["raw_query_retry_wait"] = 0
 
             module_instance = ModuleClass(event.data, **kwargs)
-
-            tasks.append((module_instance, asyncio.create_task(module_instance.dispatch())))
-
-        for module_instance, task in tasks:
-            if await task:
+            task = asyncio.create_task(module_instance.dispatch())
+            tasks.append((module_instance, task))
+
+        async for completed_task in self.helpers.as_completed([task for _, task in tasks]):
+            module_instance = next((m for m, t in tasks if t == completed_task), None)
+            try:
+                task_result = await completed_task
+            except Exception as e:
+                self.warning(f"Task for {module_instance} raised an error: {e}")
+                task_result = None
+
+            if task_result:
                 results = module_instance.analyze()
                 if results and len(results) > 0:
                     for r in results:
@@ -99,7 +114,7 @@ class baddns(BaseModule):
                                 context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
                         else:
-                            self.warning(f"Got unrecognized confidence level: {r['confidence']}")
+                            self.warning(f"Got unrecognized confidence level: {r_dict['confidence']}")
 
                         found_domains = r_dict.get("found_domains", None)
                         if found_domains:
@@ -111,3 +126,4 @@ class baddns(BaseModule):
                                     tags=[f"baddns-{module_instance.name.lower()}"],
                                     context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {{event.data}}',
                                 )
+                await module_instance.cleanup()

bbot/modules/baddns_direct.py

@@ -0,0 +1,92 @@
+from baddns.base import get_all_modules
+from baddns.lib.loader import load_signatures
+from .base import BaseModule
+
+import logging
+
+
+class baddns_direct(BaseModule):
+    watched_events = ["URL", "STORAGE_BUCKET"]
+    produced_events = ["FINDING", "VULNERABILITY"]
+    flags = ["active", "safe", "subdomain-enum", "baddns", "cloud-enum"]
+    meta = {
+        "description": "Check for unusual subdomain / service takeover edge cases that require direct detection",
+        "created_date": "2024-01-29",
+        "author": "@liquidsec",
+    }
+    options = {"custom_nameservers": []}
+    options_desc = {
+        "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
+    }
+    module_threads = 8
+    deps_pip = ["baddns~=1.4.13"]
+
+    scope_distance_modifier = 1
+
+    async def setup(self):
+        self.preset.core.logger.include_logger(logging.getLogger("baddns"))
+        self.custom_nameservers = self.config.get("custom_nameservers", []) or None
+        if self.custom_nameservers:
+            self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers)
+        self.only_high_confidence = self.config.get("only_high_confidence", False)
+        self.signatures = load_signatures()
+        return True
+
+    def select_modules(self):
+        selected_modules = []
+        for m in get_all_modules():
+            if m.name in ["CNAME"]:
+                selected_modules.append(m)
+        return selected_modules
+
+    async def handle_event(self, event):
+        CNAME_direct_module = self.select_modules()[0]
+        kwargs = {
+            "http_client_class": self.scan.helpers.web.AsyncClient,
+            "dns_client": self.scan.helpers.dns.resolver,
+            "custom_nameservers": self.custom_nameservers,
+            "signatures": self.signatures,
+            "direct_mode": True,
+        }
+
+        CNAME_direct_instance = CNAME_direct_module(event.host, **kwargs)
+        if await CNAME_direct_instance.dispatch():
+            results = CNAME_direct_instance.analyze()
+            if results and len(results) > 0:
+                for r in results:
+                    r_dict = r.to_dict()
+
+                    data = {
+                        "description": f"Possible [{r_dict['signature']}] via direct BadDNS analysis. Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]",
+                        "host": str(event.host),
+                    }
+
+                    await self.emit_event(
+                        data,
+                        "FINDING",
+                        event,
+                        tags=[f"baddns-{CNAME_direct_module.name.lower()}"],
+                        context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
+                    )
+        await CNAME_direct_instance.cleanup()
+
+    async def filter_event(self, event):
+        if event.type == "STORAGE_BUCKET":
+            if str(event.module).startswith("bucket_"):
+                return False
+            self.debug(f"Processing STORAGE_BUCKET for {event.host}")
+        if event.type == "URL":
+            if event.scope_distance > 0:
+                self.debug(
+                    f"Rejecting {event.host} due to not being in scope (scope distance: {str(event.scope_distance)})"
+                )
+                return False
+            if "cdn-cloudflare" not in event.tags:
+                self.debug(f"Rejecting {event.host} due to not being behind CloudFlare")
+                return False
+            if "status-200" in event.tags or "status-301" in event.tags:
+                self.debug(f"Rejecting {event.host} due to lack of non-standard status code")
+                return False
+
+            self.debug(f"Passed all checks and is processing {event.host}")
+        return True

bbot/modules/baddns_zone.py

@@ -1,4 +1,3 @@
-from baddns.base import get_all_modules
 from .baddns import baddns as baddns_module
 
 
@@ -17,14 +16,10 @@ class baddns_zone(baddns_module):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.1.815"]
+    deps_pip = ["baddns~=1.4.13"]
 
-    def select_modules(self):
-        selected_modules = []
-        for m in get_all_modules():
-            if m.name in ["NSEC", "zonetransfer"]:
-                selected_modules.append(m)
-        return selected_modules
+    def set_modules(self):
+        self.enabled_submodules = ["NSEC", "zonetransfer"]
 
     # minimize nsec records feeding back into themselves
     async def filter_event(self, event):

bbot/modules/badsecrets.py

@@ -17,18 +17,19 @@ class badsecrets(BaseModule):
     options_desc = {
         "custom_secrets": "Include custom secrets loaded from a local file",
     }
-    deps_pip = ["badsecrets~=0.4.490"]
+    deps_pip = ["badsecrets~=0.6.21"]
 
     async def setup(self):
         self.custom_secrets = None
         custom_secrets = self.config.get("custom_secrets", None)
         if custom_secrets:
-            if Path(custom_secrets).is_file():
+            secrets_path = Path(custom_secrets).expanduser()
+            if secrets_path.is_file():
                 self.custom_secrets = custom_secrets
                 self.info(f"Successfully loaded secrets file [{custom_secrets}]")
             else:
                 self.warning(f"custom secrets file [{custom_secrets}] is not valid")
-                return None, "Custom secrets file not valid"
+                return False, "Custom secrets file not valid"
         return True
 
     @property