bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/jadx.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+from subprocess import CalledProcessError
+from bbot.modules.internal.base import BaseModule
+
+
+class jadx(BaseModule):
+    watched_events = ["FILESYSTEM"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "safe"]
+    meta = {
+        "description": "Decompile APKs and XAPKs using JADX",
+        "created_date": "2024-11-04",
+        "author": "@domwhewell-sage",
+    }
+    options = {
+        "threads": 4,
+    }
+    options_desc = {
+        "threads": "Maximum jadx threads for extracting apk's, default: 4",
+    }
+    deps_common = ["java"]
+    deps_ansible = [
+        {
+            "name": "Create jadx directory",
+            "file": {"path": "#{BBOT_TOOLS}/jadx", "state": "directory", "mode": "0755"},
+        },
+        {
+            "name": "Download jadx",
+            "unarchive": {
+                "src": "https://github.com/skylot/jadx/releases/download/v1.5.0/jadx-1.5.0.zip",
+                "include": ["lib/jadx-1.5.0-all.jar", "bin/jadx"],
+                "dest": "#{BBOT_TOOLS}/jadx",
+                "remote_src": True,
+            },
+        },
+    ]
+
+    allowed_file_types = ["java archive", "android application package"]
+
+    async def setup(self):
+        self.threads = self.config.get("threads", 4)
+        return True
+
+    async def filter_event(self, event):
+        if "file" in event.tags:
+            if event.data["magic_description"].lower() not in self.allowed_file_types:
+                return False, f"Jadx is not able to decompile this file type: {event.data['magic_description']}"
+        else:
+            return False, "Event is not a file"
+        return True
+
+    async def handle_event(self, event):
+        path = Path(event.data["path"])
+        output_dir = path.parent / path.name.replace(".", "_")
+        self.helpers.mkdir(output_dir)
+        success = await self.decompile_apk(path, output_dir)
+
+        # If jadx was able to decompile the java archive, emit an event
+        if success:
+            await self.emit_event(
+                {"path": str(output_dir)},
+                "FILESYSTEM",
+                tags="folder",
+                parent=event,
+                context=f'extracted "{path}" to: {output_dir}',
+            )
+        else:
+            output_dir.rmdir()
+
+    async def decompile_apk(self, path, output_dir):
+        command = [
+            f"{self.scan.helpers.tools_dir}/jadx/bin/jadx",
+            "--threads-count",
+            self.threads,
+            "--output-dir",
+            str(output_dir),
+            str(path),
+        ]
+        try:
+            output = await self.run_process(command, check=True)
+        except CalledProcessError as e:
+            self.warning(f"Error decompiling {path}. STDOUT: {e.stdout} STDERR: {repr(e.stderr)}")
+            return False
+        if not (output_dir / "resources").exists() and not (output_dir / "sources").exists():
+            self.warning(f"JADX was unable to decompile {path}: (STDOUT: {output.stdout} STDERR: {output.stderr})")
+            return False
+        return True

bbot/modules/leakix.py

@@ -15,31 +15,32 @@ class leakix(subdomain_enum_apikey):
     }
 
     base_url = "https://leakix.net"
+    ping_url = f"{base_url}/host/1.1.1.1"
 
     async def setup(self):
         ret = await super(subdomain_enum_apikey, self).setup()
-        self.headers = {"Accept": "application/json"}
         self.api_key = self.config.get("api_key", "")
         if self.api_key:
-            self.headers["api-key"] = self.api_key
             return await self.require_api_key()
         return ret
 
-    async def ping(self):
-        url = f"{self.base_url}/host/1.2.3.4.5"
-        r = await self.helpers.request(url, headers=self.headers)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) != 401, resp_content
+    def prepare_api_request(self, url, kwargs):
+        if self.api_key:
+            kwargs["headers"]["api-key"] = self.api_key
+            kwargs["headers"]["Accept"] = "application/json"
+        return url, kwargs
 
     async def request_url(self, query):
         url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}"
-        response = await self.request_with_fail_count(url, headers=self.headers)
+        response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
+        results = set()
         json = r.json()
         if json:
             for entry in json:
                 subdomain = entry.get("subdomain", "")
                 if subdomain:
-                    yield subdomain
+                    results.add(subdomain)
+        return results

bbot/modules/myssl.py

@@ -15,9 +15,9 @@ class myssl(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}?domain={self.helpers.quote(query)}"
-        return await self.request_with_fail_count(url)
+        return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, dict):

bbot/modules/newsletters.py

@@ -46,11 +46,11 @@ class newsletters(BaseModule):
                 body = _event.data["body"]
                 soup = self.helpers.beautifulsoup(body, "html.parser")
                 if soup is False:
-                    self.debug(f"BeautifulSoup returned False")
+                    self.debug("BeautifulSoup returned False")
                     return
                 result = self.find_type(soup)
                 if result:
-                    description = f"Found a Newsletter Submission Form that could be used for email bombing attacks"
+                    description = "Found a Newsletter Submission Form that could be used for email bombing attacks"
                     data = {"host": str(_event.host), "description": description, "url": _event.data["url"]}
                     await self.emit_event(
                         data,

bbot/modules/otx.py

@@ -15,12 +15,14 @@ class otx(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
-        return self.request_with_fail_count(url)
+        return self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         j = r.json()
         if isinstance(j, dict):
             for entry in j.get("passive_dns", []):
                 subdomain = entry.get("hostname", "")
                 if subdomain:
-                    yield subdomain
+                    results.add(subdomain)
+        return results

bbot/modules/output/asset_inventory.py

@@ -91,15 +91,15 @@ class asset_inventory(CSV):
             self.assets[hostkey].absorb_event(event)
 
     async def report(self):
-        stats = dict()
-        totals = dict()
+        stats = {}
+        totals = {}
 
         def increment_stat(stat, value):
             try:
                 totals[stat] += 1
             except KeyError:
                 totals[stat] = 1
-            if not stat in stats:
+            if stat not in stats:
                 stats[stat] = {}
             try:
                 stats[stat][value] += 1
@@ -259,17 +259,17 @@ class Asset:
         # ips
         self.ip_addresses = set(_make_ip_list(row.get("IP (External)", "")))
         self.ip_addresses.update(set(_make_ip_list(row.get("IP (Internal)", ""))))
-        # If user reqests a recheck dont import the following fields to force them to be rechecked
+        # If user requests a recheck dont import the following fields to force them to be rechecked
         if not self.recheck:
             # ports
             ports = [i.strip() for i in row.get("Open Ports", "").split(",")]
-            self.ports.update(set(i for i in ports if i and is_port(i)))
+            self.ports.update({i for i in ports if i and is_port(i)})
             # findings
             findings = [i.strip() for i in row.get("Findings", "").splitlines()]
-            self.findings.update(set(i for i in findings if i))
+            self.findings.update({i for i in findings if i})
             # technologies
             technologies = [i.strip() for i in row.get("Technologies", "").splitlines()]
-            self.technologies.update(set(i for i in technologies if i))
+            self.technologies.update({i for i in technologies if i})
             # risk rating
             risk_rating = row.get("Risk Rating", "").strip()
             if risk_rating and risk_rating.isdigit() and int(risk_rating) > self.risk_rating:

bbot/modules/output/base.py

@@ -24,7 +24,7 @@ class BaseOutputModule(BaseModule):
         if event.type in ("FINISHED",):
             return True, "its type is FINISHED"
         if self.errored:
-            return False, f"module is in error state"
+            return False, "module is in error state"
         # exclude non-watched types
         if not any(t in self.get_watched_events() for t in ("*", event.type)):
             return False, "its type is not in watched_events"

bbot/modules/output/csv.py

@@ -55,7 +55,6 @@ class CSV(BaseOutputModule):
     async def handle_event(self, event):
         # ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"]
         discovery_path = getattr(event, "discovery_path", [])
-        discovery_path = [e[-1] for e in discovery_path]
         self.writerow(
             {
                 "Event type": getattr(event, "type", ""),
@@ -65,7 +64,7 @@ class CSV(BaseOutputModule):
                 ),
                 "Source Module": str(getattr(event, "module_sequence", "")),
                 "Scope Distance": str(getattr(event, "scope_distance", "")),
-                "Event Tags": ",".join(sorted(list(getattr(event, "tags", [])))),
+                "Event Tags": ",".join(sorted(getattr(event, "tags", []))),
                 "Discovery Path": " --> ".join(discovery_path),
             }
         )

bbot/modules/output/http.py

@@ -1,4 +1,3 @@
-from bbot.errors import WebError
 from bbot.modules.output.base import BaseOutputModule
 
 
@@ -52,16 +51,23 @@ class HTTP(BaseOutputModule):
 
     async def handle_event(self, event):
         while 1:
-            try:
-                await self.helpers.request(
-                    url=self.url,
-                    method=self.method,
-                    auth=self.auth,
-                    headers=self.headers,
-                    json=event.json(siem_friendly=self.siem_friendly),
-                    raise_error=True,
-                )
-                break
-            except WebError as e:
-                self.warning(f"Error sending {event}: {e}, retrying...")
-                await self.helpers.sleep(1)
+            response = await self.helpers.request(
+                url=self.url,
+                method=self.method,
+                auth=self.auth,
+                headers=self.headers,
+                json=event.json(siem_friendly=self.siem_friendly),
+            )
+            is_success = False if response is None else response.is_success
+            if not is_success:
+                status_code = getattr(response, "status_code", 0)
+                self.warning(f"Error sending {event} (HTTP status code: {status_code}), retrying...")
+                body = getattr(response, "text", "")
+                self.debug(body)
+                if status_code == 429:
+                    sleep_interval = 10
+                else:
+                    sleep_interval = 1
+                await self.helpers.sleep(sleep_interval)
+                continue
+            break

bbot/modules/output/mysql.py

@@ -0,0 +1,51 @@
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class MySQL(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a MySQL database"}
+    options = {
+        "username": "root",
+        "password": "bbotislife",
+        "host": "localhost",
+        "port": 3306,
+        "database": "bbot",
+    }
+    options_desc = {
+        "username": "The username to connect to MySQL",
+        "password": "The password to connect to MySQL",
+        "host": "The server running MySQL",
+        "port": "The port to connect to MySQL",
+        "database": "The database name to connect to",
+    }
+    deps_pip = ["sqlmodel", "aiomysql"]
+    protocol = "mysql+aiomysql"
+
+    async def create_database(self):
+        from sqlalchemy import text
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        # Create the engine for the initial connection to the server
+        initial_engine = create_async_engine(self.connection_string().rsplit("/", 1)[0])
+
+        async with initial_engine.connect() as conn:
+            # Check if the database exists
+            result = await conn.execute(text(f"SHOW DATABASES LIKE '{self.database}'"))
+            database_exists = result.scalar() is not None
+
+            # Create the database if it does not exist
+            if not database_exists:
+                # Use aiomysql directly to create the database
+                import aiomysql
+
+                raw_conn = await aiomysql.connect(
+                    user=self.username,
+                    password=self.password,
+                    host=self.host,
+                    port=self.port,
+                )
+                try:
+                    async with raw_conn.cursor() as cursor:
+                        await cursor.execute(f"CREATE DATABASE {self.database}")
+                finally:
+                    await raw_conn.ensure_closed()

bbot/modules/output/neo4j.py

@@ -1,10 +1,15 @@
 import json
+import logging
 from contextlib import suppress
 from neo4j import AsyncGraphDatabase
 
 from bbot.modules.output.base import BaseOutputModule
 
 
+# silence annoying neo4j logger
+logging.getLogger("neo4j").setLevel(logging.CRITICAL)
+
+
 class neo4j(BaseOutputModule):
     """
     # start Neo4j in the background with docker
@@ -48,7 +53,7 @@ class neo4j(BaseOutputModule):
                 ),
             )
             self.session = self.driver.session()
-            await self.handle_event(self.scan.root_event)
+            await self.session.run("Match () Return 1 Limit 1")
         except Exception as e:
             return False, f"Error setting up Neo4j: {e}"
         return True
@@ -110,7 +115,7 @@ class neo4j(BaseOutputModule):
 
         cypher = f"""UNWIND $events AS event
         MERGE (_:{event_type} {{ id: event.id }})
-        SET _ += event
+        SET _ += properties(event)
         RETURN event.data as event_data, event.id as event_id, elementId(_) as neo4j_id"""
         neo4j_ids = {}
         # insert events

bbot/modules/output/postgres.py

@@ -0,0 +1,49 @@
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class Postgres(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a SQLite database"}
+    options = {
+        "username": "postgres",
+        "password": "bbotislife",
+        "host": "localhost",
+        "port": 5432,
+        "database": "bbot",
+    }
+    options_desc = {
+        "username": "The username to connect to Postgres",
+        "password": "The password to connect to Postgres",
+        "host": "The server running Postgres",
+        "port": "The port to connect to Postgres",
+        "database": "The database name to connect to",
+    }
+    deps_pip = ["sqlmodel", "asyncpg"]
+    protocol = "postgresql+asyncpg"
+
+    async def create_database(self):
+        import asyncpg
+        from sqlalchemy import text
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        # Create the engine for the initial connection to the server
+        initial_engine = create_async_engine(self.connection_string().rsplit("/", 1)[0])
+
+        async with initial_engine.connect() as conn:
+            # Check if the database exists
+            result = await conn.execute(text(f"SELECT 1 FROM pg_database WHERE datname = '{self.database}'"))
+            database_exists = result.scalar() is not None
+
+            # Create the database if it does not exist
+            if not database_exists:
+                # Use asyncpg directly to create the database
+                raw_conn = await asyncpg.connect(
+                    user=self.username,
+                    password=self.password,
+                    host=self.host,
+                    port=self.port,
+                )
+                try:
+                    await raw_conn.execute(f"CREATE DATABASE {self.database}")
+                finally:
+                    await raw_conn.close()

bbot/modules/output/slack.py

@@ -16,7 +16,6 @@ class Slack(WebhookOutputModule):
         "event_types": "Types of events to send",
         "min_severity": "Only allow VULNERABILITY events of this severity or higher",
     }
-    good_status_code = 200
     content_key = "text"
 
     def format_message_str(self, event):

bbot/modules/output/sqlite.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from bbot.modules.templates.sql import SQLTemplate
+
+
+class SQLite(SQLTemplate):
+    watched_events = ["*"]
+    meta = {"description": "Output scan data to a SQLite database"}
+    options = {
+        "database": "",
+    }
+    options_desc = {
+        "database": "The path to the sqlite database file",
+    }
+    deps_pip = ["sqlmodel", "aiosqlite"]
+
+    async def setup(self):
+        db_file = self.config.get("database", "")
+        if not db_file:
+            db_file = self.scan.home / "output.sqlite"
+        db_file = Path(db_file)
+        if not db_file.is_absolute():
+            db_file = self.scan.home / db_file
+        self.db_file = db_file
+        self.db_file.parent.mkdir(parents=True, exist_ok=True)
+        return await super().setup()
+
+    def connection_string(self, mask_password=False):
+        return f"sqlite+aiosqlite:///{self.db_file}"

bbot/modules/output/stdout.py

@@ -20,7 +20,7 @@ class Stdout(BaseOutputModule):
 
     async def setup(self):
         self.text_format = self.config.get("format", "text").strip().lower()
-        if not self.text_format in self.format_choices:
+        if self.text_format not in self.format_choices:
             return (
                 False,
                 f'Invalid text format choice, "{self.text_format}" (choices: {",".join(self.format_choices)})',
@@ -33,7 +33,7 @@ class Stdout(BaseOutputModule):
 
     async def filter_event(self, event):
         if self.accept_event_types:
-            if not event.type in self.accept_event_types:
+            if event.type not in self.accept_event_types:
                 return False, f'Event type "{event.type}" is not in the allowed event_types'
         return True
 

bbot/modules/output/teams.py

@@ -10,14 +10,115 @@ class Teams(WebhookOutputModule):
     }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
-        "webhook_url": "Discord webhook URL",
+        "webhook_url": "Teams webhook URL",
         "event_types": "Types of events to send",
         "min_severity": "Only allow VULNERABILITY events of this severity or higher",
     }
     _module_threads = 5
-    good_status_code = 200
-    content_key = "text"
 
-    def evaluate_response(self, response):
-        text = getattr(response, "text", "")
-        return text == "1"
+    async def handle_event(self, event):
+        while 1:
+            data = self.format_message(event)
+
+            response = await self.helpers.request(
+                url=self.webhook_url,
+                method="POST",
+                json=data,
+            )
+            status_code = getattr(response, "status_code", 0)
+            if self.evaluate_response(response):
+                break
+            else:
+                response_data = getattr(response, "text", "")
+                try:
+                    retry_after = response.json().get("retry_after", 1)
+                except Exception:
+                    retry_after = 1
+                self.verbose(
+                    f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds"
+                )
+                await self.helpers.sleep(retry_after)
+
+    def trim_message(self, message):
+        if len(message) > self.message_size_limit:
+            message = message[: self.message_size_limit - 3] + "..."
+        return message
+
+    def format_message_str(self, event):
+        items = []
+        msg = self.trim_message(event.data)
+        items.append({"type": "TextBlock", "text": f"{msg}", "wrap": True})
+        items.append({"type": "FactSet", "facts": [{"title": "Tags:", "value": ", ".join(event.tags)}]})
+        return items
+
+    def format_message_other(self, event):
+        items = [{"type": "FactSet", "facts": []}]
+        for key, value in event.data.items():
+            if key != "severity":
+                msg = self.trim_message(str(value))
+                items[0]["facts"].append({"title": f"{key}:", "value": msg})
+        return items
+
+    def get_severity_color(self, event):
+        color = "Accent"
+        if event.type == "VULNERABILITY":
+            severity = event.data.get("severity", "UNKNOWN")
+            if severity == "CRITICAL":
+                color = "Attention"
+            elif severity == "HIGH":
+                color = "Attention"
+            elif severity == "MEDIUM":
+                color = "Warning"
+            elif severity == "LOW":
+                color = "Good"
+        return color
+
+    def format_message(self, event):
+        adaptive_card = {
+            "type": "message",
+            "attachments": [
+                {
+                    "contentType": "application/vnd.microsoft.card.adaptive",
+                    "contentUrl": None,
+                    "content": {
+                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                        "type": "AdaptiveCard",
+                        "version": "1.2",
+                        "msteams": {"width": "full"},
+                        "body": [],
+                    },
+                }
+            ],
+        }
+        heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "size": "Large", "style": "heading"}
+        body = adaptive_card["attachments"][0]["content"]["body"]
+        body.append(heading)
+        if event.type in ("VULNERABILITY", "FINDING"):
+            subheading = {
+                "type": "TextBlock",
+                "text": event.data.get("severity", "UNKNOWN"),
+                "spacing": "None",
+                "size": "Large",
+                "wrap": True,
+            }
+            subheading["color"] = self.get_severity_color(event)
+            body.append(subheading)
+        main_text = {
+            "type": "ColumnSet",
+            "separator": True,
+            "spacing": "Medium",
+            "columns": [
+                {
+                    "type": "Column",
+                    "width": "stretch",
+                    "items": [],
+                }
+            ],
+        }
+        if isinstance(event.data, str):
+            items = self.format_message_str(event)
+        else:
+            items = self.format_message_other(event)
+        main_text["columns"][0]["items"] = items
+        body.append(main_text)
+        return adaptive_card

bbot/modules/paramminer_headers.py

@@ -15,7 +15,7 @@ class paramminer_headers(BaseModule):
     meta = {
         "description": "Use smart brute-force to check for common HTTP header parameters",
         "created_date": "2022-04-15",
-        "author": "@pmueller",
+        "author": "@liquidsec",
     }
     options = {
         "wordlist": "",  # default is defined within setup function
@@ -82,7 +82,6 @@ class paramminer_headers(BaseModule):
     header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$")
 
     async def setup(self):
-
         self.recycle_words = self.config.get("recycle_words", True)
         self.event_dict = {}
         self.already_checked = set()
@@ -90,11 +89,11 @@ class paramminer_headers(BaseModule):
         if not wordlist:
             wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}"
         self.debug(f"Using wordlist: [{wordlist}]")
-        self.wl = set(
+        self.wl = {
             h.strip().lower()
             for h in self.helpers.read_file(await self.helpers.wordlist(wordlist))
             if len(h) > 0 and "%" not in h
-        )
+        }
 
         # check against the boring list (if the option is set)
         if self.config.get("skip_boring_words", True):
@@ -157,7 +156,6 @@ class paramminer_headers(BaseModule):
             )
 
     async def handle_event(self, event):
-
         # If recycle words is enabled, we will collect WEB_PARAMETERS we find to build our list in finish()
         # We also collect any parameters of type "SPECULATIVE"
         if event.type == "WEB_PARAMETER":
@@ -201,7 +199,7 @@ class paramminer_headers(BaseModule):
             return
         for count, args, kwargs in self.gen_count_args(url):
             r = await self.helpers.request(*args, **kwargs)
-            if r is not None and not ((str(r.status_code)[0] in ("4", "5"))):
+            if r is not None and not (str(r.status_code)[0] in ("4", "5")):
                 return count
 
     def gen_count_args(self, url):
@@ -240,8 +238,7 @@ class paramminer_headers(BaseModule):
         return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1))
 
     async def finish(self):
-
-        untested_matches = sorted(list(self.extracted_words_master.copy()))
+        untested_matches = sorted(self.extracted_words_master.copy())
         for url, (event, batch_size) in list(self.event_dict.items()):
             try:
                 compare_helper = self.helpers.http_compare(url)