bbot 2.1.0.4959rc0__tar.gz → 2.1.2.5156rc0__tar.gz

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.0.4959rc0
+Version: 2.1.2.5156rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
@@ -14,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Security
 Requires-Dist: ansible (>=7.3,<9.0)
 Requires-Dist: ansible-runner (>=2.3.2,<3.0.0)
@@ -28,7 +29,7 @@ Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: lxml (>=4.9.2,<6.0.0)
 Requires-Dist: mmh3 (>=4.1,<6.0)
 Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
-Requires-Dist: psutil (>=5.9.4,<6.0.0)
+Requires-Dist: psutil (>=5.9.4,<7.0.0)
 Requires-Dist: pycryptodome (>=3.17,<4.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pyjwt (>=2.7.0,<3.0.0)
@@ -107,13 +108,13 @@ config:
     threads: 25
     brute_threads: 1000
   # put your API keys here
-  modules:
-    github:
-      api_key: ""
-    chaos:
-      api_key: ""
-    securitytrails:
-      api_key: ""
+  # modules:
+  #   github:
+  #     api_key: ""
+  #   chaos:
+  #     api_key: ""
+  #   securitytrails:
+  #     api_key: ""
 
 ```
 
@@ -267,6 +268,7 @@ include:
   - paramminer
   - dirbust-light
   - web-screenshots
+  - baddns-thorough
 
 config:
   modules:

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/README.md

@@ -55,13 +55,13 @@ config:
     threads: 25
     brute_threads: 1000
   # put your API keys here
-  modules:
-    github:
-      api_key: ""
-    chaos:
-      api_key: ""
-    securitytrails:
-      api_key: ""
+  # modules:
+  #   github:
+  #     api_key: ""
+  #   chaos:
+  #     api_key: ""
+  #   securitytrails:
+  #     api_key: ""
 
 ```
 
@@ -215,6 +215,7 @@ include:
   - paramminer
   - dirbust-light
   - web-screenshots
+  - baddns-thorough
 
 config:
   modules:

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.0.4959rc"
+__version__ = "v2.1.2.5156rc"
 
 from .scanner import Scanner, Preset

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/config/logger.py

@@ -5,6 +5,7 @@ from copy import copy
 import multiprocessing
 import logging.handlers
 from pathlib import Path
+from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
@@ -71,10 +72,36 @@ class BBOTLogger:
             # Start the QueueListener
             self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())
             self.listener.start()
-            atexit.register(self.listener.stop)
+            atexit.register(self.cleanup_logging)
 
         self.log_level = logging.INFO
 
+    def cleanup_logging(self):
+        # Close the queue handler
+        with suppress(Exception):
+            self.queue_handler.close()
+
+        # Clean root logger
+        root_logger = logging.getLogger()
+        for handler in list(root_logger.handlers):
+            with suppress(Exception):
+                root_logger.removeHandler(handler)
+            with suppress(Exception):
+                handler.close()
+
+        # Clean all other loggers
+        for logger in logging.Logger.manager.loggerDict.values():
+            if hasattr(logger, "handlers"):  # Logger, not PlaceHolder
+                for handler in list(logger.handlers):
+                    with suppress(Exception):
+                        logger.removeHandler(handler)
+                    with suppress(Exception):
+                        handler.close()
+
+        # Stop queue listener
+        with suppress(Exception):
+            self.listener.stop()
+
     def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
         if logging_queue is None:
             logging_queue = self.queue

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/event/base.py

@@ -25,6 +25,7 @@ from bbot.core.helpers import (
     is_domain,
     is_subdomain,
     is_ip,
+    is_ip_type,
     is_ptr,
     is_uri,
     url_depth,
@@ -157,7 +158,7 @@ class BaseEvent:
         Raises:
             ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
-        self.uuid = uuid.uuid4()
+        self._uuid = uuid.uuid4()
         self._id = None
         self._hash = None
         self._data = None
@@ -352,6 +353,12 @@ class BaseEvent:
                 return 80
         return self._port
 
+    @property
+    def netloc(self):
+        if self.host and is_ip_type(self.host, network=False):
+            return make_netloc(self.host, self.port)
+        return None
+
     @property
     def host_stem(self):
         """
@@ -440,11 +447,6 @@ class BaseEvent:
         no_host_information = not bool(self.host)
         return self._always_emit or always_emit_tags or no_host_information
 
-    @property
-    def quick_emit(self):
-        no_host_information = not bool(self.host)
-        return self._quick_emit or no_host_information
-
     @property
     def id(self):
         """
@@ -454,6 +456,13 @@ class BaseEvent:
             self._id = f"{self.type}:{self.data_hash.hex()}"
         return self._id
 
+    @property
+    def uuid(self):
+        """
+        A universally unique identifier for the event
+        """
+        return f"{self.type}:{self._uuid}"
+
     @property
     def data_hash(self):
         """
@@ -746,7 +755,7 @@ class BaseEvent:
         """
         j = dict()
         # type, ID, scope description
-        for i in ("type", "id", "uuid", "scope_description"):
+        for i in ("type", "id", "uuid", "scope_description", "netloc"):
             v = getattr(self, i, "")
             if v:
                 j.update({i: str(v)})
@@ -765,6 +774,8 @@ class BaseEvent:
             j["host"] = str(self.host)
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
             j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        if isinstance(self.port, int):
+            j["port"] = self.port
         # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
@@ -1537,6 +1548,13 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     _always_emit_tags = ["target"]
 
 
+class MOBILE_APP(DictEvent):
+    _always_emit = True
+
+    def _pretty_string(self):
+        return self.data["url"]
+
+
 def make_event(
     data,
     event_type=None,
@@ -1707,7 +1725,7 @@ def event_from_json(j, siem_friendly=False):
         event = make_event(**kwargs)
         event_uuid = j.get("uuid", None)
         if event_uuid is not None:
-            event.uuid = uuid.UUID(event_uuid)
+            event._uuid = uuid.UUID(event_uuid.split(":")[-1])
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
@@ -1719,7 +1737,8 @@ def event_from_json(j, siem_friendly=False):
             event._parent_id = parent_id
         parent_uuid = j.get("parent_uuid", None)
         if parent_uuid is not None:
-            event._parent_uuid = uuid.UUID(parent_uuid)
+            parent_type, parent_uuid = parent_uuid.split(":", 1)
+            event._parent_uuid = parent_type + ":" + str(uuid.UUID(parent_uuid))
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -248,7 +248,7 @@ class DepsInstaller:
         return success
 
     def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
-        _ansible_args = {"ansible_connection": "local"}
+        _ansible_args = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable}
         if ansible_args is not None:
             _ansible_args.update(ansible_args)
         module_args = None
@@ -342,7 +342,15 @@ class DepsInstaller:
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
         # command: package_name
-        core_deps = {"unzip": "unzip", "curl": "curl"}
+        core_deps = {
+            "unzip": "unzip",
+            "zipinfo": "unzip",
+            "curl": "curl",
+            "git": "git",
+            "make": "make",
+            "gcc": "gcc",
+            "bash": "bash",
+        }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):
                 to_install.add(package_name)

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/helpers/misc.py

@@ -621,12 +621,13 @@ def is_ip(d, version=None):
     return False
 
 
-def is_ip_type(i):
+def is_ip_type(i, network=None):
     """
     Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module.
 
     Args:
         i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check.
+        network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False.
 
     Returns:
         bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise.
@@ -639,6 +640,12 @@ def is_ip_type(i):
         >>> is_ip_type("192.168.1.0/24")
         False
     """
+    if network is not None:
+        is_network = ipaddress._BaseNetwork in i.__class__.__mro__
+        if network:
+            return is_network
+        else:
+            return not is_network
     return ipaddress._IPAddressBase in i.__class__.__mro__
 
 
@@ -1260,7 +1267,7 @@ def gen_numbers(n, padding=2):
     return results
 
 
-def make_netloc(host, port):
+def make_netloc(host, port=None):
     """Constructs a network location string from a given host and port.
 
     Args:
@@ -1289,7 +1296,7 @@ def make_netloc(host, port):
     if is_ip(host, version=6):
         host = f"[{host}]"
     if port is None:
-        return host
+        return str(host)
     return f"{host}:{port}"
 
 

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/helpers/regexes.py

@@ -54,6 +54,9 @@ ptr_regex = re.compile(_ptr_regex)
 # uuid regex
 _uuid_regex = r"[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
 uuid_regex = re.compile(_uuid_regex, re.I)
+# event uuid regex
+_event_uuid_regex = r"[0-9A-Z_]+:[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
+event_uuid_regex = re.compile(_event_uuid_regex, re.I)
 
 _open_port_regexes = (
     _dns_name_regex + r":[0-9]{1,5}",

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/core/helpers/web/web.py

@@ -1,4 +1,3 @@
-import re
 import logging
 import warnings
 from pathlib import Path
@@ -464,53 +463,6 @@ class WebHelper(EngineClient):
             log.debug(f"Error parsing beautifulsoup: {e}")
             return False
 
-    user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
-    pass_keywords = [re.compile(r, re.I) for r in ["pass"]]
-
-    def is_login_page(self, html):
-        """
-        TODO: convert this into an excavate YARA rule
-
-        Determines if the provided HTML content contains a login page.
-
-        This function parses the HTML to search for forms with input fields typically used for
-        authentication. If it identifies password fields or a combination of username and password
-        fields, it returns True.
-
-        Args:
-            html (str): The HTML content to analyze.
-
-        Returns:
-            bool: True if the HTML contains a login page, otherwise False.
-
-        Examples:
-            >>> is_login_page('<form><input type="text" name="username"><input type="password" name="password"></form>')
-            True
-
-            >>> is_login_page('<form><input type="text" name="search"></form>')
-            False
-        """
-        try:
-            soup = BeautifulSoup(html, "html.parser")
-        except Exception as e:
-            log.debug(f"Error parsing html: {e}")
-            return False
-
-        forms = soup.find_all("form")
-
-        # first, check for obvious password fields
-        for form in forms:
-            if form.find_all("input", {"type": "password"}):
-                return True
-
-        # next, check for forms that have both a user-like and password-like field
-        for form in forms:
-            user_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.user_keywords)
-            pass_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.pass_keywords)
-            if user_fields and pass_fields:
-                return True
-        return False
-
     def response_to_json(self, response):
         """
         Convert web response to JSON object, similar to the output of `httpx -irr -json`

bbot-2.1.2.5156rc0/bbot/modules/apkpure.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from bbot.modules.base import BaseModule
+
+
+class apkpure(BaseModule):
+    watched_events = ["MOBILE_APP"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "safe", "code-enum"]
+    meta = {
+        "description": "Download android applications from apkpure.com",
+        "created_date": "2024-10-11",
+        "author": "@domwhewell-sage",
+    }
+    options = {"output_folder": ""}
+    options_desc = {"output_folder": "Folder to download apk's to"}
+
+    async def setup(self):
+        output_folder = self.config.get("output_folder")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "apk_files"
+        else:
+            self.output_dir = self.scan.home / "apk_files"
+        self.helpers.mkdir(self.output_dir)
+        return await super().setup()
+
+    async def filter_event(self, event):
+        if event.type == "MOBILE_APP":
+            if "android" not in event.tags:
+                return False, "event is not an android app"
+        return True
+
+    async def handle_event(self, event):
+        app_id = event.data.get("id", "")
+        path = await self.download_apk(app_id)
+        if path:
+            await self.emit_event(
+                {"path": str(path)},
+                "FILESYSTEM",
+                tags=["apk", "file"],
+                parent=event,
+                context=f'{{module}} downloaded the apk "{app_id}" to: {path}',
+            )
+
+    async def download_apk(self, app_id):
+        path = None
+        url = f"https://d.apkpure.com/b/XAPK/{app_id}?version=latest"
+        self.helpers.mkdir(self.output_dir / app_id)
+        file_destination = self.output_dir / app_id / f"{app_id}.xapk"
+        result = await self.helpers.download(url, warn=False, filename=file_destination)
+        if result:
+            self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}')
+            path = file_destination
+        return path

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/modules/badsecrets.py

@@ -23,12 +23,13 @@ class badsecrets(BaseModule):
         self.custom_secrets = None
         custom_secrets = self.config.get("custom_secrets", None)
         if custom_secrets:
-            if Path(custom_secrets).is_file():
+            secrets_path = Path(custom_secrets).expanduser()
+            if secrets_path.is_file():
                 self.custom_secrets = custom_secrets
                 self.info(f"Successfully loaded secrets file [{custom_secrets}]")
             else:
                 self.warning(f"custom secrets file [{custom_secrets}] is not valid")
-                return None, "Custom secrets file not valid"
+                return False, "Custom secrets file not valid"
         return True
 
     @property

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/modules/base.py

@@ -984,8 +984,11 @@ class BaseModule:
     def _outgoing_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
+
+        We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules)
+        to inadvertently suppress a non-internal event.
         """
-        return hash((event, self.name))
+        return hash((event, self.name, event.internal, event.always_emit))
 
     def get_per_host_hash(self, event):
         """
@@ -1559,7 +1562,7 @@ class BaseModule:
             self.trace()
 
 
-class InterceptModule(BaseModule):
+class BaseInterceptModule(BaseModule):
     """
     An Intercept Module is a special type of high-priority module that gets early access to events.
 
@@ -1571,7 +1574,6 @@ class InterceptModule(BaseModule):
     """
 
     accept_dupes = True
-    suppress_dupes = False
     _intercept = True
 
     async def _worker(self):

bbot-2.1.2.5156rc0/bbot/modules/bufferoverrun.py

@@ -0,0 +1,48 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class BufferOverrun(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query BufferOverrun's TLS API for subdomains",
+        "created_date": "2024-10-23",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "commercial": False}
+    options_desc = {"api_key": "BufferOverrun API key", "commercial": "Use commercial API"}
+
+    base_url = "https://tls.bufferover.run/dns"
+    commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns"
+
+    async def setup(self):
+        self.commercial = self.config.get("commercial", False)
+        return await super().setup()
+
+    def prepare_api_request(self, url, kwargs):
+        if self.commercial:
+            kwargs["headers"]["x-rapidapi-host"] = "bufferover-run-tls.p.rapidapi.com"
+            kwargs["headers"]["x-rapidapi-key"] = self.api_key
+        else:
+            kwargs["headers"]["x-api-key"] = self.api_key
+        return url, kwargs
+
+    async def request_url(self, query):
+        url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
+        return await self.api_request(url)
+
+    def parse_results(self, r, query):
+        j = r.json()
+        subdomains_set = set()
+        if isinstance(j, dict):
+            results = j.get("Results", [])
+            for result in results:
+                parts = result.split(",")
+                if len(parts) > 4:
+                    subdomain = parts[4].strip()
+                    if subdomain and subdomain.endswith(f".{query}"):
+                        subdomains_set.add(subdomain)
+            for subdomain in subdomains_set:
+                yield subdomain

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.4",
+        "version": "3.3.5",
         "tags": "",
         "templates": "",
         "severity": "",

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/modules/dotnetnuke.py

@@ -155,24 +155,25 @@ class dotnetnuke(BaseModule):
 
                 # InstallWizard SuperUser Privilege Escalation
                 result = await self.helpers.request(f'{event.data["url"]}/Install/InstallWizard.aspx')
-                if result.status_code == 200:
-                    result_confirm = await self.helpers.request(
-                        f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
-                    )
-                    if result_confirm.status_code == 500:
-                        description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
-                        await self.emit_event(
-                            {
-                                "severity": "CRITICAL",
-                                "description": description,
-                                "host": str(event.host),
-                                "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
-                            },
-                            "VULNERABILITY",
-                            event,
-                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                if result:
+                    if result.status_code == 200:
+                        result_confirm = await self.helpers.request(
+                            f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
                         )
-                        return
+                        if result_confirm.status_code == 500:
+                            description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
+                            await self.emit_event(
+                                {
+                                    "severity": "CRITICAL",
+                                    "description": description,
+                                    "host": str(event.host),
+                                    "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
+                                },
+                                "VULNERABILITY",
+                                event,
+                                context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                            )
+                            return
 
                 # DNNImageHandler.ashx Blind SSRF
                 self.event_dict[event.data["url"]] = event

{bbot-2.1.0.4959rc0 → bbot-2.1.2.5156rc0}/bbot/modules/generic_ssrf.py

@@ -137,10 +137,10 @@ class Generic_XXE(BaseSubmodule):
         post_body = f"""<?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE foo [
 <!ELEMENT foo ANY >
-<!ENTITY % {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
+<!ENTITY {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
 ]>
 <foo>&{rand_entity};</foo>"""
-        test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
+        test_url = event.parsed_url.geturl()
         r = await self.parent_module.helpers.curl(
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )