bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/core/helpers/web/web.py

@@ -1,7 +1,5 @@
-import re
 import logging
 import warnings
-import traceback
 from pathlib import Path
 from bs4 import BeautifulSoup
 
@@ -21,7 +19,6 @@ log = logging.getLogger("bbot.core.helpers.web")
 
 
 class WebHelper(EngineClient):
-
     SERVER_CLASS = HTTPEngine
     ERROR_CLASS = WebError
 
@@ -60,7 +57,7 @@ class WebHelper(EngineClient):
         self.ssl_verify = self.config.get("ssl_verify", False)
         engine_debug = self.config.get("engine", {}).get("debug", False)
         super().__init__(
-            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.radix_only},
+            server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.minimal},
             debug=engine_debug,
         )
 
@@ -121,7 +118,16 @@ class WebHelper(EngineClient):
         Note:
             If the web request fails, it will return None unless `raise_error` is `True`.
         """
-        return await self.run_and_return("request", *args, **kwargs)
+        raise_error = kwargs.get("raise_error", False)
+        result = await self.run_and_return("request", *args, **kwargs)
+        if isinstance(result, dict) and "_request_error" in result:
+            if raise_error:
+                error_msg = result["_request_error"]
+                response = result["_response"]
+                error = self.ERROR_CLASS(error_msg)
+                error.response = response
+                raise error
+        return result
 
     async def request_batch(self, urls, *args, **kwargs):
         """
@@ -199,6 +205,7 @@ class WebHelper(EngineClient):
             >>> filepath = await self.helpers.download("https://www.evilcorp.com/passwords.docx", cache_hrs=24)
         """
         success = False
+        raise_error = kwargs.get("raise_error", False)
         filename = kwargs.pop("filename", self.parent_helper.cache_filename(url))
         filename = truncate_filename(Path(filename).resolve())
         kwargs["filename"] = filename
@@ -211,7 +218,16 @@ class WebHelper(EngineClient):
             log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}")
             success = True
         else:
-            success = await self.run_and_return("download", url, **kwargs)
+            result = await self.run_and_return("download", url, **kwargs)
+            if isinstance(result, dict) and "_download_error" in result:
+                if raise_error:
+                    error_msg = result["_download_error"]
+                    response = result["_response"]
+                    error = self.ERROR_CLASS(error_msg)
+                    error.response = response
+                    raise error
+            elif result:
+                success = True
 
         if success:
             return filename
@@ -245,7 +261,7 @@ class WebHelper(EngineClient):
         """
         if not path:
             raise WordlistError(f"Invalid wordlist: {path}")
-        if not "cache_hrs" in kwargs:
+        if "cache_hrs" not in kwargs:
             kwargs["cache_hrs"] = 720
         if self.parent_helper.is_url(path):
             filename = await self.download(str(path), **kwargs)
@@ -269,66 +285,6 @@ class WebHelper(EngineClient):
                     f.write(line)
             return truncated_filename
 
-    async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
-        """
-        An asynchronous generator function for iterating through paginated API data.
-
-        This function continuously makes requests to a specified API URL, incrementing the page number
-        or applying a custom pagination function, and yields the received data one page at a time.
-        It is well-suited for APIs that provide paginated results.
-
-        Args:
-            url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.
-            page_size (int, optional): The number of items per page. Defaults to 100.
-            json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
-            next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
-            **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
-
-        Yields:
-            dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.
-
-        Note:
-            The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.
-
-        Examples:
-            >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')
-            >>> try:
-            >>>     async for page in agen:
-            >>>         subdomains = page["subdomains"]
-            >>>         self.hugesuccess(subdomains)
-            >>>         if not subdomains:
-            >>>             break
-            >>> finally:
-            >>>     agen.aclose()
-        """
-        page = 1
-        offset = 0
-        result = None
-        while 1:
-            if result and callable(next_key):
-                try:
-                    new_url = next_key(result)
-                except Exception as e:
-                    log.debug(f"Failed to extract next page of results from {url}: {e}")
-                    log.debug(traceback.format_exc())
-            else:
-                new_url = url.format(page=page, page_size=page_size, offset=offset)
-            result = await self.request(new_url, **requests_kwargs)
-            if result is None:
-                log.verbose(f"api_page_iter() got no response for {url}")
-                break
-            try:
-                if json:
-                    result = result.json()
-                yield result
-            except Exception:
-                log.warning(f'Error in api_page_iter() for url: "{new_url}"')
-                log.trace(traceback.format_exc())
-                break
-            finally:
-                offset += page_size
-                page += 1
-
     async def curl(self, *args, **kwargs):
         """
         An asynchronous function that runs a cURL command with specified arguments and options.
@@ -394,7 +350,7 @@ class WebHelper(EngineClient):
                     headers[hk] = hv
 
             # add the timeout
-            if not "timeout" in kwargs:
+            if "timeout" not in kwargs:
                 timeout = http_timeout
 
             curl_command.append("-m")
@@ -495,7 +451,7 @@ class WebHelper(EngineClient):
             Perform an html parse of the 'markup' argument and return a soup instance
 
             >>> email_type = soup.find(type="email")
-            Searches the soup instance for all occurances of the passed in argument
+            Searches the soup instance for all occurrences of the passed in argument
         """
         try:
             soup = BeautifulSoup(
@@ -506,53 +462,6 @@ class WebHelper(EngineClient):
             log.debug(f"Error parsing beautifulsoup: {e}")
             return False
 
-    user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
-    pass_keywords = [re.compile(r, re.I) for r in ["pass"]]
-
-    def is_login_page(self, html):
-        """
-        TODO: convert this into an excavate YARA rule
-
-        Determines if the provided HTML content contains a login page.
-
-        This function parses the HTML to search for forms with input fields typically used for
-        authentication. If it identifies password fields or a combination of username and password
-        fields, it returns True.
-
-        Args:
-            html (str): The HTML content to analyze.
-
-        Returns:
-            bool: True if the HTML contains a login page, otherwise False.
-
-        Examples:
-            >>> is_login_page('<form><input type="text" name="username"><input type="password" name="password"></form>')
-            True
-
-            >>> is_login_page('<form><input type="text" name="search"></form>')
-            False
-        """
-        try:
-            soup = BeautifulSoup(html, "html.parser")
-        except Exception as e:
-            log.debug(f"Error parsing html: {e}")
-            return False
-
-        forms = soup.find_all("form")
-
-        # first, check for obvious password fields
-        for form in forms:
-            if form.find_all("input", {"type": "password"}):
-                return True
-
-        # next, check for forms that have both a user-like and password-like field
-        for form in forms:
-            user_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.user_keywords)
-            pass_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.pass_keywords)
-            if user_fields and pass_fields:
-                return True
-        return False
-
     def response_to_json(self, response):
         """
         Convert web response to JSON object, similar to the output of `httpx -irr -json`

bbot/core/helpers/wordcloud.py

@@ -111,7 +111,7 @@ class WordCloud(dict):
         results = set()
         for word in words:
             h = hash(word)
-            if not h in results:
+            if h not in results:
                 results.add(h)
                 yield (word,)
         if numbers > 0:
@@ -119,7 +119,7 @@ class WordCloud(dict):
                 for word in words:
                     for number_mutation in self.get_number_mutations(word, n=numbers, padding=number_padding):
                         h = hash(number_mutation)
-                        if not h in results:
+                        if h not in results:
                             results.add(h)
                             yield (number_mutation,)
         for word in words:
@@ -322,7 +322,7 @@ class WordCloud(dict):
 
     @property
     def default_filename(self):
-        return self.parent_helper.preset.scan.home / f"wordcloud.tsv"
+        return self.parent_helper.preset.scan.home / "wordcloud.tsv"
 
     def save(self, filename=None, limit=None):
         """
@@ -357,7 +357,7 @@ class WordCloud(dict):
                 log.debug(f"Saved word cloud ({len(self):,} words) to {filename}")
                 return True, filename
             else:
-                log.debug(f"No words to save")
+                log.debug("No words to save")
         except Exception as e:
             import traceback
 
@@ -421,7 +421,7 @@ class Mutator(dict):
     def mutate(self, word, max_mutations=None, mutations=None):
         if mutations is None:
             mutations = self.top_mutations(max_mutations)
-        for mutation, count in mutations.items():
+        for mutation in mutations.keys():
             ret = []
             for s in mutation:
                 if s is not None:

bbot/core/modules.py

@@ -153,7 +153,7 @@ class ModuleLoader:
                 else:
                     log.debug(f"Preloading {module_name} from disk")
                     if module_dir.name == "modules":
-                        namespace = f"bbot.modules"
+                        namespace = "bbot.modules"
                     else:
                         namespace = f"bbot.modules.{module_dir.name}"
                     try:
@@ -235,7 +235,7 @@ class ModuleLoader:
         return self.__preloaded
 
     def get_recursive_dirs(self, *dirs):
-        dirs = set(Path(d).resolve() for d in dirs)
+        dirs = {Path(d).resolve() for d in dirs}
         for d in list(dirs):
             if not d.is_dir():
                 continue
@@ -337,77 +337,77 @@ class ModuleLoader:
             # look for classes
             if type(root_element) == ast.ClassDef:
                 for class_attr in root_element.body:
-
                     # class attributes that are dictionaries
                     if type(class_attr) == ast.Assign and type(class_attr.value) == ast.Dict:
                         # module options
-                        if any([target.id == "options" for target in class_attr.targets]):
+                        if any(target.id == "options" for target in class_attr.targets):
                             config.update(ast.literal_eval(class_attr.value))
                         # module options
-                        elif any([target.id == "options_desc" for target in class_attr.targets]):
+                        elif any(target.id == "options_desc" for target in class_attr.targets):
                             options_desc.update(ast.literal_eval(class_attr.value))
                         # module metadata
-                        elif any([target.id == "meta" for target in class_attr.targets]):
+                        elif any(target.id == "meta" for target in class_attr.targets):
                             meta = ast.literal_eval(class_attr.value)
 
                     # class attributes that are lists
                     if type(class_attr) == ast.Assign and type(class_attr.value) == ast.List:
                         # flags
-                        if any([target.id == "flags" for target in class_attr.targets]):
+                        if any(target.id == "flags" for target in class_attr.targets):
                             for flag in class_attr.value.elts:
                                 if type(flag.value) == str:
                                     flags.add(flag.value)
                         # watched events
-                        elif any([target.id == "watched_events" for target in class_attr.targets]):
+                        elif any(target.id == "watched_events" for target in class_attr.targets):
                             for event_type in class_attr.value.elts:
                                 if type(event_type.value) == str:
                                     watched_events.add(event_type.value)
                         # produced events
-                        elif any([target.id == "produced_events" for target in class_attr.targets]):
+                        elif any(target.id == "produced_events" for target in class_attr.targets):
                             for event_type in class_attr.value.elts:
                                 if type(event_type.value) == str:
                                     produced_events.add(event_type.value)
 
                         # bbot module dependencies
-                        elif any([target.id == "deps_modules" for target in class_attr.targets]):
+                        elif any(target.id == "deps_modules" for target in class_attr.targets):
                             for dep_module in class_attr.value.elts:
                                 if type(dep_module.value) == str:
                                     deps_modules.add(dep_module.value)
                         # python dependencies
-                        elif any([target.id == "deps_pip" for target in class_attr.targets]):
+                        elif any(target.id == "deps_pip" for target in class_attr.targets):
                             for dep_pip in class_attr.value.elts:
                                 if type(dep_pip.value) == str:
                                     deps_pip.append(dep_pip.value)
-                        elif any([target.id == "deps_pip_constraints" for target in class_attr.targets]):
+                        elif any(target.id == "deps_pip_constraints" for target in class_attr.targets):
                             for dep_pip in class_attr.value.elts:
                                 if type(dep_pip.value) == str:
                                     deps_pip_constraints.append(dep_pip.value)
                         # apt dependencies
-                        elif any([target.id == "deps_apt" for target in class_attr.targets]):
+                        elif any(target.id == "deps_apt" for target in class_attr.targets):
                             for dep_apt in class_attr.value.elts:
                                 if type(dep_apt.value) == str:
                                     deps_apt.append(dep_apt.value)
                         # bash dependencies
-                        elif any([target.id == "deps_shell" for target in class_attr.targets]):
+                        elif any(target.id == "deps_shell" for target in class_attr.targets):
                             for dep_shell in class_attr.value.elts:
                                 deps_shell.append(ast.literal_eval(dep_shell))
                         # ansible playbook
-                        elif any([target.id == "deps_ansible" for target in class_attr.targets]):
+                        elif any(target.id == "deps_ansible" for target in class_attr.targets):
                             ansible_tasks = ast.literal_eval(class_attr.value)
                         # shared/common module dependencies
-                        elif any([target.id == "deps_common" for target in class_attr.targets]):
+                        elif any(target.id == "deps_common" for target in class_attr.targets):
                             for dep_common in class_attr.value.elts:
                                 if type(dep_common.value) == str:
                                     deps_common.append(dep_common.value)
 
         for task in ansible_tasks:
-            if not "become" in task:
+            if "become" not in task:
                 task["become"] = False
             # don't sudo brew
-            elif os_platform() == "darwin" and ("package" in task and task.get("become", False) == True):
+            elif os_platform() == "darwin" and ("package" in task and task.get("become", False) is True):
                 task["become"] = False
 
         preloaded_data = {
+            "path": str(module_file.resolve()),
             "watched_events": sorted(watched_events),
             "produced_events": sorted(produced_events),
             "flags": sorted(flags),
@@ -436,8 +436,8 @@ class ModuleLoader:
                     f'Error while preloading module "{module_file}": No shared dependency named "{dep_common}" (choices: {common_choices})'
                 )
         for ansible_task in ansible_task_list:
-            if any(x == True for x in search_dict_by_key("become", ansible_task)) or any(
-                x == True for x in search_dict_by_key("ansible_become", ansible_tasks)
+            if any(x is True for x in search_dict_by_key("become", ansible_task)) or any(
+                x is True for x in search_dict_by_key("ansible_become", ansible_tasks)
             ):
                 preloaded_data["sudo"] = True
         return preloaded_data
@@ -467,14 +467,23 @@ class ModuleLoader:
             >>> isinstance(module, object)
             True
         """
-        namespace = self._preloaded[module_name]["namespace"]
-        import_path = f"{namespace}.{module_name}"
-        module_variables = importlib.import_module(import_path, "bbot")
+        preloaded = self._preloaded[module_name]
+        namespace = preloaded["namespace"]
+        try:
+            module_path = preloaded["path"]
+        except KeyError:
+            module_path = preloaded["cache_key"][0]
+        full_namespace = f"{namespace}.{module_name}"
+
+        spec = importlib.util.spec_from_file_location(full_namespace, module_path)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[full_namespace] = module
+        spec.loader.exec_module(module)
 
         # for every top-level variable in the .py file
-        for variable in module_variables.__dict__.keys():
+        for variable in module.__dict__.keys():
             # get its value
-            value = getattr(module_variables, variable)
+            value = getattr(module, variable)
             with suppress(AttributeError):
                 # if it has watched_events and produced_events
                 if all(
@@ -531,7 +540,7 @@ class ModuleLoader:
                     with suppress(KeyError):
                         choices.remove(modname)
                     if event_type not in resolve_choices:
-                        resolve_choices[event_type] = dict()
+                        resolve_choices[event_type] = {}
                     deps = resolve_choices[event_type]
                     self.add_or_create(deps, "required_by", modname)
                     for c in choices:
@@ -630,7 +639,7 @@ class ModuleLoader:
     def modules_options_table(self, modules=None, mod_type=None):
         table = []
         header = ["Config Option", "Type", "Description", "Default"]
-        for module_name, module_options in self.modules_options(modules, mod_type).items():
+        for module_options in self.modules_options(modules, mod_type).values():
             table += module_options
         return make_table(table, header)
 

bbot/core/multiprocess.py

@@ -0,0 +1,58 @@
+import os
+import atexit
+from contextlib import suppress
+
+
+class SharedInterpreterState:
+    """
+    A class to track the primary BBOT process.
+
+    Used to prevent spawning multiple unwanted processes with multiprocessing.
+    """
+
+    def __init__(self):
+        self.main_process_var_name = "_BBOT_MAIN_PID"
+        self.scan_process_var_name = "_BBOT_SCAN_PID"
+        atexit.register(self.cleanup)
+
+    @property
+    def is_main_process(self):
+        is_main_process = self.main_pid == os.getpid()
+        return is_main_process
+
+    @property
+    def is_scan_process(self):
+        is_scan_process = os.getpid() == self.scan_pid
+        return is_scan_process
+
+    @property
+    def main_pid(self):
+        main_pid = int(os.environ.get(self.main_process_var_name, 0))
+        if main_pid == 0:
+            main_pid = os.getpid()
+            # if main PID is not set, set it to the current PID
+            os.environ[self.main_process_var_name] = str(main_pid)
+        return main_pid
+
+    @property
+    def scan_pid(self):
+        scan_pid = int(os.environ.get(self.scan_process_var_name, 0))
+        if scan_pid == 0:
+            scan_pid = os.getpid()
+            # if scan PID is not set, set it to the current PID
+            os.environ[self.scan_process_var_name] = str(scan_pid)
+        return scan_pid
+
+    def update_scan_pid(self):
+        os.environ[self.scan_process_var_name] = str(os.getpid())
+
+    def cleanup(self):
+        with suppress(Exception):
+            if self.is_main_process:
+                with suppress(KeyError):
+                    del os.environ[self.main_process_var_name]
+                with suppress(KeyError):
+                    del os.environ[self.scan_process_var_name]
+
+
+SHARED_INTERPRETER_STATE = SharedInterpreterState()

bbot/core/shared_deps.py

@@ -79,13 +79,23 @@ DEP_CHROMIUM = [
         "ignore_errors": True,
     },
     {
-        "name": "Install Chromium dependencies (Debian)",
+        "name": "Install Chromium dependencies (Ubuntu 24.04)",
         "package": {
-            "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
+            "name": "libasound2t64,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
             "state": "present",
         },
         "become": True,
-        "when": "ansible_facts['os_family'] == 'Debian'",
+        "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04'",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install Chromium dependencies (Other Debian-based)",
+        "package": {
+            "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
+            "state": "present",
+        },
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian' and not (ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04')",
         "ignore_errors": True,
     },
     {
@@ -149,6 +159,39 @@ DEP_MASSCAN = [
     },
 ]
 
+DEP_JAVA = [
+    {
+        "name": "Check if Java is installed",
+        "command": "which java",
+        "register": "java_installed",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install latest JRE (Debian)",
+        "package": {"name": ["default-jre"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Arch)",
+        "package": {"name": ["jre-openjdk"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Archlinux' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Fedora)",
+        "package": {"name": ["which", "java-latest-openjdk-headless"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'RedHat' and java_installed.rc != 0",
+    },
+    {
+        "name": "Install latest JRE (Alpine)",
+        "package": {"name": ["openjdk11"], "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Alpine' and java_installed.rc != 0",
+    },
+]
+
 # shared module dependencies -- ffuf, massdns, chromium, etc.
 SHARED_DEPS = {}
 for var, val in list(locals().items()):