bbot 2.6.1.6913rc0__py3-none-any.whl → 2.7.0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.6.1.6913rc"
+__version__ = "v2.7.0"
 
 from .scanner import Scanner, Preset
 

bbot/core/engine.py

@@ -636,7 +636,7 @@ class EngineServer(EngineBase):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ class BloomFilter:
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""

bbot/core/helpers/dns/dns.py

@@ -38,7 +38,6 @@ class DNSHelper(EngineClient):
         _wildcard_cache (dict): Cache for wildcard detection results.
         _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.
         resolver_file (Path): File containing system's current resolver nameservers.
-        filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True.
 
     Args:
         parent_helper: The parent helper object with configuration details and utilities.

bbot/core/helpers/dns/engine.py

@@ -86,8 +86,6 @@ class DNSEngine(EngineServer):
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
-        self.filter_bad_ptrs = self.dns_config.get("filter_ptrs", True)
-
     async def resolve(self, query, **kwargs):
         """Resolve DNS names and IP addresses to their corresponding results.
 

bbot/core/helpers/files.py

@@ -9,7 +9,7 @@ from .misc import rm_at_exit
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):

bbot/core/helpers/git.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def sanitize_git_repo(repo_folder: Path):
+    # sanitizing the git config is infeasible since there are too many different ways to do evil things
+    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    config_file = repo_folder / ".git" / "config"
+    if config_file.exists():
+        config_file.rename(repo_folder / "git_config_original")
+    # move the index file
+    index_file = repo_folder / ".git" / "index"
+    if index_file.exists():
+        index_file.rename(repo_folder / "git_index_original")
+    # move the hooks folder
+    hooks_folder = repo_folder / ".git" / "hooks"
+    if hooks_folder.exists():
+        hooks_folder.rename(repo_folder / "git_hooks_original")

bbot/core/helpers/misc.py

@@ -17,6 +17,7 @@ from unidecode import unidecode  # noqa F401
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
+from .git import *  # noqa F401
 from .url import *  # noqa F401
 from ... import errors
 from . import regexes as bbot_regexes

bbot/core/helpers/ntlm.py

@@ -17,11 +17,9 @@ class StrStruct(object):
         self.alloc = alloc
         self.offset = offset
         self.raw = raw[offset : offset + length]
-        self.utf16 = False
 
         if len(self.raw) >= 2 and self.raw[1] == "\0":
             self.string = self.raw.decode("utf-16")
-            self.utf16 = True
         else:
             self.string = self.raw
 

bbot/core/helpers/regex.py

@@ -65,7 +65,7 @@ class RegexHelper:
 
         while tasks:  # While there are tasks pending
             # Wait for the first task to complete
-            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+            done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
 
             for task in done:
                 result = task.result()

bbot/core/modules.py

@@ -512,60 +512,6 @@ class ModuleLoader:
                         # then we have a module
                         return value
 
-    def recommend_dependencies(self, modules):
-        """
-        Returns a dictionary containing missing dependencies and their suggested resolutions
-
-        Needs work. For this we should probably be building a dependency graph
-        """
-        resolve_choices = {}
-        # step 1: build a dictionary containing event types and their associated modules
-        # {"IP_ADDRESS": set("masscan", "ipneighbor", ...)}
-        watched = {}
-        produced = {}
-        for modname in modules:
-            preloaded = self._preloaded.get(modname)
-            if preloaded:
-                for event_type in preloaded.get("watched_events", []):
-                    self.add_or_create(watched, event_type, modname)
-                for event_type in preloaded.get("produced_events", []):
-                    self.add_or_create(produced, event_type, modname)
-        watched_all = {}
-        produced_all = {}
-        for modname, preloaded in self.preloaded().items():
-            if preloaded:
-                for event_type in preloaded.get("watched_events", []):
-                    self.add_or_create(watched_all, event_type, modname)
-                for event_type in preloaded.get("produced_events", []):
-                    self.add_or_create(produced_all, event_type, modname)
-
-        # step 2: check to see if there are missing dependencies
-        for modname in modules:
-            preloaded = self._preloaded.get(modname)
-            module_type = preloaded.get("type", "unknown")
-            if module_type != "scan":
-                continue
-            watched_events = preloaded.get("watched_events", [])
-            missing_deps = {e: not self.check_dependency(e, modname, produced) for e in watched_events}
-            if all(missing_deps.values()):
-                for event_type in watched_events:
-                    if event_type == "SCAN":
-                        continue
-                    choices = produced_all.get(event_type, [])
-                    choices = set(choices)
-                    with suppress(KeyError):
-                        choices.remove(modname)
-                    if event_type not in resolve_choices:
-                        resolve_choices[event_type] = {}
-                    deps = resolve_choices[event_type]
-                    self.add_or_create(deps, "required_by", modname)
-                    for c in choices:
-                        choice_type = self._preloaded.get(c, {}).get("type", "unknown")
-                        if choice_type == "scan":
-                            self.add_or_create(deps, "recommended", c)
-
-        return resolve_choices
-
     def check_dependency(self, event_type, modname, produced):
         if event_type not in produced:
             return False

bbot/defaults.yml

@@ -187,8 +187,10 @@ url_extension_blacklist:
   - mov
   - flv
   - webm
-# Distribute URLs with these extensions only to httpx (these are omitted from output)
-url_extension_httpx_only:
+
+# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`
+# They are also excluded from output. If you want to see them in output, remove them from this list.
+url_extension_special:
   - js
 
 # These url extensions are almost always static, so we exclude them from modules that fuzz things

bbot/modules/base.py

@@ -53,6 +53,8 @@ class BaseModule:
 
         in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.
 
+        accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False.
+
         options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default.
 
         options_desc (Dict): Descriptions for options, e.g., {"api_key": "API Key"}. Empty dict by default.
@@ -97,7 +99,7 @@ class BaseModule:
     scope_distance_modifier = 0
     target_only = False
     in_scope_only = False
-
+    accept_url_special = False
     _module_threads = 1
     _batch_size = 1
 
@@ -785,10 +787,14 @@ class BaseModule:
             if "target" not in event.tags:
                 return False, "it did not meet target_only filter criteria"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, "its extension was listed in url_extension_httpx_only"
+        # limit js URLs to modules that opt in to receive them
+        if (not self.accept_url_special) and event.type.startswith("URL"):
+            extension = getattr(event, "url_extension", "")
+            if extension in self.scan.url_extension_special:
+                return (
+                    False,
+                    f"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs",
+                )
 
         return True, "precheck succeeded"
 

bbot/modules/dnstlsrpt.py

@@ -44,20 +44,17 @@ class dnstlsrpt(BaseModule):
         "emit_emails": True,
         "emit_raw_dns_records": False,
         "emit_urls": True,
-        "emit_vulnerabilities": True,
     }
     options_desc = {
         "emit_emails": "Emit EMAIL_ADDRESS events",
         "emit_raw_dns_records": "Emit RAW_DNS_RECORD events",
         "emit_urls": "Emit URL_UNVERIFIED events",
-        "emit_vulnerabilities": "Emit VULNERABILITY events",
     }
 
     async def setup(self):
         self.emit_emails = self.config.get("emit_emails", True)
         self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False)
         self.emit_urls = self.config.get("emit_urls", True)
-        self.emit_vulnerabilities = self.config.get("emit_vulnerabilities", True)
         return await super().setup()
 
     def _incoming_dedup_hash(self, event):
@@ -139,6 +136,3 @@ class dnstlsrpt(BaseModule):
                                                     tags=tags.append(f"tlsrpt-record-{key}"),
                                                     parent=event,
                                                 )
-
-
-# EOF

bbot/modules/git_clone.py

@@ -24,44 +24,69 @@ class git_clone(github):
 
     async def setup(self):
         output_folder = self.config.get("output_folder")
-        if output_folder:
-            self.output_dir = Path(output_folder) / "git_repos"
-        else:
-            self.output_dir = self.scan.temp_dir / "git_repos"
+        self.output_dir = Path(output_folder) / "git_repos" if output_folder else self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 
     async def filter_event(self, event):
-        if event.type == "CODE_REPOSITORY":
-            if "git" not in event.tags:
-                return False, "event is not a git repository"
+        if event.type == "CODE_REPOSITORY" and "git" not in event.tags:
+            return False, "event is not a git repository"
         return True
 
     async def handle_event(self, event):
-        repo_url = event.data.get("url")
-        repo_path = await self.clone_git_repository(repo_url)
-        if repo_path:
-            self.verbose(f"Cloned {repo_url} to {repo_path}")
-            codebase_event = self.make_event({"path": str(repo_path)}, "FILESYSTEM", tags=["git"], parent=event)
+        repository_url = event.data.get("url")
+        repository_path = await self.clone_git_repository(repository_url)
+        if repository_path:
+            self.verbose(f"Cloned {repository_url} to {repository_path}")
+            codebase_event = self.make_event({"path": str(repository_path)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
                 codebase_event,
-                context=f"{{module}} downloaded git repo at {repo_url} to {{event.type}}: {repo_path}",
+                context=f"{{module}} cloned git repository at {repository_url} to {{event.type}}: {repository_path}",
             )
 
     async def clone_git_repository(self, repository_url):
         owner = repository_url.split("/")[-2]
         folder = self.output_dir / owner
         self.helpers.mkdir(folder)
-        if self.api_key:
-            url = repository_url.replace("https://github.com", f"https://user:{self.api_key}@github.com")
-        else:
-            url = repository_url
-        command = ["git", "-C", folder, "clone", url]
+
+        command = ["git", "-C", folder, "clone", repository_url]
+        env = {"GIT_TERMINAL_PROMPT": "0"}
+
         try:
-            output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, check=True)
+            hostname = self.helpers.urlparse(repository_url).hostname
+            if hostname and self.api_key:
+                _, domain = self.helpers.split_domain(hostname)
+                # only use the api key if the domain is github.com
+                if domain == "github.com":
+                    env["GIT_HELPER"] = (
+                        f'!f() {{ case "$1" in get) '
+                        f"echo username=x-access-token; "
+                        f"echo password={self.api_key};; "
+                        f'esac; }}; f "$@"'
+                    )
+                    command = (
+                        command[:1]
+                        + [
+                            "-c",
+                            "credential.helper=",
+                            "-c",
+                            "credential.useHttpPath=true",
+                            "--config-env=credential.helper=GIT_HELPER",
+                        ]
+                        + command[1:]
+                    )
+
+            output = await self.run_process(command, env=env, check=True)
         except CalledProcessError as e:
-            self.debug(f"Error cloning {url}. STDERR: {repr(e.stderr)}")
+            self.debug(f"Error cloning {repository_url}. STDERR: {repr(e.stderr)}")
             return
 
         folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
-        return folder / folder_name
+        repo_folder = folder / folder_name
+
+        # sanitize the repo
+        # this moves the git config, index file, and hooks folder out of the .git folder to prevent nasty things
+        # Note: the index file can be regenerated by running "git checkout HEAD -- ."
+        self.helpers.sanitize_git_repo(repo_folder)
+
+        return repo_folder

bbot/modules/gitdumper.py

@@ -1,5 +1,4 @@
 import asyncio
-import regex as re
 from pathlib import Path
 from subprocess import CalledProcessError
 from bbot.modules.base import BaseModule
@@ -35,7 +34,6 @@ class gitdumper(BaseModule):
         else:
             self.output_dir = self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
-        self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
         self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
         self.obj_regex = self.helpers.re.compile(r"[a-f0-9]{40}")
         self.pack_regex = self.helpers.re.compile(r"pack-([a-f0-9]{40})\.pack")
@@ -131,7 +129,6 @@ class gitdumper(BaseModule):
         else:
             result = await self.git_fuzz(repo_url, repo_folder)
         if result:
-            await self.sanitize_config(repo_folder)
             await self.git_checkout(repo_folder)
             codebase_event = self.make_event({"path": str(repo_folder)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
@@ -251,15 +248,6 @@ class gitdumper(BaseModule):
             self.debug(f"Unable to download git files to {folder}")
             return False
 
-    async def sanitize_config(self, folder):
-        config_file = folder / ".git/config"
-        if config_file.exists():
-            with config_file.open("r", encoding="utf-8", errors="ignore") as file:
-                content = file.read()
-                sanitized = await self.helpers.re.sub(self.unsafe_regex, r"# \g<0>", content)
-            with config_file.open("w", encoding="utf-8") as file:
-                file.write(sanitized)
-
     async def git_catfile(self, hash, option="-t", folder=Path()):
         command = ["git", "cat-file", option, hash]
         try:
@@ -270,8 +258,10 @@ class gitdumper(BaseModule):
         return output.stdout
 
     async def git_checkout(self, folder):
+        self.helpers.sanitize_git_repo(folder)
         self.verbose(f"Running git checkout to reconstruct the git repository at {folder}")
-        command = ["git", "checkout", "."]
+        # we do "checkout head -- ." because the sanitization deletes the index file, and it needs to be reconstructed
+        command = ["git", "checkout", "HEAD", "--", "."]
         try:
             await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
         except CalledProcessError as e:

bbot/modules/graphql_introspection.py

@@ -27,7 +27,6 @@ class graphql_introspection(BaseModule):
             self.output_dir = Path(output_folder) / "graphql-schemas"
         else:
             self.output_dir = self.scan.home / "graphql-schemas"
-        self.helpers.mkdir(self.output_dir)
         return True
 
     async def filter_event(self, event):
@@ -120,7 +119,10 @@ fragment TypeRef on __Type {
             }
             response = await self.helpers.request(**request_args)
             if not response or response.status_code != 200:
-                self.debug(f"Failed to get GraphQL schema for {url} (status code {response.status_code})")
+                self.debug(
+                    f"Failed to get GraphQL schema for {url} "
+                    f"{f'(status code {response.status_code})' if response else ''}"
+                )
                 continue
             try:
                 response_json = response.json()
@@ -128,6 +130,7 @@ fragment TypeRef on __Type {
                 self.debug(f"Failed to parse JSON for {url}")
                 continue
             if response_json.get("data", {}).get("__schema", {}).get("types", []):
+                self.helpers.mkdir(self.output_dir)
                 filename = f"schema-{self.helpers.tagify(url)}.json"
                 filename = self.output_dir / filename
                 with open(filename, "w") as f:

bbot/modules/httpx.py

@@ -50,6 +50,8 @@ class httpx(BaseModule):
     _shuffle_incoming_queue = False
     _batch_size = 500
     _priority = 2
+    # accept Javascript URLs
+    accept_url_special = True
 
     async def setup(self):
         self.threads = self.config.get("threads", 50)

bbot/modules/iis_shortnames.py

@@ -116,13 +116,6 @@ class iis_shortnames(BaseModule):
 
         return duplicates
 
-    async def threaded_request(self, method, url, affirmative_status_code, c):
-        r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10)
-        if r is not None:
-            if r.status_code == affirmative_status_code:
-                return True, c
-        return None, c
-
     async def solve_valid_chars(self, method, target, affirmative_status_code):
         confirmed_chars = []
         confirmed_exts = []

bbot/modules/internal/unarchive.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from contextlib import suppress
 from bbot.modules.internal.base import BaseInternalModule
 from bbot.core.helpers.libmagic import get_magic_info, get_compression
 
@@ -62,15 +63,20 @@ class unarchive(BaseInternalModule):
                 context=f'extracted "{path}" to: {output_dir}',
             )
         else:
-            output_dir.rmdir()
+            with suppress(OSError):
+                output_dir.rmdir()
 
     async def extract_file(self, path, output_dir):
         extension, mime_type, description, confidence = get_magic_info(path)
         compression_format = get_compression(mime_type)
         cmd_list = self.compression_methods.get(compression_format, [])
         if cmd_list:
-            if not output_dir.exists():
-                self.helpers.mkdir(output_dir)
+            # output dir must not already exist
+            try:
+                output_dir.mkdir(exist_ok=False)
+            except FileExistsError:
+                self.warning(f"Destination directory {output_dir} already exists, aborting unarchive for {path}")
+                return False
             command = [s.format(filename=path, extract_dir=output_dir) for s in cmd_list]
             try:
                 await self.run_process(command, check=True)

bbot/modules/lightfuzz/lightfuzz.py

@@ -34,6 +34,7 @@ class lightfuzz(BaseModule):
         self.event_dict = {}
         self.interactsh_subdomain_tags = {}
         self.interactsh_instance = None
+        self.interactsh_domain = None
         self.disable_post = self.config.get("disable_post", False)
         self.enabled_submodules = self.config.get("enabled_submodules")
         self.interactsh_disable = self.scan.config.get("interactsh_disable", False)
@@ -51,13 +52,16 @@ class lightfuzz(BaseModule):
             self.submodules[submodule_name] = submodule_class
 
         interactsh_needed = any(submodule.uses_interactsh for submodule in self.submodules.values())
-
         if interactsh_needed and not self.interactsh_disable:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
+                if not self.interactsh_domain:
+                    self.warning("Interactsh failure: No domain returned from self.interactsh_instance.register()")
+                    self.interactsh_instance = None
             except InteractshError as e:
                 self.warning(f"Interactsh failure: {e}")
+                self.interactsh_instance = None
         return True
 
     async def interactsh_callback(self, r):

bbot/modules/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.4.7",
+        "version": "3.4.10",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/modules/output/base.py

@@ -38,11 +38,6 @@ class BaseOutputModule(BaseModule):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, (f"Omitting {event} from output because it's marked as httpx-only")
-
         # omit certain event types
         if event._omit:
             if "target" in event.tags: