bbot 2.3.0.5546rc0__py3-none-any.whl → 2.3.1.5815rc0__py3-none-any.whl

bbot/modules/base.py

@@ -160,8 +160,7 @@ class BaseModule:
         self._api_request_failures = 0
 
         self._tasks = []
-        self._event_received = asyncio.Condition()
-        self._event_queued = asyncio.Condition()
+        self._event_received = None
 
         # used for optional "per host" tracking
         self._per_host_tracker = set()
@@ -409,6 +408,12 @@ class BaseModule:
         """
         return getattr(self, "api_key", "")
 
+    @property
+    def event_received(self):
+        if self._event_received is None:
+            self._event_received = asyncio.Condition()
+        return self._event_received
+
     def get_watched_events(self):
         """Retrieve the set of events that the module is interested in observing.
 
@@ -658,11 +663,12 @@ class BaseModule:
                         await asyncio.sleep(0.1)
                         continue
 
+                    # if batch wasn't big enough, we wait for the next event before continuing
                     if self.batch_size > 1:
                         submitted = await self._handle_batch()
                         if not submitted:
-                            async with self._event_received:
-                                await self._event_received.wait()
+                            async with self.event_received:
+                                await self.event_received.wait()
 
                     else:
                         try:
@@ -874,8 +880,8 @@ class BaseModule:
                 self.debug(f"Queueing {event} because {reason}")
             try:
                 self.incoming_event_queue.put_nowait(event)
-                async with self._event_received:
-                    self._event_received.notify()
+                async with self.event_received:
+                    self.event_received.notify()
                 if event.type != "FINISHED":
                     self.scan._new_activity = True
             except AttributeError:
@@ -1148,7 +1154,7 @@ class BaseModule:
             kwargs["url"] = new_url
 
             r = await self.helpers.request(**kwargs)
-            success = False if r is None else r.is_success
+            success = r is not None and self._api_response_is_success(r)
 
             if success:
                 self._api_request_failures = 0
@@ -1163,11 +1169,13 @@ class BaseModule:
                     )
                 else:
                     # sleep for a bit if we're being rate limited
-                    if status_code == 429:
+                    retry_after = self._get_retry_after(r)
+                    if retry_after or status_code == 429:
+                        sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
                         self.verbose(
-                            f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)"
+                            f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})"
                         )
-                        await asyncio.sleep(self._429_sleep_interval)
+                        await asyncio.sleep(sleep_interval)
                     elif self._api_keys:
                         # if request failed, cycle API keys and try again
                         self.cycle_api_key()
@@ -1176,7 +1184,30 @@ class BaseModule:
 
         return r
 
-    async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
+    def _get_retry_after(self, r):
+        # try to get retry_after from headers first
+        headers = getattr(r, "headers", {})
+        retry_after = headers.get("Retry-After", None)
+        if retry_after is None:
+            # then look in body json
+            with suppress(Exception):
+                body_json = r.json()
+                if isinstance(body_json, dict):
+                    retry_after = body_json.get("retry_after", None)
+        if retry_after is not None:
+            return float(retry_after)
+
+    def _prepare_api_iter_req(self, url, page, page_size, offset, **requests_kwargs):
+        """
+        Default function for preparing an API request for iterating through paginated data.
+        """
+        url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
+        return url, requests_kwargs
+
+    def _api_response_is_success(self, r):
+        return r.is_success
+
+    async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):
         """
         An asynchronous generator function for iterating through paginated API data.
 
@@ -1189,6 +1220,7 @@ class BaseModule:
             page_size (int, optional): The number of items per page. Defaults to 100.
             json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
             next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+            iter_key (callable, optional): A function that builds each new request based on the page number, page size, and offset. Defaults to a simple implementation that autoreplaces {page} and {page_size} in the url.
             **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
 
         Yields:
@@ -1206,11 +1238,13 @@ class BaseModule:
             >>>         if not subdomains:
             >>>             break
             >>> finally:
-            >>>     agen.aclose()
+            >>>     await agen.aclose()
         """
         page = 1
         offset = 0
         result = None
+        if iter_key is None:
+            iter_key = self._prepare_api_iter_req
         while 1:
             if result and callable(next_key):
                 try:
@@ -1219,13 +1253,13 @@ class BaseModule:
                     self.debug(f"Failed to extract next page of results from {url}: {e}")
                     self.debug(traceback.format_exc())
             else:
-                new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
-            result = await self.api_request(new_url, **requests_kwargs)
+                new_url, new_kwargs = iter_key(url, page, page_size, offset, **requests_kwargs)
+            result = await self.api_request(new_url, **new_kwargs)
             if result is None:
                 self.verbose(f"api_page_iter() got no response for {url}")
                 break
             try:
-                if json:
+                if _json:
                     result = result.json()
                 yield result
             except Exception:

bbot/modules/censys.py

@@ -72,7 +72,7 @@ class censys(subdomain_enum_apikey):
                     error = d.get("error", "")
                     if error:
                         self.warning(error)
-                self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i+1}')
+                self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i + 1}')
                 self.debug(f"Response: {resp.text}")
                 break
             else:

bbot/modules/deadly/dastardly.py

@@ -12,7 +12,7 @@ class dastardly(BaseModule):
         "author": "@domwhewell-sage",
     }
 
-    deps_pip = ["lxml~=4.9.2"]
+    deps_pip = ["lxml~=5.3.0"]
     deps_common = ["docker"]
     per_hostport_only = True
 
@@ -37,8 +37,8 @@ class dastardly(BaseModule):
         self.verbose(f"Running Dastardly scan against {host}")
         command, output_file = self.construct_command(host)
         finished_proc = await self.run_process(command, sudo=True)
-        self.debug(f'dastardly stdout: {getattr(finished_proc, "stdout", "")}')
-        self.debug(f'dastardly stderr: {getattr(finished_proc, "stderr", "")}')
+        self.debug(f"dastardly stdout: {getattr(finished_proc, 'stdout', '')}")
+        self.debug(f"dastardly stderr: {getattr(finished_proc, 'stderr', '')}")
         for testsuite in self.parse_dastardly_xml(output_file):
             url = testsuite.endpoint
             for testcase in testsuite.testcases:

bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.7",
+        "version": "3.3.8",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/modules/dehashed.py

@@ -90,7 +90,7 @@ class dehashed(subdomain_enum):
         url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
         page = 0
         num_entries = 0
-        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False)
+        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, _json=False)
         async for result in agen:
             result_json = {}
             with suppress(Exception):
@@ -110,6 +110,6 @@ class dehashed(subdomain_enum):
                     self.info(
                         f"{domain} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results."
                     )
-                agen.aclose()
+                await agen.aclose()
                 break
             yield entries

bbot/modules/dnsbrute_mutations.py

@@ -121,7 +121,9 @@ class dnsbrute_mutations(BaseModule):
                         break
 
                     if mutations:
-                        self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(trimmed_found)})")
+                        self.info(
+                            f"Trying {len(mutations):,} mutations against {domain} ({i + 1}/{len(trimmed_found)})"
+                        )
                         results = await self.helpers.dns.brute(self, query, mutations)
                         try:
                             mutation_run = self._mutation_run_counter[domain]

bbot/modules/docker_pull.py

@@ -191,7 +191,7 @@ class docker_pull(BaseModule):
             layer_filenames = []
             layer_digests = await self.get_layers(manifest)
             for i, layer_digest in enumerate(layer_digests):
-                self.verbose(f"Downloading layer {i+1}/{len(layer_digests)} from {repository}:{tag}")
+                self.verbose(f"Downloading layer {i + 1}/{len(layer_digests)} from {repository}:{tag}")
                 blob, layer_filename = await self.download_and_get_filename(registry, repository, layer_digest)
                 layer_filenames.append(layer_filename)
                 await self.write_file_to_tar(tar, layer_filename, blob)

bbot/modules/dockerhub.py

@@ -64,7 +64,7 @@ class dockerhub(BaseModule):
     async def get_repos(self, username):
         repos = []
         url = f"{self.api_url}/repositories/{username}?page_size=25&page=" + "{page}"
-        agen = self.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, _json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -85,5 +85,5 @@ class dockerhub(BaseModule):
                     if image_name and namespace:
                         repos.append("https://hub.docker.com/r/" + namespace + "/" + image_name)
         finally:
-            agen.aclose()
+            await agen.aclose()
         return repos

bbot/modules/dotnetnuke.py

@@ -95,7 +95,7 @@ class dotnetnuke(BaseModule):
 
         if detected is True:
             # DNNPersonalization Deserialization Detection
-            for probe_url in [f'{event.data["url"]}/__', f'{event.data["url"]}/', f'{event.data["url"]}']:
+            for probe_url in [f"{event.data['url']}/__", f"{event.data['url']}/", f"{event.data['url']}"]:
                 result = await self.helpers.request(probe_url, cookies=self.exploit_probe)
                 if result:
                     if "for 16-bit app support" in result.text and "[extensions]" in result.text:
@@ -115,7 +115,7 @@ class dotnetnuke(BaseModule):
             if "endpoint" not in event.tags:
                 # NewsArticlesSlider ImageHandler.ashx File Read
                 result = await self.helpers.request(
-                    f'{event.data["url"]}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx?img=~/web.config'
+                    f"{event.data['url']}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx?img=~/web.config"
                 )
                 if result:
                     if "<configuration>" in result.text:
@@ -125,16 +125,16 @@ class dotnetnuke(BaseModule):
                                 "severity": "CRITICAL",
                                 "description": description,
                                 "host": str(event.host),
-                                "url": f'{event.data["url"]}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx',
+                                "url": f"{event.data['url']}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx",
                             },
                             "VULNERABILITY",
                             event,
-                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                            context=f"{{module}} scanned {event.data['url']} and found critical {{event.type}}: {description}",
                         )
 
                 # DNNArticle GetCSS.ashx File Read
                 result = await self.helpers.request(
-                    f'{event.data["url"]}/DesktopModules/DNNArticle/getcss.ashx?CP=%2fweb.config&smid=512&portalid=3'
+                    f"{event.data['url']}/DesktopModules/DNNArticle/getcss.ashx?CP=%2fweb.config&smid=512&portalid=3"
                 )
                 if result:
                     if "<configuration>" in result.text:
@@ -144,19 +144,19 @@ class dotnetnuke(BaseModule):
                                 "severity": "CRITICAL",
                                 "description": description,
                                 "host": str(event.host),
-                                "url": f'{event.data["url"]}/Desktopmodules/DNNArticle/GetCSS.ashx/?CP=%2fweb.config',
+                                "url": f"{event.data['url']}/Desktopmodules/DNNArticle/GetCSS.ashx/?CP=%2fweb.config",
                             },
                             "VULNERABILITY",
                             event,
-                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                            context=f"{{module}} scanned {event.data['url']} and found critical {{event.type}}: {description}",
                         )
 
                 # InstallWizard SuperUser Privilege Escalation
-                result = await self.helpers.request(f'{event.data["url"]}/Install/InstallWizard.aspx')
+                result = await self.helpers.request(f"{event.data['url']}/Install/InstallWizard.aspx")
                 if result:
                     if result.status_code == 200:
                         result_confirm = await self.helpers.request(
-                            f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
+                            f"{event.data['url']}/Install/InstallWizard.aspx?__viewstate=1"
                         )
                         if result_confirm.status_code == 500:
                             description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
@@ -165,11 +165,11 @@ class dotnetnuke(BaseModule):
                                     "severity": "CRITICAL",
                                     "description": description,
                                     "host": str(event.host),
-                                    "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
+                                    "url": f"{event.data['url']}/Install/InstallWizard.aspx",
                                 },
                                 "VULNERABILITY",
                                 event,
-                                context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                                context=f"{{module}} scanned {event.data['url']} and found critical {{event.type}}: {description}",
                             )
                             return
 
@@ -180,7 +180,7 @@ class dotnetnuke(BaseModule):
                     self.interactsh_subdomain_tags[subdomain_tag] = event
 
                     await self.helpers.request(
-                        f'{event.data["url"]}/DnnImageHandler.ashx?mode=file&url=http://{subdomain_tag}.{self.interactsh_domain}'
+                        f"{event.data['url']}/DnnImageHandler.ashx?mode=file&url=http://{subdomain_tag}.{self.interactsh_domain}"
                     )
                 else:
                     self.debug(

bbot/modules/extractous.py

@@ -65,7 +65,7 @@ class extractous(BaseModule):
         "extensions": "File extensions to parse",
     }
 
-    deps_pip = ["extractous"]
+    deps_pip = ["extractous~=0.3.0"]
     scope_distance_modifier = 1
 
     async def setup(self):

bbot/modules/ffuf_shortnames.py

@@ -1,3 +1,4 @@
+import pickle
 import re
 import random
 import string
@@ -5,32 +6,10 @@ import string
 from bbot.modules.deadly.ffuf import ffuf
 
 
-def find_common_prefixes(strings, minimum_set_length=4):
-    prefix_candidates = [s[:i] for s in strings if len(s) == 6 for i in range(3, 6)]
-    frequency_dict = {item: prefix_candidates.count(item) for item in prefix_candidates}
-    frequency_dict = {k: v for k, v in frequency_dict.items() if v >= minimum_set_length}
-    prefix_list = list(set(frequency_dict.keys()))
-
-    found_prefixes = set()
-    for prefix in prefix_list:
-        prefix_frequency = frequency_dict[prefix]
-        is_substring = False
-
-        for k, v in frequency_dict.items():
-            if prefix != k:
-                if prefix in k:
-                    is_substring = True
-        if not is_substring:
-            found_prefixes.add(prefix)
-        else:
-            if prefix_frequency > v and (len(k) - len(prefix) == 1):
-                found_prefixes.add(prefix)
-    return list(found_prefixes)
-
-
 class ffuf_shortnames(ffuf):
     watched_events = ["URL_HINT"]
     produced_events = ["URL_UNVERIFIED"]
+    deps_pip = ["numpy"]
     flags = ["aggressive", "active", "iis-shortnames", "web-thorough"]
     meta = {
         "description": "Use ffuf in combination IIS shortnames",
@@ -41,54 +20,118 @@ class ffuf_shortnames(ffuf):
     options = {
         "wordlist": "",  # default is defined within setup function
         "wordlist_extensions": "",  # default is defined within setup function
-        "lines": 1000000,
         "max_depth": 1,
         "version": "2.0.0",
         "extensions": "",
         "ignore_redirects": True,
         "find_common_prefixes": False,
         "find_delimiters": True,
+        "max_predictions": 250,
     }
 
     options_desc = {
         "wordlist": "Specify wordlist to use when finding directories",
         "wordlist_extensions": "Specify wordlist to use when making extension lists",
-        "lines": "take only the first N lines from the wordlist when finding directories",
         "max_depth": "the maximum directory depth to attempt to solve",
         "version": "ffuf version",
         "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
         "ignore_redirects": "Explicitly ignore redirects (301,302)",
         "find_common_prefixes": "Attempt to automatically detect common prefixes and make additional ffuf runs against them",
         "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them",
+        "max_predictions": "The maximum number of predictions to generate per shortname prefix",
     }
 
     deps_common = ["ffuf"]
 
     in_scope_only = True
 
+    def generate_templist(self, prefix, shortname_type):
+        virtual_file = []
+
+        for prediction, score in self.predict(prefix, self.max_predictions, model=shortname_type):
+            self.debug(f"Got prediction: [{prediction}] from prefix [{prefix}] with score [{score}]")
+            virtual_file.append(prediction)
+        virtual_file.append(self.canary)
+        return self.helpers.tempfile(virtual_file, pipe=False), len(virtual_file)
+
+    def predict(self, prefix, n=25, model="endpoint"):
+        predictor_name = f"{model}_predictor"
+        predictor = getattr(self, predictor_name)
+        return predictor.predict(prefix, n)
+
+    @staticmethod
+    def find_common_prefixes(strings, minimum_set_length=4):
+        prefix_candidates = [s[:i] for s in strings if len(s) == 6 for i in range(3, 6)]
+        frequency_dict = {item: prefix_candidates.count(item) for item in prefix_candidates}
+        frequency_dict = {k: v for k, v in frequency_dict.items() if v >= minimum_set_length}
+        prefix_list = list(set(frequency_dict.keys()))
+
+        found_prefixes = set()
+        for prefix in prefix_list:
+            prefix_frequency = frequency_dict[prefix]
+            is_substring = False
+
+            for k, v in frequency_dict.items():
+                if prefix != k:
+                    if prefix in k:
+                        is_substring = True
+            if not is_substring:
+                found_prefixes.add(prefix)
+            else:
+                if prefix_frequency > v and (len(k) - len(prefix) == 1):
+                    found_prefixes.add(prefix)
+        return list(found_prefixes)
+
     async def setup(self):
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
-        wordlist = self.config.get("wordlist", "")
-        if not wordlist:
-            wordlist = f"{self.helpers.wordlist_dir}/ffuf_shortname_candidates.txt"
-        self.debug(f"Using [{wordlist}] for shortname candidate list")
-        self.wordlist = await self.helpers.wordlist(wordlist)
-        self.wordlist_lines = self.generate_wordlist(self.wordlist)
-
         wordlist_extensions = self.config.get("wordlist_extensions", "")
         if not wordlist_extensions:
             wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
         self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
+        self.ignore_redirects = self.config.get("ignore_redirects")
+        self.max_predictions = self.config.get("max_predictions")
 
-        try:
-            self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True)
-            self.debug(f"Using custom extensions: [{','.join(self.extensions)}]")
-        except ValueError as e:
-            return False, f"Error parsing extensions: {e}"
+        class MinimalWordPredictor:
+            def __init__(self):
+                self.word_frequencies = {}
 
-        self.ignore_redirects = self.config.get("ignore_redirects")
+            def predict(self, prefix, top_n):
+                prefix = prefix.lower()
+                matches = [(word, freq) for word, freq in self.word_frequencies.items() if word.startswith(prefix)]
+
+                if not matches:
+                    return []
+
+                matches.sort(key=lambda x: x[1], reverse=True)
+                matches = matches[:top_n]
+
+                max_freq = matches[0][1]
+                return [(word, freq / max_freq) for word, freq in matches]
+
+        class CustomUnpickler(pickle.Unpickler):
+            def find_class(self, module, name):
+                if name == "MinimalWordPredictor":
+                    return MinimalWordPredictor
+                return super().find_class(module, name)
+
+        endpoint_model = await self.helpers.download(
+            "https://raw.githubusercontent.com/blacklanternsecurity/wordpredictor/refs/heads/main/trained_models/endpoints.bin"
+        )
+        directory_model = await self.helpers.download(
+            "https://raw.githubusercontent.com/blacklanternsecurity/wordpredictor/refs/heads/main/trained_models/directories.bin"
+        )
+
+        self.debug(f"Loading endpoint model from: {endpoint_model}")
+        with open(endpoint_model, "rb") as f:
+            unpickler = CustomUnpickler(f)
+            self.endpoint_predictor = unpickler.load()
+
+        self.debug(f"Loading directory model from: {directory_model}")
+        with open(directory_model, "rb") as f:
+            unpickler = CustomUnpickler(f)
+            self.directory_predictor = unpickler.load()
 
         self.per_host_collection = {}
         self.shortname_to_event = {}
@@ -123,6 +166,14 @@ class ffuf_shortnames(ffuf):
     async def handle_event(self, event):
         filename_hint = re.sub(r"~\d", "", event.parsed_url.path.rsplit(".", 1)[0].split("/")[-1]).lower()
 
+        if "shortname-endpoint" in event.tags:
+            shortname_type = "endpoint"
+        elif "shortname-directory" in event.tags:
+            shortname_type = "directory"
+        else:
+            self.error("ffuf_shortnames received URL_HINT without proper 'shortname-' tag")
+            return
+
         host = f"{event.parent.parsed_url.scheme}://{event.parent.parsed_url.netloc}/"
         if host not in self.per_host_collection.keys():
             self.per_host_collection[host] = [(filename_hint, event.parent.data)]
@@ -135,11 +186,11 @@ class ffuf_shortnames(ffuf):
         root_stub = "/".join(event.parsed_url.path.split("/")[:-1])
         root_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{root_stub}/"
 
-        if "shortname-file" in event.tags:
+        if shortname_type == "endpoint":
             used_extensions = self.build_extension_list(event)
 
         if len(filename_hint) == 6:
-            tempfile, tempfile_len = self.generate_templist(prefix=filename_hint)
+            tempfile, tempfile_len = self.generate_templist(filename_hint, shortname_type)
             self.verbose(
                 f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]"
             )
@@ -149,7 +200,7 @@ class ffuf_shortnames(ffuf):
             tempfile_len = 1
 
         if tempfile_len > 0:
-            if "shortname-file" in event.tags:
+            if shortname_type == "endpoint":
                 for ext in used_extensions:
                     async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"):
                         await self.emit_event(
@@ -160,7 +211,7 @@ class ffuf_shortnames(ffuf):
                             context=f"{{module}} brute-forced {ext.upper()} files at {root_url} and found {{event.type}}: {{event.data}}",
                         )
 
-            elif "shortname-directory" in event.tags:
+            elif shortname_type == "directory":
                 async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]):
                     r_url = f"{r['url'].rstrip('/')}/"
                     await self.emit_event(
@@ -177,7 +228,7 @@ class ffuf_shortnames(ffuf):
                 if delimiter_r:
                     delimiter, prefix, partial_hint = delimiter_r
                     self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
-                    tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+                    tempfile, tempfile_len = self.generate_templist(partial_hint, "directory")
                     ffuf_prefix = f"{prefix}{delimiter}"
                     async for r in self.execute_ffuf(tempfile, root_url, prefix=ffuf_prefix, exts=["/"]):
                         await self.emit_event(
@@ -188,13 +239,13 @@ class ffuf_shortnames(ffuf):
                             context=f'{{module}} brute-forced directories with detected prefix "{ffuf_prefix}" and found {{event.type}}: {{event.data}}',
                         )
 
-            elif "shortname-file" in event.tags:
+            elif "shortname-endpoint" in event.tags:
                 for ext in used_extensions:
                     delimiter_r = self.find_delimiter(filename_hint)
                     if delimiter_r:
                         delimiter, prefix, partial_hint = delimiter_r
                         self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
-                        tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+                        tempfile, tempfile_len = self.generate_templist(partial_hint, "endpoint")
                         ffuf_prefix = f"{prefix}{delimiter}"
                         async for r in self.execute_ffuf(tempfile, root_url, prefix=ffuf_prefix, suffix=f".{ext}"):
                             await self.emit_event(
@@ -213,17 +264,25 @@ class ffuf_shortnames(ffuf):
             for host, hint_tuple_list in per_host_collection.items():
                 hint_list = [x[0] for x in hint_tuple_list]
 
-                common_prefixes = find_common_prefixes(hint_list)
+                common_prefixes = self.find_common_prefixes(hint_list)
                 for prefix in common_prefixes:
                     self.verbose(f"Found common prefix: [{prefix}] for host [{host}]")
                     for hint_tuple in hint_tuple_list:
                         hint, url = hint_tuple
                         if hint.startswith(prefix):
+                            if "shortname-endpoint" in self.shortname_to_event[hint].tags:
+                                shortname_type = "endpoint"
+                            elif "shortname-directory" in self.shortname_to_event[hint].tags:
+                                shortname_type = "directory"
+                            else:
+                                self.error("ffuf_shortnames received URL_HINT without proper 'shortname-' tag")
+                                continue
+
                             partial_hint = hint[len(prefix) :]
 
                             # safeguard to prevent loading the entire wordlist
                             if len(partial_hint) > 0:
-                                tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+                                tempfile, tempfile_len = self.generate_templist(partial_hint, shortname_type)
 
                                 if "shortname-directory" in self.shortname_to_event[hint].tags:
                                     self.verbose(
@@ -238,7 +297,7 @@ class ffuf_shortnames(ffuf):
                                             tags=[f"status-{r['status']}"],
                                             context=f'{{module}} brute-forced directories with common prefix "{prefix}" and found {{event.type}}: {{event.data}}',
                                         )
-                                elif "shortname-file" in self.shortname_to_event[hint].tags:
+                                elif shortname_type == "endpoint":
                                     used_extensions = self.build_extension_list(self.shortname_to_event[hint])
 
                                     for ext in used_extensions:

bbot/modules/filedownload.py

@@ -65,6 +65,7 @@ class filedownload(BaseModule):
             "swp",  #  Swap File (temporary file, often Vim)
             "sxw",  #  OpenOffice.org Writer document
             "tar.gz",  # Gzip-Compressed Tar Archive
+            "tgz",  #  Gzip-Compressed Tar Archive
             "tar",  #  Tar Archive
             "txt",  #  Plain Text Document
             "vbs",  #  Visual Basic Script
@@ -76,6 +77,11 @@ class filedownload(BaseModule):
             "yaml",  #  YAML Ain't Markup Language
             "yml",  #  YAML Ain't Markup Language
             "zip",  #  Zip Archive
+            "lzma",  #  LZMA Compressed File
+            "rar",  #  RAR Compressed File
+            "7z",  #  7-Zip Compressed File
+            "xz",  #  XZ Compressed File
+            "bz2",  #  Bzip2 Compressed File
         ],
         "max_filesize": "10MB",
         "base_64_encoded_file": "false",