bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/core/helpers/dns/engine.py

@@ -7,7 +7,6 @@ import traceback
 from cachetools import LRUCache
 from contextlib import suppress
 
-from bbot.errors import DNSWildcardBreak
 from bbot.core.engine import EngineServer
 from bbot.core.helpers.async_helpers import NamedLock
 from bbot.core.helpers.dns.helpers import extract_targets
@@ -16,7 +15,6 @@ from bbot.core.helpers.misc import (
     rand_string,
     parent_domain,
     domain_parents,
-    clean_dns_record,
 )
 
 
@@ -26,7 +24,6 @@ all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"]
 
 
 class DNSEngine(EngineServer):
-
     CMDS = {
         0: "resolve",
         1: "resolve_raw",
@@ -57,7 +54,7 @@ class DNSEngine(EngineServer):
         dns_omit_queries = self.dns_config.get("omit_queries", None)
         if not dns_omit_queries:
             dns_omit_queries = []
-        self.dns_omit_queries = dict()
+        self.dns_omit_queries = {}
         for d in dns_omit_queries:
             d = d.split(":")
             if len(d) == 2:
@@ -75,7 +72,7 @@ class DNSEngine(EngineServer):
             self.wildcard_ignore = []
         self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore])
         self.wildcard_tests = self.dns_config.get("wildcard_tests", 5)
-        self._wildcard_cache = dict()
+        self._wildcard_cache = {}
         # since wildcard detection takes some time, This is to prevent multiple
         # modules from kicking off wildcard detection for the same domain at the same time
         self._wildcard_lock = NamedLock()
@@ -85,7 +82,7 @@ class DNSEngine(EngineServer):
         self._last_connectivity_warning = time.time()
         # keeps track of warnings issued for wildcard detection to prevent duplicate warnings
         self._dns_warnings = set()
-        self._errors = dict()
+        self._errors = {}
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
@@ -208,8 +205,8 @@ class DNSEngine(EngineServer):
         retries = kwargs.pop("retries", self.retries)
         use_cache = kwargs.pop("use_cache", True)
         tries_left = int(retries) + 1
-        parent_hash = hash(f"{parent}:{rdtype}")
-        dns_cache_hash = hash(f"{query}:{rdtype}")
+        parent_hash = hash((parent, rdtype))
+        dns_cache_hash = hash((query, rdtype))
         while tries_left > 0:
             try:
                 if use_cache:
@@ -297,7 +294,7 @@ class DNSEngine(EngineServer):
         tries_left = int(retries) + 1
         results = []
         errors = []
-        dns_cache_hash = hash(f"{query}:PTR")
+        dns_cache_hash = hash((query, "PTR"))
         while tries_left > 0:
             try:
                 if use_cache:
@@ -361,8 +358,7 @@ class DNSEngine(EngineServer):
         ):
             query = args[0]
             rdtype = kwargs["type"]
-            for answer in answers:
-                yield ((query, rdtype), (answer, errors))
+            yield ((query, rdtype), (answers, errors))
 
     async def _catch(self, callback, *args, **kwargs):
         """
@@ -397,18 +393,21 @@ class DNSEngine(EngineServer):
             self.log.trace(traceback.format_exc())
         return []
 
-    async def is_wildcard(self, query, ips=None, rdtype=None):
+    async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
         Use this method to check whether a *host* is a wildcard entry
 
         This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain.
 
+        It works by making a bunch of random DNS queries to the parent domain, compiling a list of wildcard IPs,
+        then comparing those to the IPs of the host in question. If the host's IP matches the wildcard ones, it's a wildcard.
+
         If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead.
 
         Args:
             query (str): The hostname to check for a wildcard entry.
-            ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query.
-            rdtype (str, optional): The DNS record type (e.g., "A", "AAAA") to consider during the check.
+            rdtypes (list): The DNS record type (e.g., "A", "AAAA") to consider during the check.
+            raw_dns_records (dict, optional): Dictionary of {rdtype: [answer1, answer2, ...], ...} containing raw dnspython answers for the query.
 
         Returns:
             dict: A dictionary indicating if the query is a wildcard for each checked DNS record type.
@@ -416,108 +415,115 @@ class DNSEngine(EngineServer):
                 Values are tuples where the first element is a boolean indicating if the query is a wildcard,
                 and the second element is the wildcard parent if it's a wildcard.
 
-        Raises:
-            ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified.
-
         Examples:
-            >>> is_wildcard("www.github.io")
-            {"A": (True, "github.io"), "AAAA": (True, "github.io")}
+            >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"])
+            {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")}
 
-            >>> is_wildcard("www.evilcorp.com", ips=["93.184.216.34"], rdtype="A")
+            >>> is_wildcard("www.evilcorp.com", rdtypes=["A"])
             {"A": (False, "evilcorp.com")}
 
         Note:
             `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
         """
-        result = {}
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
 
-        parent = parent_domain(query)
-        parents = list(domain_parents(query))
+        result = {}
 
-        if rdtype is not None:
-            if isinstance(rdtype, str):
-                rdtype = [rdtype]
-            rdtypes_to_check = rdtype
-        else:
-            rdtypes_to_check = all_rdtypes
-
-        query_baseline = dict()
-        # if the caller hasn't already done the work of resolving the IPs
-        if ips is None:
-            # then resolve the query for all rdtypes
-            queries = [(query, t) for t in rdtypes_to_check]
-            async for (query, _rdtype), (answers, errors) in self.resolve_raw_batch(queries):
-                answers = extract_targets(answers)
+        # if the work of resolving hasn't been done yet, do it
+        if raw_dns_records is None:
+            raw_dns_records = {}
+            queries = [(query, rdtype) for rdtype in rdtypes]
+            async for (_, rdtype), (answers, errors) in self.resolve_raw_batch(queries):
                 if answers:
-                    query_baseline[_rdtype] = set([a[1] for a in answers])
+                    for answer in answers:
+                        try:
+                            raw_dns_records[rdtype].add(answer)
+                        except KeyError:
+                            raw_dns_records[rdtype] = {answer}
                 else:
                     if errors:
-                        self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection")
-                        result[_rdtype] = (None, parent)
-                        continue
-        else:
-            # otherwise, we can skip all that
-            cleaned_ips = set([clean_dns_record(ip) for ip in ips])
-            if not cleaned_ips:
-                raise ValueError("Valid IPs must be specified")
-            query_baseline[rdtype] = cleaned_ips
-        if not query_baseline:
+                        self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection")
+                        result[rdtype] = ("ERROR", query)
+
+        # clean + process the raw records into a baseline
+        baseline = {}
+        baseline_raw = {}
+        for rdtype, answers in raw_dns_records.items():
+            for answer in answers:
+                text_answer = answer.to_text()
+                try:
+                    baseline_raw[rdtype].add(text_answer)
+                except KeyError:
+                    baseline_raw[rdtype] = {text_answer}
+                for _, host in extract_targets(answer):
+                    try:
+                        baseline[rdtype].add(host)
+                    except KeyError:
+                        baseline[rdtype] = {host}
+
+        # if it's unresolved, it's a big nope
+        if not raw_dns_records:
             return result
 
         # once we've resolved the base query and have IP addresses to work with
         # we can compare the IPs to the ones we have on file for wildcards
 
+        # only bother to check the rdypes that actually resolve
+        rdtypes_to_check = set(raw_dns_records)
+
         # for every parent domain, starting with the shortest
-        try:
-            for host in parents[::-1]:
-                # make sure we've checked that domain for wildcards
-                await self.is_wildcard_domain(host)
-
-                # for every rdtype
-                for _rdtype in list(query_baseline):
-                    # get the IPs from above
-                    query_ips = query_baseline.get(_rdtype, set())
-                    host_hash = hash(host)
-
-                    if host_hash in self._wildcard_cache:
-                        # then get its IPs from our wildcard cache
-                        wildcard_rdtypes = self._wildcard_cache[host_hash]
-
-                        # then check to see if our IPs match the wildcard ones
-                        if _rdtype in wildcard_rdtypes:
-                            wildcard_ips = wildcard_rdtypes[_rdtype]
-                            # if our IPs match the wildcard ones, then ladies and gentlemen we have a wildcard
-                            is_wildcard = any(r in wildcard_ips for r in query_ips)
-
-                            if is_wildcard and not result.get(_rdtype, (None, None))[0] is True:
-                                result[_rdtype] = (True, host)
-
-                    # if we've reached a point where the dns name is a complete wildcard, class can be dismissed early
-                    base_query_rdtypes = set(query_baseline)
-                    wildcard_rdtypes_set = set([k for k, v in result.items() if v[0] is True])
-                    if base_query_rdtypes and wildcard_rdtypes_set and base_query_rdtypes == wildcard_rdtypes_set:
-                        self.log.debug(
-                            f"Breaking from wildcard detection for {query} at {host} because base query rdtypes ({base_query_rdtypes}) == wildcard rdtypes ({wildcard_rdtypes_set})"
-                        )
-                        raise DNSWildcardBreak()
-
-        except DNSWildcardBreak:
-            pass
-
-        for _rdtype, answers in query_baseline.items():
-            if answers and _rdtype not in result:
-                result[_rdtype] = (False, query)
+        parents = list(domain_parents(query))
+        for parent in parents[::-1]:
+            # check if the parent domain is set up with wildcards
+            wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)
+
+            # for every rdtype
+            for rdtype in list(baseline_raw):
+                # skip if we already found a wildcard for this rdtype
+                if rdtype in result:
+                    continue
+
+                # get our baseline IPs from above
+                _baseline = baseline.get(rdtype, set())
+                _baseline_raw = baseline_raw.get(rdtype, set())
+
+                wildcard_rdtypes = wildcard_results.get(parent, {})
+                wildcards = wildcard_rdtypes.get(rdtype, None)
+                if wildcards is None:
+                    continue
+                wildcards, wildcard_raw = wildcards
+
+                if wildcard_raw:
+                    # skip this rdtype from now on
+                    rdtypes_to_check.remove(rdtype)
+
+                    # check if any of our baseline IPs are in the wildcard results
+                    is_wildcard = any(r in wildcards for r in _baseline)
+                    is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw)
+
+                    # if there are any matches, we have a wildcard
+                    if is_wildcard or is_wildcard_raw:
+                        result[rdtype] = (True, parent)
+                    else:
+                        # otherwise, it's still suspicious, because we had random stuff resolve at this level
+                        result[rdtype] = ("POSSIBLE", parent)
+
+        # any rdtype that wasn't a wildcard, mark it as False
+        for rdtype, answers in baseline_raw.items():
+            if answers and rdtype not in result:
+                result[rdtype] = (False, query)
 
         return result
 
-    async def is_wildcard_domain(self, domain, log_info=False):
+    async def is_wildcard_domain(self, domain, rdtypes):
         """
         Check whether a given host or its children make use of wildcard DNS entries. Wildcard DNS can have
         various implications, particularly in subdomain enumeration and subdomain takeovers.
 
         Args:
             domain (str): The domain to check for wildcard DNS entries.
-            log_info (bool, optional): Whether to log the result of the check. Defaults to False.
+            rdtypes (list): Which DNS record types to check.
 
         Returns:
             dict: A dictionary where the keys are the parent domains that have wildcard DNS entries,
@@ -531,60 +537,74 @@ class DNSEngine(EngineServer):
             >>> is_wildcard_domain("example.com")
             {}
         """
-        wildcard_domain_results = {}
-
-        rdtypes_to_check = set(all_rdtypes)
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
+        rdtypes = set(rdtypes)
 
+        wildcard_results = {}
         # make a list of its parents
         parents = list(domain_parents(domain, include_self=True))
         # and check each of them, beginning with the highest parent (i.e. the root domain)
         for i, host in enumerate(parents[::-1]):
-            # have we checked this host before?
-            host_hash = hash(host)
-            async with self._wildcard_lock.lock(host_hash):
-                # if we've seen this host before
-                if host_hash in self._wildcard_cache:
-                    wildcard_domain_results[host] = self._wildcard_cache[host_hash]
-                    continue
+            host_results = {}
+            queries = [((host, rdtype), {}) for rdtype in rdtypes]
+            async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool(
+                self._is_wildcard_zone, args_kwargs=queries
+            ):
+                # if we hit a wildcard, we can skip this rdtype from now on
+                if results_raw:
+                    rdtypes.remove(rdtype)
+                    host_results[rdtype] = results, results_raw
+
+            if host_results:
+                wildcard_results[host] = host_results
+
+        return wildcard_results
+
+    async def _is_wildcard_zone(self, host, rdtype):
+        """
+        Check whether a specific DNS zone+rdtype has a wildcard configuration
+        """
+        rdtype = rdtype.upper()
 
-                self.log.verbose(f"Checking if {host} is a wildcard")
+        # have we checked this host before?
+        host_hash = hash((host, rdtype))
+        async with self._wildcard_lock.lock(host_hash):
+            # if we've seen this host before
+            try:
+                wildcard_results, wildcard_results_raw = self._wildcard_cache[host_hash]
+                self.debug(f"Got {host}:{rdtype} from cache")
+            except KeyError:
+                wildcard_results = set()
+                wildcard_results_raw = set()
+                self.debug(f"Checking if {host}:{rdtype} is a wildcard")
 
                 # determine if this is a wildcard domain
-
                 # resolve a bunch of random subdomains of the same parent
-                is_wildcard = False
-                wildcard_results = dict()
-
                 rand_queries = []
-                for rdtype in rdtypes_to_check:
-                    for _ in range(self.wildcard_tests):
-                        rand_query = f"{rand_string(digits=False, length=10)}.{host}"
-                        rand_queries.append((rand_query, rdtype))
+                for _ in range(self.wildcard_tests):
+                    rand_query = f"{rand_string(digits=False, length=10)}.{host}"
+                    rand_queries.append((rand_query, rdtype))
 
                 async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(rand_queries, use_cache=False):
-                    answers = extract_targets(answers)
-                    if answers:
-                        is_wildcard = True
-                        if not rdtype in wildcard_results:
-                            wildcard_results[rdtype] = set()
-                        wildcard_results[rdtype].update(set(a[1] for a in answers))
-                        # we know this rdtype is a wildcard
-                        # so we don't need to check it anymore
-                        with suppress(KeyError):
-                            rdtypes_to_check.remove(rdtype)
-
-                self._wildcard_cache.update({host_hash: wildcard_results})
-                wildcard_domain_results.update({host: wildcard_results})
-                if is_wildcard:
-                    wildcard_rdtypes_str = ",".join(sorted([t.upper() for t, r in wildcard_results.items() if r]))
-                    log_fn = self.log.verbose
-                    if log_info:
-                        log_fn = self.log.info
-                    log_fn(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}")
+                    for answer in answers:
+                        # consider both the raw record
+                        wildcard_results_raw.add(answer.to_text())
+                        # and all the extracted hosts
+                        for _, t in extract_targets(answer):
+                            wildcard_results.add(t)
+
+                if wildcard_results:
+                    self.log.info(f"Encountered domain with wildcard DNS ({rdtype}): *.{host}")
                 else:
-                    self.log.verbose(f"Finished checking {host}, it is not a wildcard")
+                    self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard")
+                self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw
 
-        return wildcard_domain_results
+        return wildcard_results, wildcard_results_raw
+
+    async def _is_wildcard(self, query, rdtypes, dns_children):
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
 
     @property
     def dns_connectivity_lock(self):
@@ -618,7 +638,7 @@ class DNSEngine(EngineServer):
                     self._last_dns_success = time.time()
                     return True
         if time.time() - self._last_connectivity_warning > interval:
-            self.log.warning(f"DNS queries are failing, please check your internet connection")
+            self.log.warning("DNS queries are failing, please check your internet connection")
             self._last_connectivity_warning = time.time()
         self._errors.clear()
         return False
@@ -631,7 +651,14 @@ class DNSEngine(EngineServer):
     def in_tests(self):
         return os.getenv("BBOT_TESTING", "") == "True"
 
-    async def _mock_dns(self, mock_data):
+    async def _mock_dns(self, mock_data, custom_lookup_fn=None):
         from .mock import MockResolver
 
-        self.resolver = MockResolver(mock_data)
+        def deserialize_function(func_source):
+            assert self.in_tests, "Can only mock when BBOT_TESTING=True"
+            if func_source is None:
+                return None
+            exec(func_source)
+            return locals()["custom_lookup"]
+
+        self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn))

bbot/core/helpers/dns/helpers.py

@@ -1,6 +1,6 @@
 import logging
 
-from bbot.core.helpers.regexes import dns_name_regex
+from bbot.core.helpers.regexes import dns_name_extraction_regex
 from bbot.core.helpers.misc import clean_dns_record, smart_decode
 
 log = logging.getLogger("bbot.core.helpers.dns")
@@ -198,7 +198,7 @@ def extract_targets(record):
     elif rdtype == "TXT":
         for s in record.strings:
             s = smart_decode(s)
-            for match in dns_name_regex.finditer(s):
+            for match in dns_name_extraction_regex.finditer(s):
                 start, end = match.span()
                 host = s[start:end]
                 add_result(rdtype, host)

bbot/core/helpers/dns/mock.py

@@ -1,10 +1,13 @@
 import dns
+import logging
 
+log = logging.getLogger("bbot.core.helpers.dns.mock")
 
-class MockResolver:
 
-    def __init__(self, mock_data=None):
+class MockResolver:
+    def __init__(self, mock_data=None, custom_lookup_fn=None):
         self.mock_data = mock_data if mock_data else {}
+        self._custom_lookup_fn = custom_lookup_fn
         self.nameservers = ["127.0.0.1"]
 
     async def resolve_address(self, ipaddr, *args, **kwargs):
@@ -13,12 +16,22 @@ class MockResolver:
         modified_kwargs["rdtype"] = "PTR"
         return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs)
 
-    def create_dns_response(self, query_name, rdtype):
-        query_name = query_name.strip(".")
-        answers = self.mock_data.get(query_name, {}).get(rdtype, [])
-        if not answers:
-            raise dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}")
+    def _lookup(self, query, rdtype):
+        query = query.strip(".")
+        ret = []
+        if self._custom_lookup_fn is not None:
+            answers = self._custom_lookup_fn(query, rdtype)
+            if answers is not None:
+                ret.extend(list(answers))
+        answers = self.mock_data.get(query, {}).get(rdtype, [])
+        if answers:
+            ret.extend(list(answers))
+        if not ret:
+            raise dns.resolver.NXDOMAIN(f"No answer found for {query} {rdtype}")
+        return ret
 
+    def create_dns_response(self, query_name, answers, rdtype):
+        query_name = query_name.strip(".")
         message_text = f"""id 1234
 opcode QUERY
 rcode NOERROR
@@ -27,10 +40,13 @@ flags QR AA RD
 {query_name}. IN {rdtype}
 ;ANSWER"""
         for answer in answers:
+            if answer == "":
+                answer = '""'
             message_text += f"\n{query_name}. 1 IN {rdtype} {answer}"
 
         message_text += "\n;AUTHORITY\n;ADDITIONAL\n"
         message = dns.message.from_text(message_text)
+        # log.verbose(message_text)
         return message
 
     async def resolve(self, query_name, rdtype=None):
@@ -49,7 +65,9 @@ flags QR AA RD
             raise dns.resolver.NXDOMAIN
 
         try:
-            response = self.create_dns_response(query_name, rdtype)
+            answers = self._lookup(query_name, rdtype)
+            log.verbose(f"Answers for {query_name}:{rdtype}: {answers}")
+            response = self.create_dns_response(query_name, answers, rdtype)
             answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response)
             return answer
         except dns.resolver.NXDOMAIN:

bbot/core/helpers/files.py

@@ -83,7 +83,7 @@ def _feed_pipe(self, pipe, content, text=True):
                     for c in content:
                         p.write(decode_fn(c) + newline)
         except BrokenPipeError:
-            log.debug(f"Broken pipe in _feed_pipe()")
+            log.debug("Broken pipe in _feed_pipe()")
         except ValueError:
             log.debug(f"Error _feed_pipe(): {traceback.format_exc()}")
     except KeyboardInterrupt:

bbot/core/helpers/helper.py

@@ -12,10 +12,11 @@ from .diff import HttpCompare
 from .regex import RegexHelper
 from .wordcloud import WordCloud
 from .interactsh import Interactsh
-from ...scanner.target import Target
 from .depsinstaller import DepsInstaller
 from .async_helpers import get_event_loop
 
+from bbot.scanner.target import BaseTarget
+
 log = logging.getLogger("bbot.core.helpers")
 
 
@@ -152,11 +153,13 @@ class ConfigAwareHelper:
         return self.temp_dir / filename
 
     def clean_old_scans(self):
-        _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
+        def _filter(x):
+            return x.is_dir() and self.regexes.scan_name_regex.match(x.name)
+
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events, **kwargs):
-        return Target(*events, **kwargs)
+    def make_target(self, *targets, **kwargs):
+        return BaseTarget(*targets, scan=self.scan, **kwargs)
 
     @property
     def config(self):

bbot/core/helpers/interactsh.py

@@ -155,7 +155,7 @@ class Interactsh:
             break
 
         if not self.server:
-            raise InteractshError(f"Failed to register with an interactsh server")
+            raise InteractshError("Failed to register with an interactsh server")
 
         log.info(
             f"Successfully registered to interactsh server {self.server} with correlation_id {self.correlation_id} [{self.domain}]"
@@ -181,7 +181,7 @@ class Interactsh:
             >>> await interactsh_client.deregister()
         """
         if not self.server or not self.correlation_id or not self.secret:
-            raise InteractshError(f"Missing required information to deregister")
+            raise InteractshError("Missing required information to deregister")
 
         headers = {}
         if self.token:
@@ -226,7 +226,7 @@ class Interactsh:
             ]
         """
         if not self.server or not self.correlation_id or not self.secret:
-            raise InteractshError(f"Missing required information to poll")
+            raise InteractshError("Missing required information to poll")
 
         headers = {}
         if self.token:

bbot/core/helpers/libmagic.py

@@ -0,0 +1,65 @@
+import puremagic
+
+
+def get_magic_info(file):
+    magic_detections = puremagic.magic_file(file)
+    if magic_detections:
+        magic_detections.sort(key=lambda x: x.confidence, reverse=True)
+        detection = magic_detections[0]
+        return detection.extension, detection.mime_type, detection.name, detection.confidence
+    return "", "", "", 0
+
+
+def get_compression(mime_type):
+    mime_type = mime_type.lower()
+    # from https://github.com/cdgriffith/puremagic/blob/master/puremagic/magic_data.json
+    compression_map = {
+        "application/arj": "arj",  # ARJ archive
+        "application/binhex": "binhex",  # BinHex encoded file
+        "application/epub+zip": "zip",  # EPUB book (Zip archive)
+        "application/fictionbook2+zip": "zip",  # FictionBook 2.0 (Zip)
+        "application/fictionbook3+zip": "zip",  # FictionBook 3.0 (Zip)
+        "application/gzip": "gzip",  # Gzip compressed file
+        "application/java-archive": "zip",  # Java Archive (JAR)
+        "application/pak": "pak",  # PAK archive
+        "application/vnd.android.package-archive": "zip",  # Android package (APK)
+        "application/vnd.comicbook-rar": "rar",  # Comic book archive (RAR)
+        "application/vnd.comicbook+zip": "zip",  # Comic book archive (Zip)
+        "application/vnd.ms-cab-compressed": "cab",  # Microsoft Cabinet archive
+        "application/vnd.palm": "palm",  # Palm OS data
+        "application/vnd.rar": "rar",  # RAR archive
+        "application/x-7z-compressed": "7z",  # 7-Zip archive
+        "application/x-ace": "ace",  # ACE archive
+        "application/x-alz": "alz",  # ALZip archive
+        "application/x-arc": "arc",  # ARC archive
+        "application/x-archive": "ar",  # Unix archive
+        "application/x-bzip2": "bzip2",  # Bzip2 compressed file
+        "application/x-compress": "compress",  # Unix compress file
+        "application/x-cpio": "cpio",  # CPIO archive
+        "application/x-gzip": "gzip",  # Gzip compressed file
+        "application/x-itunes-ipa": "zip",  # iOS application archive (IPA)
+        "application/x-java-pack200": "pack200",  # Java Pack200 archive
+        "application/x-lha": "lha",  # LHA archive
+        "application/x-lrzip": "lrzip",  # Long Range ZIP
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/x-lz4": "lz4",  # LZ4 compressed file
+        "application/x-lzip": "lzip",  # Lzip compressed file
+        "application/x-lzma": "lzma",  # LZMA compressed file
+        "application/x-par2": "par2",  # PAR2 recovery file
+        "application/x-qpress": "qpress",  # Qpress archive
+        "application/x-rar-compressed": "rar",  # RAR archive
+        "application/x-sit": "sit",  # StuffIt archive
+        "application/x-stuffit": "sit",  # StuffIt archive
+        "application/x-tar": "tar",  # Tar archive
+        "application/x-tgz": "tgz",  # Gzip compressed Tar archive
+        "application/x-webarchive": "zip",  # Web archive (Zip)
+        "application/x-xar": "xar",  # XAR archive
+        "application/x-xz": "xz",  # XZ compressed file
+        "application/x-zip-compressed-fb2": "zip",  # Zip archive (FB2)
+        "application/x-zoo": "zoo",  # Zoo archive
+        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
+        "application/zip": "zip",  # Zip archive
+        "application/zstd": "zstd",  # Zstandard compressed file
+    }
+
+    return compression_map.get(mime_type, "")