bbot 2.1.2.5156rc0__tar.gz → 2.3.0.5328rc0__tar.gz

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.1.2.5156rc0
+Version: 2.3.0.5328rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0
-Keywords: python,cli,automation,osint,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
+Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
 Author: TheTechromancer
 Requires-Python: >=3.9,<4.0
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -20,7 +20,7 @@ Requires-Dist: ansible (>=7.3,<9.0)
 Requires-Dist: ansible-runner (>=2.3.2,<3.0.0)
 Requires-Dist: beautifulsoup4 (>=4.12.2,<5.0.0)
 Requires-Dist: cachetools (>=5.3.2,<6.0.0)
-Requires-Dist: cloudcheck (>=5.0.0.350,<6.0.0.0)
+Requires-Dist: cloudcheck (>=6.0.0.602,<7.0.0.0)
 Requires-Dist: deepdiff (>=6.2.3,<8.0.0)
 Requires-Dist: dnspython (>=2.4.2,<3.0.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0)
@@ -30,11 +30,12 @@ Requires-Dist: lxml (>=4.9.2,<6.0.0)
 Requires-Dist: mmh3 (>=4.1,<6.0)
 Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
 Requires-Dist: psutil (>=5.9.4,<7.0.0)
+Requires-Dist: puremagic (>=1.28,<2.0)
 Requires-Dist: pycryptodome (>=3.17,<4.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pyjwt (>=2.7.0,<3.0.0)
 Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
-Requires-Dist: radixtarget (>=1.0.0.15,<2.0.0.0)
+Requires-Dist: radixtarget (>=2.0.0.50,<3.0.0.0)
 Requires-Dist: regex (>=2024.4.16,<2025.0.0)
 Requires-Dist: setproctitle (>=1.3.3,<2.0.0)
 Requires-Dist: socksio (>=1.0.0,<2.0.0)
@@ -144,6 +145,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row
@@ -410,6 +415,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
     - **Modules**
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
+        - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/README.md

@@ -91,6 +91,10 @@ description: Recursive web spider
 modules:
   - httpx
 
+blacklist:
+  # Prevent spider from invalidating sessions by logging out
+  - "RE:/.*(sign|log)[_-]?out"
+
 config:
   web:
     # how many links to follow in a row
@@ -357,6 +361,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
     - **Modules**
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
+        - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.1.2.5156rc"
+__version__ = "v2.3.0.5328rc"
 
 from .scanner import Scanner, Preset

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/cli.py

@@ -174,7 +174,7 @@ async def _main():
             if sys.stdin.isatty():
 
                 # warn if any targets belong directly to a cloud provider
-                for event in scan.target.events:
+                for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":
                         cloudcheck_result = scan.helpers.cloudcheck(event.host)
                         if cloudcheck_result:

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/config/logger.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import atexit
 import logging
@@ -9,6 +10,7 @@ from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
+from ..multiprocess import SHARED_INTERPRETER_STATE
 
 
 debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
@@ -65,8 +67,9 @@ class BBOTLogger:
 
         self.listener = None
 
-        self.process_name = multiprocessing.current_process().name
-        if self.process_name == "MainProcess":
+        # if we haven't set up logging yet, do it now
+        if not "_BBOT_LOGGING_SETUP" in os.environ:
+            os.environ["_BBOT_LOGGING_SETUP"] = "1"
             self.queue = multiprocessing.Queue()
             self.setup_queue_handler()
             # Start the QueueListener
@@ -113,7 +116,7 @@ class BBOTLogger:
 
         self.core_logger.setLevel(log_level)
         # disable asyncio logging for child processes
-        if self.process_name != "MainProcess":
+        if not SHARED_INTERPRETER_STATE.is_main_process:
             logging.getLogger("asyncio").setLevel(logging.ERROR)
 
     def addLoggingLevel(self, levelName, levelNum, methodName=None):

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/core.py

@@ -6,6 +6,7 @@ from contextlib import suppress
 from omegaconf import OmegaConf
 
 from bbot.errors import BBOTError
+from .multiprocess import SHARED_INTERPRETER_STATE
 
 
 DEFAULT_CONFIG = None
@@ -41,9 +42,23 @@ class BBOTCore:
         self.logger
         self.log = logging.getLogger("bbot.core")
 
+        self._prep_multiprocessing()
+
+    def _prep_multiprocessing(self):
         import multiprocessing
+        from .helpers.process import BBOTProcess
+
+        if SHARED_INTERPRETER_STATE.is_main_process:
+            # if this is the main bbot process, set the logger and queue for the first time
+            from functools import partialmethod
 
-        self.process_name = multiprocessing.current_process().name
+            BBOTProcess.__init__ = partialmethod(
+                BBOTProcess.__init__, log_level=self.logger.log_level, log_queue=self.logger.queue
+            )
+
+        # this makes our process class the default for process pools, etc.
+        mp_context = multiprocessing.get_context("spawn")
+        mp_context.Process = BBOTProcess
 
     @property
     def home(self):
@@ -187,12 +202,14 @@ class BBOTCore:
         if os.environ.get("BBOT_TESTING", "") == "True":
             process = self.create_thread(*args, **kwargs)
         else:
-            if self.process_name == "MainProcess":
+            if SHARED_INTERPRETER_STATE.is_scan_process:
                 from .helpers.process import BBOTProcess
 
                 process = BBOTProcess(*args, **kwargs)
             else:
-                raise BBOTError(f"Tried to start server from process {self.process_name}")
+                import multiprocessing
+
+                raise BBOTError(f"Tried to start server from process {multiprocessing.current_process().name}")
         process.daemon = True
         return process
 

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/engine.py

@@ -10,6 +10,7 @@ import traceback
 import contextlib
 import contextvars
 import zmq.asyncio
+import multiprocessing
 from pathlib import Path
 from concurrent.futures import CancelledError
 from contextlib import asynccontextmanager, suppress
@@ -17,6 +18,7 @@ from contextlib import asynccontextmanager, suppress
 from bbot.core import CORE
 from bbot.errors import BBOTEngineError
 from bbot.core.helpers.async_helpers import get_event_loop
+from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
 from bbot.core.helpers.misc import rand_string, in_exception_chain
 
 
@@ -264,10 +266,8 @@ class EngineClient(EngineBase):
         return [s for s in self.CMDS if isinstance(s, str)]
 
     def start_server(self):
-        import multiprocessing
-
         process_name = multiprocessing.current_process().name
-        if process_name == "MainProcess":
+        if SHARED_INTERPRETER_STATE.is_scan_process:
             kwargs = dict(self.server_kwargs)
             # if we're in tests, we use a single event loop to avoid weird race conditions
             # this allows us to more easily mock http, etc.
@@ -641,7 +641,7 @@ class EngineServer(EngineBase):
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):
                     self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})")
-                    for task in tasks:
+                    for task in list(tasks):
                         task.cancel()
                         self._await_cancelled_task(task)
                 else:
@@ -683,5 +683,5 @@ class EngineServer(EngineBase):
         for client_id in list(self.tasks):
             await self.cancel_task(client_id)
         for client_id, tasks in self.child_tasks.items():
-            for task in tasks:
+            for task in list(tasks):
                 await self._await_cancelled_task(task)

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/event/base.py

@@ -341,6 +341,21 @@ class BaseEvent:
             return self.host
         return self._host_original
 
+    @property
+    def host_filterable(self):
+        """
+        A string version of the event that's used for regex-based blacklisting.
+
+        For example, the user can specify "REGEX:.*.evilcorp.com" in their blacklist, and this regex
+        will be applied against this property.
+        """
+        parsed_url = getattr(self, "parsed_url", None)
+        if parsed_url is not None:
+            return parsed_url.geturl()
+        if self.host is not None:
+            return str(self.host)
+        return ""
+
     @property
     def port(self):
         self.host
@@ -503,12 +518,13 @@ class BaseEvent:
             for t in list(self.tags):
                 if t.startswith("distance-"):
                     self.remove_tag(t)
-            if scope_distance == 0:
-                self.add_tag("in-scope")
-                self.remove_tag("affiliate")
-            else:
-                self.remove_tag("in-scope")
-                self.add_tag(f"distance-{new_scope_distance}")
+            if self.host:
+                if scope_distance == 0:
+                    self.add_tag("in-scope")
+                    self.remove_tag("affiliate")
+                else:
+                    self.remove_tag("in-scope")
+                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
@@ -1002,13 +1018,15 @@ class ClosestHostEvent(DictHostEvent):
                     if parent_url is not None:
                         self.data["url"] = parent_url.geturl()
                 # inherit closest path
-                if not "path" in self.data and isinstance(parent.data, dict):
+                if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE":
                     parent_path = parent.data.get("path", None)
                     if parent_path is not None:
                         self.data["path"] = parent_path
                 # inherit closest host
                 if parent.host:
                     self.data["host"] = str(parent.host)
+                    # we do this to refresh the hash
+                    self.data = self.data
                     break
         # die if we still haven't found a host
         if not self.host:
@@ -1018,20 +1036,21 @@ class ClosestHostEvent(DictHostEvent):
 class DictPathEvent(DictEvent):
     def sanitize_data(self, data):
         new_data = dict(data)
+        new_data["path"] = str(new_data["path"])
         file_blobs = getattr(self.scan, "_file_blobs", False)
         folder_blobs = getattr(self.scan, "_folder_blobs", False)
         blob = None
         try:
-            data_path = Path(data["path"])
-            if data_path.is_file():
+            self._data_path = Path(data["path"])
+            if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
-                    with open(data_path, "rb") as file:
+                    with open(self._data_path, "rb") as file:
                         blob = file.read()
-            elif data_path.is_dir():
+            elif self._data_path.is_dir():
                 self.add_tag("folder")
                 if folder_blobs:
-                    blob = self._tar_directory(data_path)
+                    blob = self._tar_directory(self._data_path)
         except KeyError:
             pass
         if blob:
@@ -1110,8 +1129,7 @@ class DnsEvent(BaseEvent):
 class IP_RANGE(DnsEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        net = ipaddress.ip_network(self.data, strict=False)
-        self.add_tag(f"ipv{net.version}")
+        self.add_tag(f"ipv{self.host.version}")
 
     def sanitize_data(self, data):
         return str(ipaddress.ip_network(str(data), strict=False))
@@ -1540,7 +1558,25 @@ class WAF(DictHostEvent):
 
 
 class FILESYSTEM(DictPathEvent):
-    pass
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self._data_path.is_file():
+            # detect type of file content using magic
+            from bbot.core.helpers.libmagic import get_magic_info, get_compression
+
+            extension, mime_type, description, confidence = get_magic_info(self.data["path"])
+            self.data["magic_extension"] = extension
+            self.data["magic_mime_type"] = mime_type
+            self.data["magic_description"] = description
+            self.data["magic_confidence"] = confidence
+            # detection compression
+            compression = get_compression(mime_type)
+            if compression:
+                self.add_tag("compressed")
+                self.add_tag(f"{compression}-archive")
+                self.data["compression"] = compression
+            # refresh hash
+            self.data = self.data
 
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
@@ -1621,23 +1657,23 @@ def make_event(
     tags = set(tags)
 
     if is_event(data):
-        data = copy(data)
-        if scan is not None and not data.scan:
-            data.scan = scan
-        if scans is not None and not data.scans:
-            data.scans = scans
+        event = copy(data)
+        if scan is not None and not event.scan:
+            event.scan = scan
+        if scans is not None and not event.scans:
+            event.scans = scans
         if module is not None:
-            data.module = module
+            event.module = module
         if parent is not None:
-            data.parent = parent
+            event.parent = parent
         if context is not None:
-            data.discovery_context = context
+            event.discovery_context = context
         if internal == True:
-            data.internal = True
+            event.internal = True
         if tags:
-            data.tags = tags.union(data.tags)
+            event.tags = tags.union(event.tags)
         event_type = data.type
-        return data
+        return event
     else:
         if event_type is None:
             event_type, data = get_event_type(data)
@@ -1667,6 +1703,13 @@ def make_event(
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
             tags.add("affiliate")
+        # Convert single-host IP_RANGE to IP_ADDRESS
+        if event_type == "IP_RANGE":
+            with suppress(Exception):
+                net = ipaddress.ip_network(data, strict=False)
+                if net.prefixlen == net.max_prefixlen:
+                    event_type = "IP_ADDRESS"
+                    data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
 

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/bloom.py

@@ -64,8 +64,15 @@ class BloomFilter:
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def __del__(self):
+    def close(self):
+        """Explicitly close the memory-mapped file."""
         self.mmap_file.close()
 
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
     def __contains__(self, item):
         return self.check(item)

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/command.py

@@ -210,9 +210,10 @@ async def _write_proc_line(proc, chunk):
         return True
     except Exception as e:
         proc_args = [str(s) for s in getattr(proc, "args", [])]
-        command = " ".join(proc_args)
-        log.warning(f"Error writing line to stdin for command: {command}: {e}")
-        log.trace(traceback.format_exc())
+        command = " ".join(proc_args).strip()
+        if command:
+            log.warning(f"Error writing line to stdin for command: {command}: {e}")
+            log.trace(traceback.format_exc())
         return False
 
 

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -14,7 +14,7 @@ from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform, rm_at_exit
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -44,6 +44,11 @@ class DepsInstaller:
         self.parent_helper.mkdir(self.command_status)
         self.setup_status = self.read_setup_status()
 
+        # make sure we're using a minimal git config
+        self.minimal_git_config = self.data_dir / "minimal_git.config"
+        self.minimal_git_config.touch()
+        os.environ["GIT_CONFIG_GLOBAL"] = str(self.minimal_git_config)
+
         self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower()
         self.ansible_debug = self.core.logger.log_level <= logging.DEBUG
         self.venv = ""
@@ -171,10 +176,13 @@ class DepsInstaller:
 
         command = [sys.executable, "-m", "pip", "install", "--upgrade"] + packages
 
-        if constraints:
-            constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
-            command.append("--constraint")
-            command.append(constraints_tempfile)
+        # if no custom constraints are provided, use the constraints of the currently installed version of bbot
+        if constraints is not None:
+            constraints = get_python_constraints()
+
+        constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
+        command.append("--constraint")
+        command.append(constraints_tempfile)
 
         process = None
         try:
@@ -350,6 +358,7 @@ class DepsInstaller:
             "make": "make",
             "gcc": "gcc",
             "bash": "bash",
+            "which": "which",
         }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/dns/helpers.py

@@ -1,6 +1,6 @@
 import logging
 
-from bbot.core.helpers.regexes import dns_name_regex
+from bbot.core.helpers.regexes import dns_name_extraction_regex
 from bbot.core.helpers.misc import clean_dns_record, smart_decode
 
 log = logging.getLogger("bbot.core.helpers.dns")
@@ -198,7 +198,7 @@ def extract_targets(record):
     elif rdtype == "TXT":
         for s in record.strings:
             s = smart_decode(s)
-            for match in dns_name_regex.finditer(s):
+            for match in dns_name_extraction_regex.finditer(s):
                 start, end = match.span()
                 host = s[start:end]
                 add_result(rdtype, host)

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/helper.py

@@ -12,10 +12,11 @@ from .diff import HttpCompare
 from .regex import RegexHelper
 from .wordcloud import WordCloud
 from .interactsh import Interactsh
-from ...scanner.target import Target
 from .depsinstaller import DepsInstaller
 from .async_helpers import get_event_loop
 
+from bbot.scanner.target import BaseTarget
+
 log = logging.getLogger("bbot.core.helpers")
 
 
@@ -155,8 +156,8 @@ class ConfigAwareHelper:
         _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events, **kwargs):
-        return Target(*events, **kwargs)
+    def make_target(self, *targets, **kwargs):
+        return BaseTarget(*targets, scan=self.scan, **kwargs)
 
     @property
     def config(self):

bbot-2.3.0.5328rc0/bbot/core/helpers/libmagic.py

@@ -0,0 +1,68 @@
+import puremagic
+
+
+def get_magic_info(file):
+
+    magic_detections = puremagic.magic_file(file)
+    if magic_detections:
+        magic_detections.sort(key=lambda x: x.confidence, reverse=True)
+        detection = magic_detections[0]
+        return detection.extension, detection.mime_type, detection.name, detection.confidence
+    return "", "", "", 0
+
+
+def get_compression(mime_type):
+    mime_type = mime_type.lower()
+    # from https://github.com/cdgriffith/puremagic/blob/master/puremagic/magic_data.json
+    compression_map = {
+        "application/gzip": "gzip",  # Gzip compressed file
+        "application/zip": "zip",  # Zip archive
+        "application/x-bzip2": "bzip2",  # Bzip2 compressed file
+        "application/x-xz": "xz",  # XZ compressed file
+        "application/x-7z-compressed": "7z",  # 7-Zip archive
+        "application/vnd.rar": "rar",  # RAR archive
+        "application/x-lzma": "lzma",  # LZMA compressed file
+        "application/x-compress": "compress",  # Unix compress file
+        "application/zstd": "zstd",  # Zstandard compressed file
+        "application/x-lz4": "lz4",  # LZ4 compressed file
+        "application/x-tar": "tar",  # Tar archive
+        "application/x-zip-compressed-fb2": "zip",  # Zip archive (FB2)
+        "application/epub+zip": "zip",  # EPUB book (Zip archive)
+        "application/pak": "pak",  # PAK archive
+        "application/x-lha": "lha",  # LHA archive
+        "application/arj": "arj",  # ARJ archive
+        "application/vnd.ms-cab-compressed": "cab",  # Microsoft Cabinet archive
+        "application/x-sit": "sit",  # StuffIt archive
+        "application/binhex": "binhex",  # BinHex encoded file
+        "application/x-lrzip": "lrzip",  # Long Range ZIP
+        "application/x-alz": "alz",  # ALZip archive
+        "application/x-tgz": "tgz",  # Gzip compressed Tar archive
+        "application/x-gzip": "gzip",  # Gzip compressed file
+        "application/x-lzip": "lzip",  # Lzip compressed file
+        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/vnd.comicbook+zip": "zip",  # Comic book archive (Zip)
+        "application/vnd.palm": "palm",  # Palm OS data
+        "application/fictionbook2+zip": "zip",  # FictionBook 2.0 (Zip)
+        "application/fictionbook3+zip": "zip",  # FictionBook 3.0 (Zip)
+        "application/x-cpio": "cpio",  # CPIO archive
+        "application/x-java-pack200": "pack200",  # Java Pack200 archive
+        "application/x-par2": "par2",  # PAR2 recovery file
+        "application/x-rar-compressed": "rar",  # RAR archive
+        "application/java-archive": "zip",  # Java Archive (JAR)
+        "application/x-webarchive": "zip",  # Web archive (Zip)
+        "application/vnd.android.package-archive": "zip",  # Android package (APK)
+        "application/x-itunes-ipa": "zip",  # iOS application archive (IPA)
+        "application/x-stuffit": "sit",  # StuffIt archive
+        "application/x-archive": "ar",  # Unix archive
+        "application/x-qpress": "qpress",  # Qpress archive
+        "application/x-xar": "xar",  # XAR archive
+        "application/x-ace": "ace",  # ACE archive
+        "application/x-zoo": "zoo",  # Zoo archive
+        "application/x-arc": "arc",  # ARC archive
+        "application/x-zstd-compressed-tar": "zstd",  # Zstandard compressed Tar archive
+        "application/x-lz4-compressed-tar": "lz4",  # LZ4 compressed Tar archive
+        "application/vnd.comicbook-rar": "rar",  # Comic book archive (RAR)
+    }
+
+    return compression_map.get(mime_type, "")

{bbot-2.1.2.5156rc0 → bbot-2.3.0.5328rc0}/bbot/core/helpers/misc.py

@@ -586,17 +586,18 @@ def is_dns_name(d, include_local=True):
     if include_local:
         if bbot_regexes.hostname_regex.match(d):
             return True
-    if bbot_regexes.dns_name_regex.match(d):
+    if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False
 
 
-def is_ip(d, version=None):
+def is_ip(d, version=None, include_network=False):
     """
     Checks if the given string or object represents a valid IP address.
 
     Args:
         d (str or ipaddress.IPvXAddress): The IP address to check.
+        include_network (bool, optional): Whether to include network types (IPv4Network or IPv6Network). Defaults to False.
         version (int, optional): The IP version to validate (4 or 6). Default is None.
 
     Returns:
@@ -612,12 +613,17 @@ def is_ip(d, version=None):
         >>> is_ip('evilcorp.com')
         False
     """
+    ip = None
     try:
         ip = ipaddress.ip_address(d)
-        if version is None or ip.version == version:
-            return True
     except Exception:
-        pass
+        if include_network:
+            try:
+                ip = ipaddress.ip_network(d, strict=False)
+            except Exception:
+                pass
+    if ip is not None and (version is None or ip.version == version):
+        return True
     return False
 
 
@@ -2807,3 +2813,21 @@ def safe_format(s, **kwargs):
     Format string while ignoring unused keys (prevents KeyError)
     """
     return s.format_map(SafeDict(kwargs))
+
+
+def get_python_constraints():
+    req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE)
+
+    def clean_requirement(req_string):
+        # Extract package name and version constraints from format like "package (>=1.0,<2.0)"
+        match = req_regex.match(req_string)
+        if match:
+            name, constraints = match.groups()
+            return f"{name.strip()}{constraints}"
+
+        return req_string
+
+    from importlib.metadata import distribution
+
+    dist = distribution("bbot")
+    return [clean_requirement(r) for r in dist.requires]