bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/core/helpers/misc.py

@@ -227,13 +227,13 @@ def split_host_port(d):
 
     match = bbot_regexes.extract_open_port_regex.match(netloc)
     if match is None:
-        raise ValueError(f'split_port() failed to parse netloc "{netloc}"')
+        raise ValueError(f'split_port() failed to parse netloc "{netloc}" (original value: {d})')
 
     host = match.group(2)
     if host is None:
         host = match.group(1)
     if host is None:
-        raise ValueError(f'split_port() failed to locate host in netloc "{netloc}"')
+        raise ValueError(f'split_port() failed to locate host in netloc "{netloc}" (original value: {d})')
 
     port = match.group(3)
     if port is None and scheme is not None:
@@ -365,7 +365,7 @@ def parent_url(u):
     if path.parent == path:
         return None
     else:
-        return urlunparse(parsed._replace(path=str(path.parent)))
+        return urlunparse(parsed._replace(path=str(path.parent), query=""))
 
 
 def url_parents(u):
@@ -391,7 +391,7 @@ def url_parents(u):
     parent_list = []
     while 1:
         parent = parent_url(u)
-        if parent == None:
+        if parent is None:
             return parent_list
         elif parent not in parent_list:
             parent_list.append(parent)
@@ -512,7 +512,7 @@ def domain_stem(domain):
         - Utilizes the `tldextract` function for domain parsing.
     """
     parsed = tldextract(str(domain))
-    return f".".join(parsed.subdomain.split(".") + parsed.domain.split(".")).strip(".")
+    return ".".join(parsed.subdomain.split(".") + parsed.domain.split(".")).strip(".")
 
 
 def ip_network_parents(i, include_self=False):
@@ -586,17 +586,18 @@ def is_dns_name(d, include_local=True):
     if include_local:
         if bbot_regexes.hostname_regex.match(d):
             return True
-    if bbot_regexes.dns_name_regex.match(d):
+    if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False
 
 
-def is_ip(d, version=None):
+def is_ip(d, version=None, include_network=False):
     """
     Checks if the given string or object represents a valid IP address.
 
     Args:
         d (str or ipaddress.IPvXAddress): The IP address to check.
+        include_network (bool, optional): Whether to include network types (IPv4Network or IPv6Network). Defaults to False.
         version (int, optional): The IP version to validate (4 or 6). Default is None.
 
     Returns:
@@ -612,21 +613,27 @@ def is_ip(d, version=None):
         >>> is_ip('evilcorp.com')
         False
     """
+    ip = None
     try:
         ip = ipaddress.ip_address(d)
-        if version is None or ip.version == version:
-            return True
     except Exception:
-        pass
+        if include_network:
+            try:
+                ip = ipaddress.ip_network(d, strict=False)
+            except Exception:
+                pass
+    if ip is not None and (version is None or ip.version == version):
+        return True
     return False
 
 
-def is_ip_type(i):
+def is_ip_type(i, network=None):
     """
     Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module.
 
     Args:
         i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check.
+        network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False.
 
     Returns:
         bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise.
@@ -639,6 +646,12 @@ def is_ip_type(i):
         >>> is_ip_type("192.168.1.0/24")
         False
     """
+    if network is not None:
+        is_network = ipaddress._BaseNetwork in i.__class__.__mro__
+        if network:
+            return is_network
+        else:
+            return not is_network
     return ipaddress._IPAddressBase in i.__class__.__mro__
 
 
@@ -908,12 +921,12 @@ def extract_params_xml(xml_data, compare_mode="getparam"):
 
 # Define valid characters for each mode based on RFCs
 valid_chars_dict = {
-    "header": set(
+    "header": {
         chr(c) for c in range(33, 127) if chr(c) in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"
-    ),
-    "getparam": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="),
-    "postparam": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="),
-    "cookie": set(chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:"/[]?={} \t'),
+    },
+    "getparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
+    "postparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
+    "cookie": {chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:"/[]?={} \t'},
 }
 
 
@@ -1135,7 +1148,7 @@ def chain_lists(
     """
     if isinstance(l, str):
         l = [l]
-    final_list = dict()
+    final_list = {}
     for entry in l:
         for s in split_regex.split(entry):
             f = s.strip()
@@ -1260,7 +1273,7 @@ def gen_numbers(n, padding=2):
     return results
 
 
-def make_netloc(host, port):
+def make_netloc(host, port=None):
     """Constructs a network location string from a given host and port.
 
     Args:
@@ -1289,7 +1302,7 @@ def make_netloc(host, port):
     if is_ip(host, version=6):
         host = f"[{host}]"
     if port is None:
-        return host
+        return str(host)
     return f"{host}:{port}"
 
 
@@ -1332,7 +1345,7 @@ def search_dict_by_key(key, d):
     if isinstance(d, dict):
         if key in d:
             yield d[key]
-        for k, v in d.items():
+        for v in d.values():
             yield from search_dict_by_key(key, v)
     elif isinstance(d, list):
         for v in d:
@@ -1399,7 +1412,7 @@ def search_dict_values(d, *regexes):
                     results.add(h)
                     yield result
     elif isinstance(d, dict):
-        for _, v in d.items():
+        for v in d.values():
             yield from search_dict_values(v, *regexes)
     elif isinstance(d, list):
         for v in d:
@@ -2384,7 +2397,7 @@ def in_exception_chain(e, exc_types):
         ...     if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
         ...         raise
     """
-    return any([isinstance(_, exc_types) for _ in get_exception_chain(e)])
+    return any(isinstance(_, exc_types) for _ in get_exception_chain(e))
 
 
 def get_traceback_details(e):
@@ -2788,3 +2801,33 @@ def top_tcp_ports(n, as_string=False):
     if as_string:
         return ",".join([str(s) for s in top_ports])
     return top_ports
+
+
+class SafeDict(dict):
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+def safe_format(s, **kwargs):
+    """
+    Format string while ignoring unused keys (prevents KeyError)
+    """
+    return s.format_map(SafeDict(kwargs))
+
+
+def get_python_constraints():
+    req_regex = re.compile(r"([^(]+)\s*\((.*)\)", re.IGNORECASE)
+
+    def clean_requirement(req_string):
+        # Extract package name and version constraints from format like "package (>=1.0,<2.0)"
+        match = req_regex.match(req_string)
+        if match:
+            name, constraints = match.groups()
+            return f"{name.strip()}{constraints}"
+
+        return req_string
+
+    from importlib.metadata import distribution
+
+    dist = distribution("bbot")
+    return [clean_requirement(r) for r in dist.requires]

bbot/core/helpers/names_generator.py

@@ -10,9 +10,11 @@ adjectives = [
     "affectionate",
     "aggravated",
     "aggrieved",
+    "agoraphobic",
     "almighty",
     "anal",
     "atrocious",
+    "autistic",
     "awkward",
     "baby",
     "begrudged",
@@ -65,6 +67,7 @@ adjectives = [
     "dramatic",
     "drunk",
     "effeminate",
+    "effervescent",
     "elden",
     "eldritch",
     "embarrassed",
@@ -75,6 +78,7 @@ adjectives = [
     "ethereal",
     "euphoric",
     "evil",
+    "expired",
     "exquisite",
     "extreme",
     "ferocious",
@@ -87,10 +91,8 @@ adjectives = [
     "foreboding",
     "frenetic",
     "frolicking",
-    "frothy",
     "furry",
     "fuzzy",
-    "gay",
     "gentle",
     "giddy",
     "glowering",
@@ -112,6 +114,7 @@ adjectives = [
     "imaginary",
     "immense",
     "immoral",
+    "impulsive",
     "incomprehensible",
     "inebriated",
     "inexplicable",
@@ -149,6 +152,7 @@ adjectives = [
     "muscular",
     "mushy",
     "mysterious",
+    "nascent",
     "naughty",
     "nefarious",
     "negligent",
@@ -163,6 +167,7 @@ adjectives = [
     "overzealous",
     "paranoid",
     "pasty",
+    "peckish",
     "pedantic",
     "pernicious",
     "perturbed",
@@ -183,7 +188,6 @@ adjectives = [
     "psychic",
     "puffy",
     "pure",
-    "queer",
     "questionable",
     "rabid",
     "raging",
@@ -210,6 +214,7 @@ adjectives = [
     "sneaky",
     "soft",
     "sophisticated",
+    "spicy",
     "spiteful",
     "squishy",
     "steamy",
@@ -269,12 +274,14 @@ adjectives = [
     "wispy",
     "witty",
     "woolly",
+    "zesty",
 ]
 
 names = [
     "aaron",
     "abigail",
     "adam",
+    "adeem",
     "alan",
     "albert",
     "alex",
@@ -412,6 +419,7 @@ names = [
     "evelyn",
     "faramir",
     "florence",
+    "fox",
     "frances",
     "francis",
     "frank",
@@ -437,6 +445,7 @@ names = [
     "gregory",
     "gus",
     "hagrid",
+    "hank",
     "hannah",
     "harold",
     "harry",
@@ -447,6 +456,7 @@ names = [
     "hermione",
     "homer",
     "howard",
+    "hunter",
     "irene",
     "isaac",
     "isabella",
@@ -505,6 +515,7 @@ names = [
     "kevin",
     "kimberly",
     "kyle",
+    "kylie",
     "lantern",
     "larry",
     "laura",
@@ -528,6 +539,7 @@ names = [
     "lupin",
     "madison",
     "magnus",
+    "marcus",
     "margaret",
     "maria",
     "marie",
@@ -626,11 +638,13 @@ names = [
     "stephen",
     "steven",
     "susan",
+    "syrina",
     "tammy",
     "taylor",
     "teresa",
     "terry",
     "theoden",
+    "theon",
     "theresa",
     "thomas",
     "tiffany",

bbot/core/helpers/process.py

@@ -1,17 +1,12 @@
 import logging
 import traceback
 import threading
-import multiprocessing
 from multiprocessing.context import SpawnProcess
 
 from .misc import in_exception_chain
 
 
-current_process = multiprocessing.current_process()
-
-
 class BBOTThread(threading.Thread):
-
     default_name = "default bbot thread"
 
     def __init__(self, *args, **kwargs):
@@ -28,7 +23,6 @@ class BBOTThread(threading.Thread):
 
 
 class BBOTProcess(SpawnProcess):
-
     default_name = "bbot process pool"
 
     def __init__(self, *args, **kwargs):
@@ -57,17 +51,3 @@ class BBOTProcess(SpawnProcess):
             if not in_exception_chain(e, (KeyboardInterrupt,)):
                 log.warning(f"Error in {self.name}: {e}")
             log.trace(traceback.format_exc())
-
-
-if current_process.name == "MainProcess":
-    # if this is the main bbot process, set the logger and queue for the first time
-    from bbot.core import CORE
-    from functools import partialmethod
-
-    BBOTProcess.__init__ = partialmethod(
-        BBOTProcess.__init__, log_level=CORE.logger.log_level, log_queue=CORE.logger.queue
-    )
-
-# this makes our process class the default for process pools, etc.
-mp_context = multiprocessing.get_context("spawn")
-mp_context.Process = BBOTProcess

bbot/core/helpers/regex.py

@@ -41,7 +41,7 @@ class RegexHelper:
         """
         if not isinstance(compiled_regexes, dict):
             raise ValueError('compiled_regexes must be a dictionary like this: {"regex_name": <compiled_regex>}')
-        for k, v in compiled_regexes.items():
+        for v in compiled_regexes.values():
             self.ensure_compiled_regex(v)
 
         tasks = {}

bbot/core/helpers/regexes.py

@@ -23,7 +23,7 @@ num_regex = re.compile(r"\d+")
 _ipv4_regex = r"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
 ipv4_regex = re.compile(_ipv4_regex, re.I)
 
-# IPv6 is complicated, so we have accomodate alternative patterns,
+# IPv6 is complicated, so we have accommodate alternative patterns,
 # :(:[A-F0-9]{1,4}){1,7} == ::1, ::ffff:1
 # ([A-F0-9]{1,4}:){1,7}: == 2001::, 2001:db8::, 2001:db8:0:1:2:3::
 # ([A-F0-9]{1,4}:){1,6}:([A-F0-9]{1,4}) == 2001::1, 2001:db8::1, 2001:db8:0:1:2:3::1
@@ -36,11 +36,12 @@ _ip_range_regexes = (
     _ipv4_regex + r"\/[0-9]{1,2}",
     _ipv6_regex + r"\/[0-9]{1,3}",
 )
-ip_range_regexes = list(re.compile(r, re.I) for r in _ip_range_regexes)
+ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
 
 # dns names with periods
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
-dns_name_regex = re.compile(_dns_name_regex, re.I)
+dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
+dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
 # dns names without periods
 _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"
@@ -54,20 +55,23 @@ ptr_regex = re.compile(_ptr_regex)
 # uuid regex
 _uuid_regex = r"[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
 uuid_regex = re.compile(_uuid_regex, re.I)
+# event uuid regex
+_event_uuid_regex = r"[0-9A-Z_]+:[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
+event_uuid_regex = re.compile(_event_uuid_regex, re.I)
 
 _open_port_regexes = (
     _dns_name_regex + r":[0-9]{1,5}",
     _hostname_regex + r":[0-9]{1,5}",
     r"\[" + _ipv6_regex + r"\]:[0-9]{1,5}",
 )
-open_port_regexes = list(re.compile(r, re.I) for r in _open_port_regexes)
+open_port_regexes = [re.compile(r, re.I) for r in _open_port_regexes]
 
 _url_regexes = (
     r"https?://" + _dns_name_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
     r"https?://" + _hostname_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
     r"https?://\[" + _ipv6_regex + r"\](?::[0-9]{1,5})?(?:(?:/|\?).*)?",
 )
-url_regexes = list(re.compile(r, re.I) for r in _url_regexes)
+url_regexes = [re.compile(r, re.I) for r in _url_regexes]
 
 _double_slash_regex = r"/{2,}"
 double_slash_regex = re.compile(_double_slash_regex)
@@ -114,7 +118,7 @@ event_type_regexes = OrderedDict(
 scan_name_regex = re.compile(r"[a-z]{3,20}_[a-z]{3,20}")
 
 
-# For use with excavate paramaters extractor
+# For use with excavate parameters extractor
 input_tag_regex = re.compile(
     r"<input[^>]+?name=[\"\']?([\.$\w]+)[\"\']?(?:[^>]*?value=[\"\']([=+\/\w]*)[\"\'])?[^>]*>"
 )
@@ -152,3 +156,5 @@ extract_host_regex = re.compile(_extract_host_regex, re.I)
 # for use in recursive_decode()
 encoded_regex = re.compile(r"%[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\[ntrbv]")
 backslash_regex = re.compile(r"(?P<slashes>\\+)(?P<char>[ntrvb])")
+
+uuid_regex = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")

bbot/core/helpers/validators.py

@@ -132,7 +132,7 @@ def validate_host(host: Union[str, ipaddress.IPv4Address, ipaddress.IPv6Address]
 @validator
 def validate_severity(severity: str):
     severity = str(severity).strip().upper()
-    if not severity in ("UNKNOWN", "INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"):
+    if severity not in ("UNKNOWN", "INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"):
         raise ValueError(f"Invalid severity: {severity}")
     return severity
 
@@ -299,7 +299,6 @@ def is_email(email):
 
 
 class Validators:
-
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
 

bbot/core/helpers/web/client.py

@@ -56,7 +56,7 @@ class BBOTAsyncClient(httpx.AsyncClient):
 
         # timeout
         http_timeout = self._web_config.get("http_timeout", 20)
-        if not "timeout" in kwargs:
+        if "timeout" not in kwargs:
             kwargs["timeout"] = http_timeout
 
         # headers

bbot/core/helpers/web/engine.py

@@ -14,7 +14,6 @@ log = logging.getLogger("bbot.core.helpers.web.engine")
 
 
 class HTTPEngine(EngineServer):
-
     CMDS = {
         0: "request",
         1: "request_batch",
@@ -81,15 +80,20 @@ class HTTPEngine(EngineServer):
         if client_kwargs:
             client = self.AsyncClient(**client_kwargs)
 
-        async with self._acatch(url, raise_error):
-            if self.http_debug:
-                log.trace(f"Web request: {str(args)}, {str(kwargs)}")
-            response = await client.request(*args, **kwargs)
-            if self.http_debug:
-                log.trace(
-                    f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
-                )
-            return response
+        try:
+            async with self._acatch(url, raise_error):
+                if self.http_debug:
+                    log.trace(f"Web request: {str(args)}, {str(kwargs)}")
+                response = await client.request(*args, **kwargs)
+                if self.http_debug:
+                    log.trace(
+                        f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
+                    )
+                return response
+        except httpx.HTTPError as e:
+            if raise_error:
+                _response = getattr(e, "response", None)
+                return {"_request_error": str(e), "_response": _response}
 
     async def request_batch(self, urls, threads=10, **kwargs):
         async for (args, _, _), response in self.task_pool(
@@ -105,8 +109,8 @@ class HTTPEngine(EngineServer):
 
     async def download(self, url, **kwargs):
         warn = kwargs.pop("warn", True)
+        raise_error = kwargs.pop("raise_error", False)
         filename = kwargs.pop("filename")
-        raise_error = kwargs.get("raise_error", False)
         try:
             result = await self.stream_request(url, **kwargs)
             if result is None:
@@ -123,7 +127,8 @@ class HTTPEngine(EngineServer):
                 log_fn = log.warning
             log_fn(f"Failed to download {url}: {e}")
             if raise_error:
-                raise
+                _response = getattr(e, "response", None)
+                return {"_download_error": str(e), "_response": _response}
 
     async def stream_request(self, url, **kwargs):
         follow_redirects = kwargs.pop("follow_redirects", True)
@@ -132,7 +137,7 @@ class HTTPEngine(EngineServer):
         if max_size is not None:
             max_size = human_to_bytes(max_size)
         kwargs["follow_redirects"] = follow_redirects
-        if not "method" in kwargs:
+        if "method" not in kwargs:
             kwargs["method"] = "GET"
         try:
             total_size = 0