bbot 2.4.2.6109rc0__tar.gz → 2.4.2.6590rc0__tar.gz

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.2.6109rc0
+Version: 2.4.2.6590rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
@@ -438,6 +438,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
         - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
+        - [Lightfuzz](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/README.md

@@ -383,6 +383,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
         - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
+        - [Lightfuzz](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.2.6109rc"
+__version__ = "v2.4.2.6590rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/event/base.py

@@ -9,9 +9,9 @@ import datetime
 import ipaddress
 import traceback
 
-from copy import copy
 from pathlib import Path
 from typing import Optional
+from copy import copy, deepcopy
 from contextlib import suppress
 from radixtarget import RadixTarget
 from pydantic import BaseModel, field_validator
@@ -40,6 +40,7 @@ from bbot.core.helpers import (
     validators,
     get_file_extension,
 )
+from bbot.core.helpers.web.envelopes import BaseEnvelope
 
 
 log = logging.getLogger("bbot.core.event")
@@ -633,6 +634,10 @@ class BaseEvent:
         elif not self._dummy:
             log.warning(f"Tried to set invalid parent on {self}: (got: {repr(parent)} ({type(parent)}))")
 
+    @property
+    def children(self):
+        return []
+
     @property
     def parent_id(self):
         parent_id = getattr(self.get_parent(), "id", None)
@@ -687,6 +692,13 @@ class BaseEvent:
             e = parent
         return parents
 
+    def clone(self):
+        # Create a shallow copy of the event first
+        cloned_event = copy(self)
+        # Re-assign a new UUID
+        cloned_event._uuid = uuid.uuid4()
+        return cloned_event
+
     def _host(self):
         return ""
 
@@ -868,7 +880,13 @@ class BaseEvent:
         j["discovery_path"] = self.discovery_path
         j["parent_chain"] = self.parent_chain
 
+        # parameter envelopes
+        parameter_envelopes = getattr(self, "envelopes", None)
+        if parameter_envelopes is not None:
+            j["envelopes"] = parameter_envelopes.to_dict()
+
         # normalize non-primitive python objects
+
         for k, v in list(j.items()):
             if k == "data":
                 continue
@@ -1368,12 +1386,56 @@ class URL_HINT(URL_UNVERIFIED):
 
 
 class WEB_PARAMETER(DictHostEvent):
+    @property
+    def children(self):
+        # if we have any subparams, raise a new WEB_PARAMETER for each one
+        children = []
+        envelopes = getattr(self, "envelopes", None)
+        if envelopes is not None:
+            subparams = sorted(list(self.envelopes.get_subparams()))
+
+            if envelopes.selected_subparam is None:
+                current_subparam = subparams[0]
+                envelopes.selected_subparam = current_subparam[0]
+                if len(subparams) > 1:
+                    for subparam, _ in subparams[1:]:
+                        clone = self.clone()
+                        clone.envelopes = deepcopy(envelopes)
+                        clone.envelopes.selected_subparam = subparam
+                        clone.parent = self
+                        children.append(clone)
+        return children
+
+    def sanitize_data(self, data):
+        original_value = data.get("original_value", None)
+        if original_value is not None:
+            try:
+                envelopes = BaseEnvelope.detect(original_value)
+                setattr(self, "envelopes", envelopes)
+            except ValueError as e:
+                log.verbose(f"Error detecting envelopes for {self}: {e}")
+        return data
+
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")
         name = self.data.get("name", "")
         param_type = self.data.get("type", "")
-        return f"{url}:{name}:{param_type}"
+        envelopes = getattr(self, "envelopes", "")
+        subparam = getattr(envelopes, "selected_subparam", "")
+
+        return f"{url}:{name}:{param_type}:{subparam}"
+
+    def _outgoing_dedup_hash(self, event):
+        return hash(
+            (
+                str(event.host),
+                event.data["url"],
+                event.data.get("name", ""),
+                event.data.get("type", ""),
+                event.data.get("envelopes", ""),
+            )
+        )
 
     def _url(self):
         return self.data["url"]
@@ -1810,7 +1872,6 @@ def make_event(
                     data = net.network_address
 
         event_class = globals().get(event_type, DefaultEvent)
-
         return event_class(
             data,
             event_type=event_type,
@@ -1868,7 +1929,6 @@ def event_from_json(j, siem_friendly=False):
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
-
         event.timestamp = datetime.datetime.fromisoformat(j["timestamp"])
         event.scope_distance = j["scope_distance"]
         parent_id = j.get("parent", None)

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/diff.py

@@ -15,22 +15,24 @@ class HttpCompare:
         parent_helper,
         method="GET",
         data=None,
+        json=None,
         allow_redirects=False,
         include_cache_buster=True,
         headers=None,
         cookies=None,
-        timeout=15,
+        timeout=10,
     ):
         self.parent_helper = parent_helper
         self.baseline_url = baseline_url
         self.include_cache_buster = include_cache_buster
         self.method = method
         self.data = data
+        self.json = json
         self.allow_redirects = allow_redirects
         self._baselined = False
         self.headers = headers
         self.cookies = cookies
-        self.timeout = 15
+        self.timeout = 10
 
     @staticmethod
     def merge_dictionaries(headers1, headers2):
@@ -53,12 +55,13 @@ class HttpCompare:
                 follow_redirects=self.allow_redirects,
                 method=self.method,
                 data=self.data,
+                json=self.json,
                 headers=self.headers,
                 cookies=self.cookies,
                 retries=2,
                 timeout=self.timeout,
             )
-            await self.parent_helper.sleep(1)
+            await self.parent_helper.sleep(0.5)
             # put random parameters in URL, headers, and cookies
             get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}
 
@@ -76,12 +79,12 @@ class HttpCompare:
                 follow_redirects=self.allow_redirects,
                 method=self.method,
                 data=self.data,
+                json=self.json,
                 retries=2,
                 timeout=self.timeout,
             )
 
             self.baseline = baseline_1
-
             if baseline_1 is None or baseline_2 is None:
                 log.debug("HTTP error while establishing baseline, aborting")
                 raise HttpCompareError(
@@ -90,6 +93,7 @@ class HttpCompare:
             if baseline_1.status_code != baseline_2.status_code:
                 log.debug("Status code not stable during baseline, aborting")
                 raise HttpCompareError("Can't get baseline from source URL")
+
             try:
                 baseline_1_json = xmltodict.parse(baseline_1.text)
                 baseline_2_json = xmltodict.parse(baseline_2.text)
@@ -105,11 +109,9 @@ class HttpCompare:
 
             for k in ddiff.keys():
                 for x in list(ddiff[k]):
-                    log.debug(f"Added {k} filter for path: {x.path()}")
                     self.ddiff_filters.append(x.path())
 
             self.baseline_json = baseline_1_json
-
             self.baseline_ignore_headers = [
                 h.lower()
                 for h in [
@@ -167,7 +169,6 @@ class HttpCompare:
         if len(ddiff.keys()) == 0:
             return True
         else:
-            log.debug(ddiff)
             return False
 
     async def compare(
@@ -178,6 +179,7 @@ class HttpCompare:
         check_reflection=False,
         method="GET",
         data=None,
+        json=None,
         allow_redirects=False,
         timeout=None,
     ):
@@ -208,6 +210,7 @@ class HttpCompare:
             follow_redirects=allow_redirects,
             method=method,
             data=data,
+            json=json,
             timeout=timeout,
         )
 

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/helper.py

@@ -12,6 +12,7 @@ from .diff import HttpCompare
 from .regex import RegexHelper
 from .wordcloud import WordCloud
 from .interactsh import Interactsh
+from .yara_helper import YaraHelper
 from .depsinstaller import DepsInstaller
 from .async_helpers import get_event_loop
 
@@ -85,6 +86,7 @@ class ConfigAwareHelper:
         self._cloud = None
 
         self.re = RegexHelper(self)
+        self.yara = YaraHelper(self)
         self._dns = None
         self._web = None
         self.config_aware_validators = self.validators.Validators(self)
@@ -129,7 +131,8 @@ class ConfigAwareHelper:
         cookies=None,
         method="GET",
         data=None,
-        timeout=15,
+        json=None,
+        timeout=10,
     ):
         return HttpCompare(
             url,
@@ -141,6 +144,7 @@ class ConfigAwareHelper:
             timeout=timeout,
             method=method,
             data=data,
+            json=json,
         )
 
     def temp_filename(self, extension=None):

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/misc.py

@@ -2,6 +2,7 @@ import os
 import sys
 import copy
 import json
+import math
 import random
 import string
 import asyncio
@@ -9,6 +10,7 @@ import logging
 import ipaddress
 import regex as re
 import subprocess as sp
+
 from pathlib import Path
 from contextlib import suppress
 from unidecode import unidecode  # noqa F401
@@ -797,17 +799,14 @@ def recursive_decode(data, max_depth=5):
     return data
 
 
-rand_pool = string.ascii_lowercase
-rand_pool_digits = rand_pool + string.digits
-
-
-def rand_string(length=10, digits=True):
+def rand_string(length=10, digits=True, numeric_only=False):
     """
     Generates a random string of specified length.
 
     Args:
         length (int, optional): The length of the random string. Defaults to 10.
         digits (bool, optional): Whether to include digits in the string. Defaults to True.
+        numeric_only (bool, optional): Whether to generate a numeric-only string. Defaults to False.
 
     Returns:
         str: A random string of the specified length.
@@ -819,11 +818,17 @@ def rand_string(length=10, digits=True):
         'ap4rsdtg5iw7ey7y3oa5'
         >>> rand_string(30, digits=False)
         'xdmyxtglqfzqktngkesyulwbfrihva'
+        >>> rand_string(15, numeric_only=True)
+        '934857349857395'
     """
-    pool = rand_pool
-    if digits:
-        pool = rand_pool_digits
-    return "".join([random.choice(pool) for _ in range(int(length))])
+    if numeric_only:
+        pool = string.digits
+    elif digits:
+        pool = string.ascii_lowercase + string.digits
+    else:
+        pool = string.ascii_lowercase
+
+    return "".join(random.choice(pool) for _ in range(length))
 
 
 def truncate_string(s, n):
@@ -885,7 +890,7 @@ def extract_params_xml(xml_data, compare_mode="getparam"):
         xml_data (str): XML-formatted string containing elements.
 
     Returns:
-        set: A set of tuples containing the tags and their corresponding text values present in the XML object.
+        set: A set of tuples containing the tags and their corresponding sanitized text values present in the XML object.
 
     Raises:
         Returns an empty set if ParseError occurs.
@@ -907,7 +912,10 @@ def extract_params_xml(xml_data, compare_mode="getparam"):
     while stack:
         current_element = stack.pop()
         if validate_parameter(current_element.tag, compare_mode):
-            tag_value_pairs.add((current_element.tag, current_element.text))
+            # Sanitize the text value
+            text_value = current_element.text.strip() if current_element.text else None
+            sanitized_value = quote(text_value, safe="") if text_value else None
+            tag_value_pairs.add((current_element.tag, sanitized_value))
         for child in current_element:
             stack.append(child)
     return tag_value_pairs
@@ -921,6 +929,7 @@ valid_chars_dict = {
     "getparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
     "postparam": {chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="},
     "cookie": {chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:"/[]?={} \t'},
+    "bodyjson": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="),
 }
 
 
@@ -1874,6 +1883,7 @@ def make_table(rows, header, **kwargs):
         | row2      | row2      |
         +-----------+-----------+
     """
+
     from tabulate import tabulate
 
     # fix IndexError: list index out of range
@@ -2772,6 +2782,21 @@ def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None):
     return d
 
 
+def calculate_entropy(data):
+    """Calculate the Shannon entropy of a byte sequence"""
+    if not data:
+        return 0
+    frequency = {}
+    for byte in data:
+        if byte in frequency:
+            frequency[byte] += 1
+        else:
+            frequency[byte] = 1
+    data_len = len(data)
+    entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values())
+    return entropy
+
+
 top_ports_cache = None
 
 
@@ -2825,3 +2850,15 @@ def get_python_constraints():
 
     dist = distribution("bbot")
     return [clean_requirement(r) for r in dist.requires]
+
+
+def is_printable(s):
+    """
+    Check if a string is printable
+    """
+    if not isinstance(s, str):
+        raise ValueError(f"Expected a string, got {type(s)}")
+
+    # Exclude control characters that break display/printing
+    s = set(s)
+    return all(ord(c) >= 32 or c in "\t\n\r" for c in s)

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/regex.py

@@ -31,6 +31,10 @@ class RegexHelper:
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs)
 
+    async def match(self, compiled_regex, *args, **kwargs):
+        self.ensure_compiled_regex(compiled_regex)
+        return await self.parent_helper.run_in_executor(compiled_regex.match, *args, **kwargs)
+
     async def sub(self, compiled_regex, *args, **kwargs):
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.sub, *args, **kwargs)

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/regexes.py

@@ -114,27 +114,64 @@ scan_name_regex = re.compile(r"[a-z]{3,20}_[a-z]{3,20}")
 
 # For use with excavate parameters extractor
 input_tag_regex = re.compile(
-    r"<input[^>]+?name=[\"\']?([\.$\w]+)[\"\']?(?:[^>]*?value=[\"\']([=+\/\w]*)[\"\'])?[^>]*>"
+    r"<input[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?\svalue=[\"\']?([:%\-\._=+\/\w\s]*)[\"\']?[^>]*?>"
 )
-jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=")
-jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}")
+input_tag_regex2 = re.compile(
+    r"<input[^>]*?\svalue=[\"\']?([:\-%\._=+\/\w\s]*)[\"\']?[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>"
+)
+input_tag_novalue_regex = re.compile(r"<input(?![^>]*\b\svalue=)[^>]*?\sname=[\"\']?([\-\._=+\/\w]*)[\"\']?[^>]*?>")
+# jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=")
+# jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}")
+# jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}")
 a_tag_regex = re.compile(r"<a[^>]*href=[\"\']([^\"\'?>]*)\?([^&\"\'=]+)=([^&\"\'=]+)")
 img_tag_regex = re.compile(r"<img[^>]*src=[\"\']([^\"\'?>]*)\?([^&\"\'=]+)=([^&\"\'=]+)")
 get_form_regex = re.compile(
-    r"<form[^>]+(?:action=[\"']?([^\s\'\"]+)[\"\']?)?[^>]*method=[\"']?[gG][eE][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
+    r"<form[^>]*\bmethod=[\"']?[gG][eE][tT][\"']?[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*>([\s\S]*?)<\/form>",
+    re.DOTALL,
+)
+get_form_regex2 = re.compile(
+    r"<form[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*\bmethod=[\"']?[gG][eE][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
     re.DOTALL,
 )
 post_form_regex = re.compile(
-    r"<form[^>]+(?:action=[\"']?([^\s\'\"]+)[\"\']?)?[^>]*method=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
+    r"<form[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*>([\s\S]*?)<\/form>",
+    re.DOTALL,
+)
+post_form_regex2 = re.compile(
+    r"<form[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
+    re.DOTALL,
+)
+post_form_regex_noaction = re.compile(
+    r"<form[^>]*(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
     re.DOTALL,
 )
+generic_form_regex = re.compile(
+    r"<form(?![^>]*\bmethod=)[^>]+(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)[^>]*>([\s\S]*?)<\/form>",
+    re.IGNORECASE | re.DOTALL,
+)
+
 select_tag_regex = re.compile(
-    r"<select[^>]+?name=[\"\']?(\w+)[\"\']?[^>]*>(?:\s*<option[^>]*?value=[\"\'](\w*)[\"\']?[^>]*>)?"
+    r"<select[^>]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*<option[^>]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?",
+    re.IGNORECASE | re.DOTALL,
 )
+
 textarea_tag_regex = re.compile(
-    r'<textarea[^>]*\bname=["\']?(\w+)["\']?[^>]*>(.*?)</textarea>', re.IGNORECASE | re.DOTALL
+    r"<textarea[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?\svalue=[\"\']?([:%\-\._=+\/\w]*)[\"\']?[^>]*?>"
+)
+textarea_tag_regex2 = re.compile(
+    r"<textarea[^>]*?\svalue=[\"\']?([:\-%\._=+\/\w]*)[\"\']?[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>"
+)
+textarea_tag_novalue_regex = re.compile(
+    r'<textarea[^>]*\bname=["\']?([_\-\.\w]+)["\']?[^>]*>(.*?)</textarea>', re.IGNORECASE | re.DOTALL
+)
+
+button_tag_regex = re.compile(
+    r"<button[^>]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\-\._=+\/\w]*)[\"\']?[^>]*?>"
+)
+button_tag_regex2 = re.compile(
+    r"<button[^>]*?value=[\"\']?([\-%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>"
 )
-tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\s\'\"\>]+)[\"\']?[^>]*>")
+tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\'\"\>]+)[\"\']?[^>]*>")
 
 valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+"
 

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/url.py

@@ -32,7 +32,10 @@ def parse_url(url):
     return urlparse(url)
 
 
-def add_get_params(url, params):
+def add_get_params(url, params, encode=True):
+    def _no_encode_quote(s, safe="/", encoding=None, errors=None):
+        return s
+
     """
     Add or update query parameters to the given URL.
 
@@ -53,10 +56,23 @@ def add_get_params(url, params):
         >>> add_get_params('https://www.evilcorp.com?foo=1', {'foo': 2})
         ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='foo=2', fragment='')
     """
-    parsed = parse_url(url)
-    old_params = dict(parse_qs(parsed.query))
-    old_params.update(params)
-    return parsed._replace(query=urlencode(old_params, doseq=True))
+    parsed = urlparse(url)
+    query_params = parsed.query.split("&")
+
+    existing_params = {}
+    for param in query_params:
+        if "=" in param:
+            k, v = param.split("=", 1)
+            existing_params[k] = v
+
+    existing_params.update(params)
+
+    if encode:
+        new_query = urlencode(existing_params, doseq=True)
+    else:
+        new_query = urlencode(existing_params, doseq=True, quote_via=_no_encode_quote)
+
+    return parsed._replace(query=new_query)
 
 
 def get_get_params(url):

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/web/client.py

@@ -52,7 +52,7 @@ class BBOTAsyncClient(httpx.AsyncClient):
         if http_debug:
             log.trace(f"Creating AsyncClient: {args}, {kwargs}")
 
-        self._persist_cookies = kwargs.pop("persist_cookies", True)
+        self._persist_cookies = kwargs.pop("persist_cookies", False)
 
         # timeout
         http_timeout = self._web_config.get("http_timeout", 20)
@@ -63,11 +63,18 @@ class BBOTAsyncClient(httpx.AsyncClient):
         headers = kwargs.get("headers", None)
         if headers is None:
             headers = {}
+
+        # cookies
+        cookies = kwargs.get("cookies", None)
+        if cookies is None:
+            cookies = {}
+
         # user agent
         user_agent = self._web_config.get("user_agent", "BBOT")
         if "User-Agent" not in headers:
             headers["User-Agent"] = user_agent
         kwargs["headers"] = headers
+        kwargs["cookies"] = cookies
         # proxy
         proxies = self._web_config.get("http_proxy", None)
         kwargs["proxy"] = proxies
@@ -78,10 +85,23 @@ class BBOTAsyncClient(httpx.AsyncClient):
             self._cookies = DummyCookies()
 
     def build_request(self, *args, **kwargs):
-        request = super().build_request(*args, **kwargs)
-        # add custom headers if the URL is in-scope
-        # TODO: re-enable this
-        if self._target.in_scope(str(request.url)):
+        if args:
+            url = args[0]
+            kwargs["url"] = url
+        url = kwargs["url"]
+
+        target_in_scope = self._target.in_scope(str(url))
+
+        if target_in_scope:
+            if not kwargs.get("cookies", None):
+                kwargs["cookies"] = {}
+            for ck, cv in self._web_config.get("http_cookies", {}).items():
+                if ck not in kwargs["cookies"]:
+                    kwargs["cookies"][ck] = cv
+
+        request = super().build_request(**kwargs)
+
+        if target_in_scope:
             for hk, hv in self._web_config.get("http_headers", {}).items():
                 hv = str(hv)
                 # don't clobber headers

{bbot-2.4.2.6109rc0 → bbot-2.4.2.6590rc0}/bbot/core/helpers/web/engine.py

@@ -8,7 +8,7 @@ from socksio.exceptions import SOCKSError
 from contextlib import asynccontextmanager
 
 from bbot.core.engine import EngineServer
-from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain
+from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain, truncate_string
 
 log = logging.getLogger("bbot.core.helpers.web.engine")
 
@@ -203,6 +203,14 @@ class HTTPEngine(EngineServer):
             else:
                 log.trace(f"Error with request to URL: {url}: {e}")
                 log.trace(traceback.format_exc())
+        except httpx.InvalidURL as e:
+            if raise_error:
+                raise
+            else:
+                log.warning(
+                    f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(e, 200)}"
+                )
+                log.trace(traceback.format_exc())
         except ssl.SSLError as e:
             msg = f"SSL error with request to URL: {url}: {e}"
             if raise_error: