bbot 2.4.2.6109rc0__py3-none-any.whl → 2.4.2.6596rc0__py3-none-any.whl

bbot/scanner/preset/preset.py

@@ -308,7 +308,7 @@ class Preset(metaclass=BasePreset):
 
     @property
     def preset_dir(self):
-        return self.bbot_home / "presets"
+        return (self.bbot_home / "presets").expanduser().resolve()
 
     @property
     def default_output_modules(self):
@@ -413,30 +413,32 @@ class Preset(metaclass=BasePreset):
         self.log_debug("Getting baked")
         # create a copy of self
         baked_preset = copy(self)
-        baked_preset.scan = scan
+
         # copy core
         baked_preset.core = self.core.copy()
-        # copy module loader
-        baked_preset._module_loader = self.module_loader.copy()
-        # prepare os environment
-        os_environ = baked_preset.environ.prepare()
-        # find and replace preloaded modules with os environ
-        # this is different from the config variable substitution because it modifies
-        #  the preloaded modules, i.e. their ansible playbooks
-        baked_preset.module_loader.find_and_replace(**os_environ)
-        # update os environ
-        os.environ.clear()
-        os.environ.update(os_environ)
 
-        # validate flags, config options
-        baked_preset.validate()
+        if scan is not None:
+            baked_preset.scan = scan
+            # copy module loader
+            baked_preset._module_loader = self.module_loader.copy()
+            # prepare os environment
+            os_environ = baked_preset.environ.prepare()
+            # find and replace preloaded modules with os environ
+            # this is different from the config variable substitution because it modifies
+            #  the preloaded modules, i.e. their ansible playbooks
+            baked_preset.module_loader.find_and_replace(**os_environ)
+            # update os environ
+            os.environ.clear()
+            os.environ.update(os_environ)
+
+            # assign baked preset to our scan
+            scan.preset = baked_preset
 
         # validate log level options
         baked_preset.apply_log_level(apply_core=scan is not None)
 
-        # assign baked preset to our scan
-        if scan is not None:
-            scan.preset = baked_preset
+        # validate flags, config options
+        baked_preset.validate()
 
         # now that our requirements / exclusions are validated, we can start enabling modules
         # enable scan modules
@@ -483,15 +485,19 @@ class Preset(metaclass=BasePreset):
         from bbot.scanner.target import BBOTTarget
 
         baked_preset._target = BBOTTarget(
-            *list(self._seeds), whitelist=self._whitelist, blacklist=self._blacklist, strict_scope=self.strict_scope
+            *list(self._seeds),
+            whitelist=self._whitelist,
+            blacklist=self._blacklist,
+            strict_scope=self.strict_scope,
         )
 
-        # evaluate conditions
-        if baked_preset.conditions:
-            from .conditions import ConditionEvaluator
+        if scan is not None:
+            # evaluate conditions
+            if baked_preset.conditions:
+                from .conditions import ConditionEvaluator
 
-            evaluator = ConditionEvaluator(baked_preset)
-            evaluator.evaluate()
+                evaluator = ConditionEvaluator(baked_preset)
+                evaluator.evaluate()
 
         self._baked = True
         return baked_preset
@@ -562,6 +568,12 @@ class Preset(metaclass=BasePreset):
         return self.scope_config.get("strict", False)
 
     def apply_log_level(self, apply_core=False):
+        """
+        Apply the log level to the preset.
+
+        Args:
+            apply_core (bool, optional): If True, apply the log level to the core logger.
+        """
         # silent takes precedence
         if self.silent:
             self.verbose = False
@@ -920,20 +932,17 @@ class Preset(metaclass=BasePreset):
         """
         Recursively find all the presets and return them as a dictionary
         """
-        preset_dir = self.preset_dir
-        home_dir = Path.home()
-
         # first, add local preset dir to PRESET_PATH
         PRESET_PATH.add_path(self.preset_dir)
 
         # ensure local preset directory exists
-        mkdir(preset_dir)
+        mkdir(self.preset_dir)
 
         global DEFAULT_PRESETS
         if DEFAULT_PRESETS is None:
             presets = {}
-            for ext in ("yml", "yaml"):
-                for preset_path in PRESET_PATH:
+            for preset_path in PRESET_PATH:
+                for ext in ("yml", "yaml"):
                     # for every yaml file
                     for original_filename in preset_path.rglob(f"**/*.{ext}"):
                         # not including symlinks
@@ -957,18 +966,14 @@ class Preset(metaclass=BasePreset):
 
                         local_preset = original_filename
                         # populate symlinks in local preset dir
-                        if not original_filename.is_relative_to(preset_dir):
+                        if not original_filename.is_relative_to(self.preset_dir):
                             relative_preset = original_filename.relative_to(preset_path)
-                            local_preset = preset_dir / relative_preset
+                            local_preset = self.preset_dir / relative_preset
                             mkdir(local_preset.parent, check_writable=False)
                             if not local_preset.exists():
                                 local_preset.symlink_to(original_filename)
 
-                        # collapse home directory into "~"
-                        if local_preset.is_relative_to(home_dir):
-                            local_preset = Path("~") / local_preset.relative_to(home_dir)
-
-                        presets[local_preset] = (loaded_preset, category, preset_path, original_filename)
+                        presets[local_preset.stem] = (loaded_preset, category, preset_path, original_filename)
 
             # sort by name
             DEFAULT_PRESETS = dict(sorted(presets.items(), key=lambda x: x[-1][0].name))

bbot/scanner/scanner.py

@@ -214,6 +214,12 @@ class Scanner:
             self.warning(
                 "You have enabled custom HTTP headers. These will be attached to all in-scope requests and all requests made by httpx."
             )
+        # custom HTTP cookies warning
+        self.custom_http_cookies = self.web_config.get("http_cookies", {})
+        if self.custom_http_cookies:
+            self.warning(
+                "You have enabled custom HTTP cookies. These will be attached to all in-scope requests and all requests made by httpx."
+            )
 
         # url file extensions
         self.url_extension_blacklist = {e.lower() for e in self.config.get("url_extension_blacklist", [])}

bbot/scripts/docs.py

@@ -198,15 +198,15 @@ def update_docs():
     update_md_files("BBOT PRESETS", bbot_presets_table)
 
     # BBOT presets
-    for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
+    for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
         preset_yaml = f"""
-```yaml title={yaml_file.name}
+```yaml title={preset_path.name}
 {loaded_preset._yaml_str}
 ```
 """
         preset_yaml_expandable = f"""
 <details>
-<summary><b><code>{yaml_file.name}</code></b></summary>
+<summary><b><code>{preset_path.name}</code></b></summary>
 
 ```yaml
 {loaded_preset._yaml_str}
@@ -218,11 +218,11 @@ def update_docs():
         update_md_files(f"BBOT {loaded_preset.name.upper()} PRESET EXPANDABLE", preset_yaml_expandable)
 
     content = []
-    for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
+    for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
         yaml_str = loaded_preset._yaml_str
         indent = " " * 4
         yaml_str = f"\n{indent}".join(yaml_str.splitlines())
-        filename = homedir_collapseuser(yaml_file)
+        filename = homedir_collapseuser(preset_path)
 
         num_modules = len(loaded_preset.scan_modules)
         modules = ", ".join(sorted([f"`{m}`" for m in loaded_preset.scan_modules]))

bbot/test/test_step_1/test__module__tests.py

@@ -18,7 +18,7 @@ def test__module__tests():
     preset = Preset()
 
     # make sure each module has a .py file
-    for module_name in preset.module_loader.preloaded():
+    for module_name, preloaded in preset.module_loader.preloaded().items():
         module_name = module_name.lower()
         assert module_name in module_test_files, f'No test file found for module "{module_name}"'
 

bbot/test/test_step_1/test_helpers.py

@@ -460,6 +460,13 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     s = "asdf {unused} {used}"
     assert helpers.safe_format(s, used="fdsa") == "asdf {unused} fdsa"
 
+    # is_printable
+    assert helpers.is_printable("asdf") is True
+    assert helpers.is_printable(r"""~!@#$^&*()_+=-<>:"?,./;'[]\{}|""") is True
+    assert helpers.is_printable("ドメイン.テスト") is True
+    assert helpers.is_printable("4") is True
+    assert helpers.is_printable("asdf\x00") is False
+
     # punycode
     assert helpers.smart_encode_punycode("ドメイン.テスト") == "xn--eckwd4c7c.xn--zckzah"
     assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah") == "ドメイン.テスト"

bbot/test/test_step_1/test_presets.py

@@ -596,7 +596,7 @@ class TestModule1(BaseModule):
 from bbot.modules.output.base import BaseOutputModule
 
 class TestModule2(BaseOutputModule):
-    pass
+    watched_events = []
 """
         )
 
@@ -607,7 +607,7 @@ class TestModule2(BaseOutputModule):
 from bbot.modules.internal.base import BaseInternalModule
 
 class TestModule3(BaseInternalModule):
-    pass
+    watched_events = []
 """
         )
 

bbot/test/test_step_1/test_web.py

@@ -478,3 +478,23 @@ async def test_web_cookies(bbot_scanner, httpx_mock):
     assert not client2.cookies
 
     await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_http_sendcookies(bbot_scanner, bbot_httpserver):
+    endpoint = "/"
+    url = bbot_httpserver.url_for(endpoint)
+    from werkzeug.wrappers import Response
+
+    def echo_cookies_handler(request):
+        cookies = request.cookies
+        cookie_str = "; ".join([f"{key}={value}" for key, value in cookies.items()])
+        return Response(f"Echoed Cookies: {cookie_str}\nEchoed Headers: {request.headers}")
+
+    bbot_httpserver.expect_request(uri=endpoint).respond_with_handler(echo_cookies_handler)
+    scan1 = bbot_scanner("127.0.0.1", config={"web": {"debug": True}})
+    r1 = await scan1.helpers.request(url, cookies={"foo": "bar"})
+
+    assert r1 is not None, "Request to self-signed SSL server went through even with ssl_verify=True"
+    assert "bar" in r1.text
+    await scan1._cleanup()

bbot/test/test_step_1/test_web_envelopes.py

@@ -0,0 +1,343 @@
+import pytest
+
+
+async def test_web_envelopes():
+    from bbot.core.helpers.web.envelopes import (
+        BaseEnvelope,
+        TextEnvelope,
+        HexEnvelope,
+        B64Envelope,
+        JSONEnvelope,
+        XMLEnvelope,
+        URLEnvelope,
+    )
+
+    # simple text
+    text_envelope = BaseEnvelope.detect("foo")
+    assert isinstance(text_envelope, TextEnvelope)
+    assert text_envelope.unpacked_data() == "foo"
+    assert text_envelope.subparams == {"__default__": "foo"}
+    expected_subparams = [([], "foo")]
+    assert list(text_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert text_envelope.get_subparam(subparam) == value
+    assert text_envelope.pack() == "foo"
+    assert text_envelope.num_envelopes == 0
+    assert text_envelope.get_subparam() == "foo"
+    text_envelope.set_subparam(value="bar")
+    assert text_envelope.get_subparam() == "bar"
+    assert text_envelope.unpacked_data() == "bar"
+
+    # simple binary
+    # binary_envelope = BaseEnvelope.detect("foo\x00")
+    # assert isinstance(binary_envelope, BinaryEnvelope)
+    # assert binary_envelope.unpacked_data == "foo\x00"
+    # assert binary_envelope.packed_data == "foo\x00"
+    # assert binary_envelope.subparams == {"__default__": "foo\x00"}
+
+    # text encoded as hex
+    hex_envelope = BaseEnvelope.detect("706172616d")
+    assert isinstance(hex_envelope, HexEnvelope)
+    assert hex_envelope.unpacked_data(recursive=True) == "param"
+    hex_inner_envelope = hex_envelope.unpacked_data(recursive=False)
+    assert isinstance(hex_inner_envelope, TextEnvelope)
+    assert hex_inner_envelope.unpacked_data(recursive=False) == "param"
+    assert hex_inner_envelope.unpacked_data(recursive=True) == "param"
+    assert list(hex_envelope.get_subparams(recursive=False)) == [([], hex_inner_envelope)]
+    assert list(hex_envelope.get_subparams(recursive=True)) == [([], "param")]
+    assert hex_inner_envelope.unpacked_data() == "param"
+    assert hex_inner_envelope.subparams == {"__default__": "param"}
+    expected_subparams = [([], "param")]
+    assert list(hex_inner_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert hex_inner_envelope.get_subparam(subparam) == value
+    assert hex_envelope.pack() == "706172616d"
+    assert hex_envelope.num_envelopes == 1
+    assert hex_envelope.get_subparam() == "param"
+    hex_envelope.set_subparam(value="asdf")
+    assert hex_envelope.get_subparam() == "asdf"
+    assert hex_envelope.unpacked_data() == "asdf"
+    assert hex_envelope.pack() == "61736466"
+
+    # text encoded as base64
+    base64_envelope = BaseEnvelope.detect("cGFyYW0=")
+    assert isinstance(base64_envelope, B64Envelope)
+    assert base64_envelope.unpacked_data() == "param"
+    base64_inner_envelope = base64_envelope.unpacked_data(recursive=False)
+    assert isinstance(base64_inner_envelope, TextEnvelope)
+    assert list(base64_envelope.get_subparams(recursive=False)) == [([], base64_inner_envelope)]
+    assert list(base64_envelope.get_subparams()) == [([], "param")]
+    assert base64_inner_envelope.pack() == "param"
+    assert base64_inner_envelope.unpacked_data() == "param"
+    assert base64_inner_envelope.subparams == {"__default__": "param"}
+    expected_subparams = [([], "param")]
+    assert list(base64_inner_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert base64_inner_envelope.get_subparam(subparam) == value
+    assert base64_envelope.num_envelopes == 1
+    base64_envelope.set_subparam(value="asdf")
+    assert base64_envelope.get_subparam() == "asdf"
+    assert base64_envelope.unpacked_data() == "asdf"
+    assert base64_envelope.pack() == "YXNkZg=="
+
+    # test inside hex inside base64
+    hex_envelope = BaseEnvelope.detect("634746795957303d")
+    assert isinstance(hex_envelope, HexEnvelope)
+    assert hex_envelope.get_subparam() == "param"
+    assert hex_envelope.unpacked_data() == "param"
+    base64_envelope = hex_envelope.unpacked_data(recursive=False)
+    assert isinstance(base64_envelope, B64Envelope)
+    assert base64_envelope.get_subparam() == "param"
+    assert base64_envelope.unpacked_data() == "param"
+    text_envelope = base64_envelope.unpacked_data(recursive=False)
+    assert isinstance(text_envelope, TextEnvelope)
+    assert text_envelope.get_subparam() == "param"
+    assert text_envelope.unpacked_data() == "param"
+    hex_envelope.set_subparam(value="asdf")
+    assert hex_envelope.get_subparam() == "asdf"
+    assert hex_envelope.unpacked_data() == "asdf"
+    assert text_envelope.get_subparam() == "asdf"
+    assert text_envelope.unpacked_data() == "asdf"
+    assert base64_envelope.get_subparam() == "asdf"
+    assert base64_envelope.unpacked_data() == "asdf"
+
+    # URL-encoded text
+    url_encoded_envelope = BaseEnvelope.detect("a%20b%20c")
+    assert isinstance(url_encoded_envelope, URLEnvelope)
+    assert url_encoded_envelope.pack() == "a%20b%20c"
+    assert url_encoded_envelope.unpacked_data() == "a b c"
+    url_inner_envelope = url_encoded_envelope.unpacked_data(recursive=False)
+    assert isinstance(url_inner_envelope, TextEnvelope)
+    assert url_inner_envelope.unpacked_data(recursive=False) == "a b c"
+    assert url_inner_envelope.unpacked_data(recursive=True) == "a b c"
+    assert list(url_encoded_envelope.get_subparams(recursive=False)) == [([], url_inner_envelope)]
+    assert list(url_encoded_envelope.get_subparams(recursive=True)) == [([], "a b c")]
+    assert url_inner_envelope.pack() == "a b c"
+    assert url_inner_envelope.unpacked_data() == "a b c"
+    assert url_inner_envelope.subparams == {"__default__": "a b c"}
+    expected_subparams = [([], "a b c")]
+    assert list(url_inner_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert url_inner_envelope.get_subparam(subparam) == value
+    assert url_encoded_envelope.num_envelopes == 1
+    url_encoded_envelope.set_subparam(value="a s d f")
+    assert url_encoded_envelope.get_subparam() == "a s d f"
+    assert url_encoded_envelope.unpacked_data() == "a s d f"
+    assert url_encoded_envelope.pack() == "a%20s%20d%20f"
+
+    # json
+    json_envelope = BaseEnvelope.detect('{"param1": "val1", "param2": {"param3": "val3"}}')
+    assert isinstance(json_envelope, JSONEnvelope)
+    assert json_envelope.pack() == '{"param1": "val1", "param2": {"param3": "val3"}}'
+    assert json_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}}
+    assert json_envelope.unpacked_data(recursive=False) == {"param1": "val1", "param2": {"param3": "val3"}}
+    assert json_envelope.unpacked_data(recursive=True) == {"param1": "val1", "param2": {"param3": "val3"}}
+    assert json_envelope.subparams == {"param1": "val1", "param2": {"param3": "val3"}}
+    expected_subparams = [
+        (["param1"], "val1"),
+        (["param2", "param3"], "val3"),
+    ]
+    assert list(json_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert json_envelope.get_subparam(subparam) == value
+    json_envelope.selected_subparam = ["param2", "param3"]
+    assert json_envelope.get_subparam() == "val3"
+    assert json_envelope.num_envelopes == 1
+
+    # prevent json over-detection
+    just_a_string = BaseEnvelope.detect("10")
+    assert not isinstance(just_a_string, JSONEnvelope)
+
+    # xml
+    xml_envelope = BaseEnvelope.detect(
+        '<root><param1 attr="attr1">val1</param1><param2><param3>val3</param3></param2></root>'
+    )
+    assert isinstance(xml_envelope, XMLEnvelope)
+    assert (
+        xml_envelope.pack()
+        == '<?xml version="1.0" encoding="utf-8"?>\n<root><param1 attr="attr1">val1</param1><param2><param3>val3</param3></param2></root>'
+    )
+    assert xml_envelope.unpacked_data() == {
+        "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    assert xml_envelope.unpacked_data(recursive=False) == {
+        "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    assert xml_envelope.unpacked_data(recursive=True) == {
+        "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    assert xml_envelope.subparams == {
+        "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    expected_subparams = [
+        (["root", "param1", "@attr"], "attr1"),
+        (["root", "param1", "#text"], "val1"),
+        (["root", "param2", "param3"], "val3"),
+    ]
+    assert list(xml_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert xml_envelope.get_subparam(subparam) == value
+    assert xml_envelope.num_envelopes == 1
+
+    # json inside base64
+    base64_json_envelope = BaseEnvelope.detect("eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19")
+    assert isinstance(base64_json_envelope, B64Envelope)
+    assert base64_json_envelope.pack() == "eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19"
+    assert base64_json_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}}
+    base64_inner_envelope = base64_json_envelope.unpacked_data(recursive=False)
+    assert isinstance(base64_inner_envelope, JSONEnvelope)
+    assert base64_inner_envelope.pack() == '{"param1": "val1", "param2": {"param3": "val3"}}'
+    assert base64_inner_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}}
+    assert base64_inner_envelope.subparams == {"param1": "val1", "param2": {"param3": "val3"}}
+    expected_subparams = [
+        (["param1"], "val1"),
+        (["param2", "param3"], "val3"),
+    ]
+    assert list(base64_json_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert base64_json_envelope.get_subparam(subparam) == value
+    assert base64_json_envelope.num_envelopes == 2
+    with pytest.raises(ValueError):
+        assert base64_json_envelope.get_subparam()
+    base64_json_envelope.selected_subparam = ["param2", "param3"]
+    assert base64_json_envelope.get_subparam() == "val3"
+
+    # xml inside url inside hex inside base64
+    nested_xml_envelope = BaseEnvelope.detect(
+        "MjUzMzYzMjUzNzMyMjUzNjY2MjUzNjY2MjUzNzM0MjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMxMjUzMjMwMjUzNjMxMjUzNzM0MjUzNzM0MjUzNzMyMjUzMzY0MjUzMjMyMjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMxMjUzMjMyMjUzMzY1MjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMxMjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMxMjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMyMjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMzMjUzMzY1MjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMzMjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMzMjUzMzY1MjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMyMjUzMzY1MjUzMzYzMjUzMjY2MjUzNzMyMjUzNjY2MjUzNjY2MjUzNzM0MjUzMzY1"
+    )
+    assert isinstance(nested_xml_envelope, B64Envelope)
+    assert nested_xml_envelope.unpacked_data() == {
+        "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    assert (
+        nested_xml_envelope.pack()
+        == "MjUzMzQzMjUzMzQ2Nzg2ZDZjMjUzMjMwNzY2NTcyNzM2OTZmNmUyNTMzNDQyNTMyMzIzMTJlMzAyNTMyMzIyNTMyMzA2NTZlNjM2ZjY0Njk2ZTY3MjUzMzQ0MjUzMjMyNzU3NDY2MmQzODI1MzIzMjI1MzM0NjI1MzM0NTI1MzA0MTI1MzM0MzcyNmY2Zjc0MjUzMzQ1MjUzMzQzNzA2MTcyNjE2ZDMxMjUzMjMwNjE3NDc0NzIyNTMzNDQyNTMyMzI3NjYxNmMzMTI1MzIzMjI1MzM0NTc2NjE2YzMxMjUzMzQzMmY3MDYxNzI2MTZkMzEyNTMzNDUyNTMzNDM3MDYxNzI2MTZkMzIyNTMzNDUyNTMzNDM3MDYxNzI2MTZkMzMyNTMzNDU3NjYxNmMzMzI1MzM0MzJmNzA2MTcyNjE2ZDMzMjUzMzQ1MjUzMzQzMmY3MDYxNzI2MTZkMzIyNTMzNDUyNTMzNDMyZjcyNmY2Zjc0MjUzMzQ1"
+    )
+    inner_hex_envelope = nested_xml_envelope.unpacked_data(recursive=False)
+    assert isinstance(inner_hex_envelope, HexEnvelope)
+    assert (
+        inner_hex_envelope.pack()
+        == "253343253346786d6c25323076657273696f6e253344253232312e30253232253230656e636f64696e672533442532327574662d38253232253346253345253041253343726f6f74253345253343706172616d312532306174747225334425323276616c3125323225334576616c312533432f706172616d31253345253343706172616d32253345253343706172616d3325334576616c332533432f706172616d332533452533432f706172616d322533452533432f726f6f74253345"
+    )
+    inner_url_envelope = inner_hex_envelope.unpacked_data(recursive=False)
+    assert isinstance(inner_url_envelope, URLEnvelope)
+    assert (
+        inner_url_envelope.pack()
+        == r"%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22utf-8%22%3F%3E%0A%3Croot%3E%3Cparam1%20attr%3D%22val1%22%3Eval1%3C/param1%3E%3Cparam2%3E%3Cparam3%3Eval3%3C/param3%3E%3C/param2%3E%3C/root%3E"
+    )
+    inner_xml_envelope = inner_url_envelope.unpacked_data(recursive=False)
+    assert isinstance(inner_xml_envelope, XMLEnvelope)
+    assert (
+        inner_xml_envelope.pack()
+        == '<?xml version="1.0" encoding="utf-8"?>\n<root><param1 attr="val1">val1</param1><param2><param3>val3</param3></param2></root>'
+    )
+    assert inner_xml_envelope.unpacked_data() == {
+        "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    assert inner_xml_envelope.subparams == {
+        "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}}
+    }
+    expected_subparams = [
+        (["root", "param1", "@attr"], "val1"),
+        (["root", "param1", "#text"], "val1"),
+        (["root", "param2", "param3"], "val3"),
+    ]
+    assert list(nested_xml_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert nested_xml_envelope.get_subparam(subparam) == value
+    assert nested_xml_envelope.num_envelopes == 4
+
+    # manipulating text inside hex
+    hex_envelope = BaseEnvelope.detect("706172616d")
+    expected_subparams = [([], "param")]
+    assert list(hex_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert hex_envelope.get_subparam(subparam) == value
+    hex_envelope.set_subparam([], "asdf")
+    expected_subparams = [([], "asdf")]
+    assert list(hex_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert hex_envelope.get_subparam(subparam) == value
+    assert hex_envelope.unpacked_data() == "asdf"
+
+    # manipulating json inside base64
+    base64_json_envelope = BaseEnvelope.detect("eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19")
+    expected_subparams = [
+        (["param1"], "val1"),
+        (["param2", "param3"], "val3"),
+    ]
+    assert list(base64_json_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert base64_json_envelope.get_subparam(subparam) == value
+    base64_json_envelope.set_subparam(["param1"], {"asdf": [None], "fdsa": 1.0})
+    expected_subparams = [
+        (["param1", "asdf"], [None]),
+        (["param1", "fdsa"], 1.0),
+        (["param2", "param3"], "val3"),
+    ]
+    assert list(base64_json_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert base64_json_envelope.get_subparam(subparam) == value
+    base64_json_envelope.set_subparam(["param2", "param3"], {"1234": [None], "4321": 1.0})
+    expected_subparams = [
+        (["param1", "asdf"], [None]),
+        (["param1", "fdsa"], 1.0),
+        (["param2", "param3", "1234"], [None]),
+        (["param2", "param3", "4321"], 1.0),
+    ]
+    assert list(base64_json_envelope.get_subparams()) == expected_subparams
+    base64_json_envelope.set_subparam(["param2"], None)
+    expected_subparams = [
+        (["param1", "asdf"], [None]),
+        (["param1", "fdsa"], 1.0),
+        (["param2"], None),
+    ]
+    assert list(base64_json_envelope.get_subparams()) == expected_subparams
+
+    # xml inside url inside base64
+    xml_envelope = BaseEnvelope.detect(
+        "JTNDP3htbCUyMHZlcnNpb249JTIyMS4wJTIyJTIwZW5jb2Rpbmc9JTIydXRmLTglMjI/JTNFJTBBJTNDcm9vdCUzRSUzQ3BhcmFtMSUyMGF0dHI9JTIydmFsMSUyMiUzRXZhbDElM0MvcGFyYW0xJTNFJTNDcGFyYW0yJTNFJTNDcGFyYW0zJTNFdmFsMyUzQy9wYXJhbTMlM0UlM0MvcGFyYW0yJTNFJTNDL3Jvb3QlM0U="
+    )
+    assert (
+        xml_envelope.pack()
+        == "JTNDJTNGeG1sJTIwdmVyc2lvbiUzRCUyMjEuMCUyMiUyMGVuY29kaW5nJTNEJTIydXRmLTglMjIlM0YlM0UlMEElM0Nyb290JTNFJTNDcGFyYW0xJTIwYXR0ciUzRCUyMnZhbDElMjIlM0V2YWwxJTNDL3BhcmFtMSUzRSUzQ3BhcmFtMiUzRSUzQ3BhcmFtMyUzRXZhbDMlM0MvcGFyYW0zJTNFJTNDL3BhcmFtMiUzRSUzQy9yb290JTNF"
+    )
+    expected_subparams = [
+        (["root", "param1", "@attr"], "val1"),
+        (["root", "param1", "#text"], "val1"),
+        (["root", "param2", "param3"], "val3"),
+    ]
+    assert list(xml_envelope.get_subparams()) == expected_subparams
+    xml_envelope.set_subparam(["root", "param1", "@attr"], "asdf")
+    expected_subparams = [
+        (["root", "param1", "@attr"], "asdf"),
+        (["root", "param1", "#text"], "val1"),
+        (["root", "param2", "param3"], "val3"),
+    ]
+    assert list(xml_envelope.get_subparams()) == expected_subparams
+    assert (
+        xml_envelope.pack()
+        == "JTNDJTNGeG1sJTIwdmVyc2lvbiUzRCUyMjEuMCUyMiUyMGVuY29kaW5nJTNEJTIydXRmLTglMjIlM0YlM0UlMEElM0Nyb290JTNFJTNDcGFyYW0xJTIwYXR0ciUzRCUyMmFzZGYlMjIlM0V2YWwxJTNDL3BhcmFtMSUzRSUzQ3BhcmFtMiUzRSUzQ3BhcmFtMyUzRXZhbDMlM0MvcGFyYW0zJTNFJTNDL3BhcmFtMiUzRSUzQy9yb290JTNF"
+    )
+    xml_envelope.set_subparam(["root", "param2", "param3"], {"1234": [None], "4321": 1.0})
+    expected_subparams = [
+        (["root", "param1", "@attr"], "asdf"),
+        (["root", "param1", "#text"], "val1"),
+        (["root", "param2", "param3", "1234"], [None]),
+        (["root", "param2", "param3", "4321"], 1.0),
+    ]
+    assert list(xml_envelope.get_subparams()) == expected_subparams
+
+    # null
+    null_envelope = BaseEnvelope.detect("null")
+    assert isinstance(null_envelope, JSONEnvelope)
+    assert null_envelope.unpacked_data() is None
+    assert null_envelope.pack() == "null"
+    expected_subparams = [([], None)]
+    assert list(null_envelope.get_subparams()) == expected_subparams
+    for subparam, value in expected_subparams:
+        assert null_envelope.get_subparam(subparam) == value
+
+    tiny_base64 = BaseEnvelope.detect("YWJi")
+    assert isinstance(tiny_base64, TextEnvelope)