bbot 2.4.2.6590rc0__tar.gz → 2.4.2.6608rc0__tar.gz

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.2.6590rc0
+Version: 2.4.2.6608rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.2.6590rc"
+__version__ = "v2.4.2.6608rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/modules/internal/excavate.py

@@ -5,7 +5,7 @@ import time
 import inspect
 import regex as re
 from pathlib import Path
-from bbot.errors import ExcavateError
+from bbot.errors import ExcavateError, ValidationError
 import bbot.core.helpers.regexes as bbot_regexes
 from bbot.modules.base import BaseInterceptModule
 from bbot.modules.internal.base import BaseInternalModule
@@ -622,14 +622,15 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                                 base_url += f"?{event.parsed_url.query}"
                             url = urljoin(base_url, endpoint)
 
+                        try:
+                            # Validate the URL before using it
+                            parsed_url = self.excavate.helpers.validators.validate_url_parsed(url)
+                        except (ValidationError, ValueError) as e:
+                            self.excavate.debug(f"Invalid URL [{url}]: {e}")
+                            continue
+
                         if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
                             if self.excavate.in_bl(parameter_name) is False:
-                                parsed_url = urlparse(url)
-                                if not parsed_url.hostname:
-                                    self.excavate.warning(
-                                        f"Error Parsing reconstructed URL [{url}] during parameter extraction, missing hostname"
-                                    )
-                                    continue
                                 description = f"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)"
                                 data = {
                                     "host": parsed_url.hostname,
@@ -848,45 +849,51 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                 urls_found = 0
                 final_url = ""
                 for url_str in results:
-                    if identifier == "url_full":
-                        if not await self.helpers.re.search(self.full_url_regex, url_str):
+                    try:
+                        if identifier == "url_full":
+                            if not await self.helpers.re.search(self.full_url_regex, url_str):
+                                self.excavate.debug(
+                                    f"Rejecting potential full URL [{url_str}] as did not match full_url_regex"
+                                )
+                                continue
+                            final_url = url_str
+                            self.excavate.debug(f"Discovered Full URL [{final_url}]")
+
+                        elif identifier == "url_attr" and hasattr(event, "parsed_url"):
+                            m = await self.helpers.re.search(self.tag_attribute_regex, url_str)
+                            if not m:
+                                self.excavate.debug(
+                                    f"Rejecting potential attribute URL [{url_str}] as did not match tag_attribute_regex"
+                                )
+                                continue
+                            unescaped_url = html.unescape(m.group(1))
+                            source_url = event.parsed_url.geturl()
+                            final_url = urldefrag(urljoin(source_url, unescaped_url)).url
+                            if not await self.helpers.re.search(self.full_url_regex_strict, final_url):
+                                self.excavate.debug(
+                                    f"Rejecting reconstructed URL [{final_url}] as did not match full_url_regex_strict"
+                                )
+                                continue
                             self.excavate.debug(
-                                f"Rejecting potential full URL [{url_str}] as did not match full_url_regex"
+                                f"Reconstructed Full URL [{final_url}] from extracted relative URL [{unescaped_url}] "
                             )
-                            continue
-                        final_url = url_str
 
-                        self.excavate.debug(f"Discovered Full URL [{final_url}]")
-                    elif identifier == "url_attr" and hasattr(event, "parsed_url"):
-                        m = await self.helpers.re.search(self.tag_attribute_regex, url_str)
-                        if not m:
-                            self.excavate.debug(
-                                f"Rejecting potential attribute URL [{url_str}] as did not match tag_attribute_regex"
+                        if final_url:
+                            # Validate the URL before using it
+                            self.excavate.helpers.validators.validate_url_parsed(final_url)
+                            if self.excavate.scan.in_scope(final_url):
+                                urls_found += 1
+                            await self.report(
+                                final_url,
+                                event,
+                                yara_rule_settings,
+                                discovery_context,
+                                event_type="URL_UNVERIFIED",
+                                urls_found=urls_found,
                             )
-                            continue
-                        unescaped_url = html.unescape(m.group(1))
-                        source_url = event.parsed_url.geturl()
-                        final_url = urldefrag(urljoin(source_url, unescaped_url)).url
-                        if not await self.helpers.re.search(self.full_url_regex_strict, final_url):
-                            self.excavate.debug(
-                                f"Rejecting reconstructed URL [{final_url}] as did not match full_url_regex_strict"
-                            )
-                            continue
-                        self.excavate.debug(
-                            f"Reconstructed Full URL [{final_url}] from extracted relative URL [{unescaped_url}] "
-                        )
-
-                    if final_url:
-                        if self.excavate.scan.in_scope(final_url):
-                            urls_found += 1
-                        await self.report(
-                            final_url,
-                            event,
-                            yara_rule_settings,
-                            discovery_context,
-                            event_type="URL_UNVERIFIED",
-                            urls_found=urls_found,
-                        )
+                    except (ValidationError, ValueError) as e:
+                        self.excavate.debug(f"Invalid URL [{url_str if not final_url else final_url}]: {e}")
+                        continue
 
         async def report_prep(self, event_data, event_type, event, tags, **kwargs):
             event_draft = self.excavate.make_event(event_data, event_type, parent=event)
@@ -1114,7 +1121,10 @@ class excavate(BaseInternalModule, BaseInterceptModule):
 
                 # Check if rule processing function exists
                 if rule_name in self.yara_preprocess_dict:
-                    await self.yara_preprocess_dict[rule_name](result, event, discovery_context)
+                    try:
+                        await self.yara_preprocess_dict[rule_name](result, event, discovery_context)
+                    except ValidationError as e:
+                        self.debug(f"ValidationError in rule {rule_name} for result {result}: {e}")
                 else:
                     self.hugewarning(f"YARA Rule {rule_name} not found in pre-compiled rules")
 

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/modules/lightfuzz/submodules/serial.py

@@ -22,7 +22,7 @@ class serial(BaseLightfuzz):
     CONTROL_PAYLOAD_PHP_RAW = "z:0:{}"
 
     BASE64_SERIALIZATION_PAYLOADS = {
-        "php_base64": "YTowOnt9",
+        "php_base64": "YToxOntpOjA7aToxO30=",
         "java_base64": "rO0ABXNyABFqYXZhLmxhbmcuQm9vbGVhbs0gcoDVnPruAgABWgAFdmFsdWV4cAA=",
         "java_base64_string_error": "rO0ABXQABHRlc3Q=",
         "java_base64_OptionalDataException": "rO0ABXcEAAAAAAEAAAABc3IAEGphdmEudXRpbC5IYXNoTWFwAAAAAAAAAAECAAJMAARrZXkxYgABAAAAAAAAAAJ4cHcBAAAAB3QABHRlc3Q=",

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/presets/web/lightfuzz-xss.yml

@@ -1,4 +1,5 @@
 description: Discover web parameters and lightly fuzz them, limited to just GET-based xss vulnerabilities. This is an example of a custom lightfuzz preset, selectively enabling a single lightfuzz module.
+
 modules:
   - httpx
   - lightfuzz

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/scanner/preset/path.py

@@ -6,6 +6,7 @@ from bbot.errors import *
 log = logging.getLogger("bbot.presets.path")
 
 DEFAULT_PRESET_PATH = Path(__file__).parent.parent.parent / "presets"
+DEFAULT_PRESET_PATH = DEFAULT_PRESET_PATH.expanduser().resolve()
 
 
 class PresetPath:
@@ -17,7 +18,7 @@ class PresetPath:
         self.paths = [DEFAULT_PRESET_PATH]
 
     def find(self, filename):
-        filename_path = Path(filename).resolve()
+        filename_path = Path(filename).expanduser().resolve()
         extension = filename_path.suffix.lower()
         file_candidates = set()
         extension_candidates = {".yaml", ".yml"}
@@ -29,16 +30,12 @@ class PresetPath:
             file_candidates.add(f"{filename_path.stem}{ext}")
         file_candidates = sorted(file_candidates)
         file_candidates_str = ",".join([str(s) for s in file_candidates])
-        paths_to_search = self.paths
         if "/" in str(filename):
-            if filename_path.parent not in paths_to_search:
-                paths_to_search.append(filename_path.parent)
-        log.debug(
-            f"Searching for preset in {[str(p) for p in paths_to_search]}, file candidates: {file_candidates_str}"
-        )
-        for path in paths_to_search:
+            self.add_path(filename_path.parent)
+        log.debug(f"Searching for {file_candidates_str} in {[str(p) for p in self.paths]}")
+        for path in self.paths:
             for candidate in file_candidates:
-                for file in path.rglob(candidate):
+                for file in path.rglob(f"**/{candidate}"):
                     if file.is_file():
                         log.verbose(f'Found preset matching "{filename}" at {file}')
                         self.add_path(file.parent)
@@ -51,14 +48,19 @@ class PresetPath:
         return ":".join([str(s) for s in self.paths])
 
     def add_path(self, path):
-        path = Path(path).resolve()
+        path = Path(path).expanduser().resolve()
+        # skip if already in paths
         if path in self.paths:
             return
+        # skip if path is a subdirectory of any path in paths
         if any(path.is_relative_to(p) for p in self.paths):
             return
+        # skip if path is not a directory
         if not path.is_dir():
             log.debug(f'Path "{path.resolve()}" is not a directory')
             return
+        # preemptively remove any paths that are subdirectories of the new path
+        self.paths = [p for p in self.paths if not p.is_relative_to(path)]
         self.paths.append(path)
 
     def __iter__(self):

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/scanner/preset/preset.py

@@ -308,7 +308,7 @@ class Preset(metaclass=BasePreset):
 
     @property
     def preset_dir(self):
-        return self.bbot_home / "presets"
+        return (self.bbot_home / "presets").expanduser().resolve()
 
     @property
     def default_output_modules(self):
@@ -413,30 +413,32 @@ class Preset(metaclass=BasePreset):
         self.log_debug("Getting baked")
         # create a copy of self
         baked_preset = copy(self)
-        baked_preset.scan = scan
+
         # copy core
         baked_preset.core = self.core.copy()
-        # copy module loader
-        baked_preset._module_loader = self.module_loader.copy()
-        # prepare os environment
-        os_environ = baked_preset.environ.prepare()
-        # find and replace preloaded modules with os environ
-        # this is different from the config variable substitution because it modifies
-        #  the preloaded modules, i.e. their ansible playbooks
-        baked_preset.module_loader.find_and_replace(**os_environ)
-        # update os environ
-        os.environ.clear()
-        os.environ.update(os_environ)
 
-        # validate flags, config options
-        baked_preset.validate()
+        if scan is not None:
+            baked_preset.scan = scan
+            # copy module loader
+            baked_preset._module_loader = self.module_loader.copy()
+            # prepare os environment
+            os_environ = baked_preset.environ.prepare()
+            # find and replace preloaded modules with os environ
+            # this is different from the config variable substitution because it modifies
+            #  the preloaded modules, i.e. their ansible playbooks
+            baked_preset.module_loader.find_and_replace(**os_environ)
+            # update os environ
+            os.environ.clear()
+            os.environ.update(os_environ)
+
+            # assign baked preset to our scan
+            scan.preset = baked_preset
 
         # validate log level options
         baked_preset.apply_log_level(apply_core=scan is not None)
 
-        # assign baked preset to our scan
-        if scan is not None:
-            scan.preset = baked_preset
+        # validate flags, config options
+        baked_preset.validate()
 
         # now that our requirements / exclusions are validated, we can start enabling modules
         # enable scan modules
@@ -483,15 +485,19 @@ class Preset(metaclass=BasePreset):
         from bbot.scanner.target import BBOTTarget
 
         baked_preset._target = BBOTTarget(
-            *list(self._seeds), whitelist=self._whitelist, blacklist=self._blacklist, strict_scope=self.strict_scope
+            *list(self._seeds),
+            whitelist=self._whitelist,
+            blacklist=self._blacklist,
+            strict_scope=self.strict_scope,
         )
 
-        # evaluate conditions
-        if baked_preset.conditions:
-            from .conditions import ConditionEvaluator
+        if scan is not None:
+            # evaluate conditions
+            if baked_preset.conditions:
+                from .conditions import ConditionEvaluator
 
-            evaluator = ConditionEvaluator(baked_preset)
-            evaluator.evaluate()
+                evaluator = ConditionEvaluator(baked_preset)
+                evaluator.evaluate()
 
         self._baked = True
         return baked_preset
@@ -562,6 +568,12 @@ class Preset(metaclass=BasePreset):
         return self.scope_config.get("strict", False)
 
     def apply_log_level(self, apply_core=False):
+        """
+        Apply the log level to the preset.
+
+        Args:
+            apply_core (bool, optional): If True, apply the log level to the core logger.
+        """
         # silent takes precedence
         if self.silent:
             self.verbose = False
@@ -920,20 +932,17 @@ class Preset(metaclass=BasePreset):
         """
         Recursively find all the presets and return them as a dictionary
         """
-        preset_dir = self.preset_dir
-        home_dir = Path.home()
-
         # first, add local preset dir to PRESET_PATH
         PRESET_PATH.add_path(self.preset_dir)
 
         # ensure local preset directory exists
-        mkdir(preset_dir)
+        mkdir(self.preset_dir)
 
         global DEFAULT_PRESETS
         if DEFAULT_PRESETS is None:
             presets = {}
-            for ext in ("yml", "yaml"):
-                for preset_path in PRESET_PATH:
+            for preset_path in PRESET_PATH:
+                for ext in ("yml", "yaml"):
                     # for every yaml file
                     for original_filename in preset_path.rglob(f"**/*.{ext}"):
                         # not including symlinks
@@ -957,18 +966,14 @@ class Preset(metaclass=BasePreset):
 
                         local_preset = original_filename
                         # populate symlinks in local preset dir
-                        if not original_filename.is_relative_to(preset_dir):
+                        if not original_filename.is_relative_to(self.preset_dir):
                             relative_preset = original_filename.relative_to(preset_path)
-                            local_preset = preset_dir / relative_preset
+                            local_preset = self.preset_dir / relative_preset
                             mkdir(local_preset.parent, check_writable=False)
                             if not local_preset.exists():
                                 local_preset.symlink_to(original_filename)
 
-                        # collapse home directory into "~"
-                        if local_preset.is_relative_to(home_dir):
-                            local_preset = Path("~") / local_preset.relative_to(home_dir)
-
-                        presets[local_preset] = (loaded_preset, category, preset_path, original_filename)
+                        presets[local_preset.stem] = (loaded_preset, category, preset_path, original_filename)
 
             # sort by name
             DEFAULT_PRESETS = dict(sorted(presets.items(), key=lambda x: x[-1][0].name))

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/scripts/docs.py

@@ -198,15 +198,15 @@ def update_docs():
     update_md_files("BBOT PRESETS", bbot_presets_table)
 
     # BBOT presets
-    for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
+    for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
         preset_yaml = f"""
-```yaml title={yaml_file.name}
+```yaml title={preset_path.name}
 {loaded_preset._yaml_str}
 ```
 """
         preset_yaml_expandable = f"""
 <details>
-<summary><b><code>{yaml_file.name}</code></b></summary>
+<summary><b><code>{preset_path.name}</code></b></summary>
 
 ```yaml
 {loaded_preset._yaml_str}
@@ -218,11 +218,11 @@ def update_docs():
         update_md_files(f"BBOT {loaded_preset.name.upper()} PRESET EXPANDABLE", preset_yaml_expandable)
 
     content = []
-    for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
+    for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():
         yaml_str = loaded_preset._yaml_str
         indent = " " * 4
         yaml_str = f"\n{indent}".join(yaml_str.splitlines())
-        filename = homedir_collapseuser(yaml_file)
+        filename = homedir_collapseuser(preset_path)
 
         num_modules = len(loaded_preset.scan_modules)
         modules = ", ".join(sorted([f"`{m}`" for m in loaded_preset.scan_modules]))

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -1418,3 +1418,17 @@ class TestExcavateBadURLs(ModuleTestBase):
 
         url_events = [e for e in events if e.type == "URL_UNVERIFIED"]
         assert sorted([e.data for e in url_events]) == sorted(["https://ssl/", "http://127.0.0.1:8888/"])
+
+
+class TestExcavateURL_InvalidPort(TestExcavate):
+    modules_overrides = ["excavate", "httpx", "hunt"]
+
+    async def setup_before_prep(self, module_test):
+        # Test URL with invalid port (greater than 65535)
+        module_test.httpserver.expect_request("/").respond_with_data(
+            '<div><img loading="lazy" src="https://asdffoo.test.notreal:9212952841/whatever.jpg" width="576" height="382" alt="...." /></div>'
+        )
+
+    def check(self, module_test, events):
+        # Verify we got the hostname
+        assert any(e.data == "asdffoo.test.notreal" for e in events)

{bbot-2.4.2.6590rc0 → bbot-2.4.2.6608rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.4.2.6590rc"
+version = "v2.4.2.6608rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",