bbot 2.7.0.6962rc0__tar.gz → 2.7.0.6989rc0__tar.gz

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.7.0.6962rc0
+Version: 2.7.0.6989rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.7.0.6962rc"
+__version__ = "v2.7.0.6989rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/core/helpers/files.py

@@ -9,7 +9,7 @@ from .misc import rm_at_exit
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/defaults.yml

@@ -187,8 +187,10 @@ url_extension_blacklist:
   - mov
   - flv
   - webm
-# Distribute URLs with these extensions only to httpx (these are omitted from output)
-url_extension_httpx_only:
+
+# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`
+# They are also excluded from output. If you want to see them in output, remove them from this list.
+url_extension_special:
   - js
 
 # These url extensions are almost always static, so we exclude them from modules that fuzz things

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/modules/base.py

@@ -53,6 +53,8 @@ class BaseModule:
 
         in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.
 
+        accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False.
+
         options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default.
 
         options_desc (Dict): Descriptions for options, e.g., {"api_key": "API Key"}. Empty dict by default.
@@ -97,7 +99,7 @@ class BaseModule:
     scope_distance_modifier = 0
     target_only = False
     in_scope_only = False
-
+    accept_url_special = False
     _module_threads = 1
     _batch_size = 1
 
@@ -785,10 +787,14 @@ class BaseModule:
             if "target" not in event.tags:
                 return False, "it did not meet target_only filter criteria"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, "its extension was listed in url_extension_httpx_only"
+        # limit js URLs to modules that opt in to receive them
+        if (not self.accept_url_special) and event.type.startswith("URL"):
+            extension = getattr(event, "url_extension", "")
+            if extension in self.scan.url_extension_special:
+                return (
+                    False,
+                    f"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs",
+                )
 
         return True, "precheck succeeded"
 

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/modules/graphql_introspection.py

@@ -27,7 +27,6 @@ class graphql_introspection(BaseModule):
             self.output_dir = Path(output_folder) / "graphql-schemas"
         else:
             self.output_dir = self.scan.home / "graphql-schemas"
-        self.helpers.mkdir(self.output_dir)
         return True
 
     async def filter_event(self, event):
@@ -128,6 +127,7 @@ fragment TypeRef on __Type {
                 self.debug(f"Failed to parse JSON for {url}")
                 continue
             if response_json.get("data", {}).get("__schema", {}).get("types", []):
+                self.helpers.mkdir(self.output_dir)
                 filename = f"schema-{self.helpers.tagify(url)}.json"
                 filename = self.output_dir / filename
                 with open(filename, "w") as f:

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/modules/httpx.py

@@ -50,6 +50,8 @@ class httpx(BaseModule):
     _shuffle_incoming_queue = False
     _batch_size = 500
     _priority = 2
+    # accept Javascript URLs
+    accept_url_special = True
 
     async def setup(self):
         self.threads = self.config.get("threads", 50)

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/modules/output/base.py

@@ -38,11 +38,6 @@ class BaseOutputModule(BaseModule):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, (f"Omitting {event} from output because it's marked as httpx-only")
-
         # omit certain event types
         if event._omit:
             if "target" in event.tags:

bbot-2.7.0.6989rc0/bbot/modules/retirejs.py

@@ -0,0 +1,238 @@
+import json
+from enum import IntEnum
+from bbot.modules.base import BaseModule
+
+
+class RetireJSSeverity(IntEnum):
+    NONE = 0
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+    CRITICAL = 4
+
+    @classmethod
+    def from_string(cls, severity_str):
+        try:
+            return cls[severity_str.upper()]
+        except (KeyError, AttributeError):
+            return cls.NONE
+
+
+class retirejs(BaseModule):
+    watched_events = ["URL_UNVERIFIED"]
+    produced_events = ["FINDING"]
+    flags = ["active", "safe", "web-thorough"]
+    meta = {
+        "description": "Detect vulnerable/out-of-date JavaScript libraries",
+        "created_date": "2025-08-19",
+        "author": "@liquidsec",
+    }
+    options = {
+        "version": "5.3.0",
+        "node_version": "18.19.1",
+        "severity": "medium",
+    }
+    options_desc = {
+        "version": "retire.js version",
+        "node_version": "Node.js version to install locally",
+        "severity": "Minimum severity level to report (none, low, medium, high, critical)",
+    }
+
+    deps_ansible = [
+        # Download Node.js binary (Linux x64)
+        {
+            "name": "Download Node.js binary (Linux x64)",
+            "get_url": {
+                "url": "https://nodejs.org/dist/v#{BBOT_MODULES_RETIREJS_NODE_VERSION}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "dest": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "mode": "0644",
+            },
+        },
+        # Extract Node.js binary (x64)
+        {
+            "name": "Extract Node.js binary (x64)",
+            "unarchive": {
+                "src": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "dest": "#{BBOT_TOOLS}",
+                "remote_src": True,
+            },
+        },
+        # Remove existing node directory if it exists
+        {
+            "name": "Remove existing node directory",
+            "file": {"path": "#{BBOT_TOOLS}/node", "state": "absent"},
+        },
+        # Rename extracted directory to 'node' (x64)
+        {
+            "name": "Rename Node.js directory (x64)",
+            "command": "mv #{BBOT_TOOLS}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64 #{BBOT_TOOLS}/node",
+        },
+        # Make Node.js binary executable
+        {
+            "name": "Make Node.js binary executable",
+            "file": {"path": "#{BBOT_TOOLS}/node/bin/node", "mode": "0755"},
+        },
+        # Make npm executable
+        {
+            "name": "Make npm executable",
+            "file": {"path": "#{BBOT_TOOLS}/node/bin/npm", "mode": "0755"},
+        },
+        # Remove existing retirejs directory if it exists
+        {
+            "name": "Remove existing retirejs directory",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "absent"},
+        },
+        # Create retire.js local directory
+        {
+            "name": "Create retire.js directory in BBOT_TOOLS",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "directory", "mode": "0755"},
+        },
+        # Install retire.js locally using local Node.js
+        {
+            "name": "Install retire.js locally",
+            "shell": "cd #{BBOT_TOOLS}/retirejs && #{BBOT_TOOLS}/node/bin/node #{BBOT_TOOLS}/node/lib/node_modules/npm/bin/npm-cli.js install retire@#{BBOT_MODULES_RETIREJS_VERSION} --no-fund --no-audit --silent --no-optional",
+            "args": {"creates": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire"},
+            "timeout": 600,
+            "ignore_errors": False,
+        },
+        # Fix retire script shebang to use our local node binary
+        {
+            "name": "Fix retire script shebang",
+            "shell": "sed -i '1s|#!/usr/bin/env node|#!#{BBOT_TOOLS}/node/bin/node|' #{BBOT_TOOLS}/retirejs/node_modules/.bin/retire",
+        },
+        # Make retire script executable
+        {
+            "name": "Make retire script executable",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire", "mode": "0755"},
+        },
+        # Create retire cache directory
+        {
+            "name": "Create retire cache directory",
+            "file": {"path": "#{BBOT_CACHE}/retire_cache", "state": "directory", "mode": "0755"},
+        },
+    ]
+
+    accept_url_special = True
+    scope_distance_modifier = 1
+    _module_threads = 4
+
+    async def setup(self):
+        excavate_enabled = self.scan.config.get("excavate")
+        if not excavate_enabled:
+            return None, "retirejs will not function without excavate enabled"
+
+        # Validate severity level
+        valid_severities = ["none", "low", "medium", "high", "critical"]
+        configured_severity = self.config.get("severity", "medium").lower()
+        if configured_severity not in valid_severities:
+            return (
+                False,
+                f"Invalid severity level '{configured_severity}'. Valid options are: {', '.join(valid_severities)}",
+            )
+
+        self.repofile = await self.helpers.download(
+            "https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository-v4.json", cache_hrs=24
+        )
+        if not self.repofile:
+            return False, "failed to download retire.js repository file"
+        return True
+
+    async def handle_event(self, event):
+        js_file = await self.helpers.request(event.data)
+        if js_file:
+            js_file_body = js_file.text
+            if js_file_body:
+                js_file_body_saved = self.helpers.tempfile(js_file_body, pipe=False, extension="js")
+                results = await self.execute_retirejs(js_file_body_saved)
+                if not results:
+                    self.warning("no output from retire.js")
+                    return
+                results_json = json.loads(results)
+                if results_json.get("data"):
+                    for file_result in results_json["data"]:
+                        for component_result in file_result.get("results", []):
+                            component = component_result.get("component", "unknown")
+                            version = component_result.get("version", "unknown")
+                            vulnerabilities = component_result.get("vulnerabilities", [])
+                            for vuln in vulnerabilities:
+                                severity = vuln.get("severity", "unknown")
+
+                                # Filter by minimum severity level
+                                min_severity = RetireJSSeverity.from_string(self.config.get("severity", "medium"))
+                                vuln_severity = RetireJSSeverity.from_string(severity)
+                                if vuln_severity < min_severity:
+                                    self.debug(
+                                        f"Skipping vulnerability with severity '{severity}' (below minimum '{min_severity.name.lower()}')"
+                                    )
+                                    continue
+
+                                identifiers = vuln.get("identifiers", {})
+                                summary = identifiers.get("summary", "Unknown vulnerability")
+                                cves = identifiers.get("CVE", [])
+                                description_parts = [
+                                    f"Vulnerable JavaScript library detected: {component} v{version}",
+                                    f"Severity: {severity.upper()}",
+                                    f"Summary: {summary}",
+                                    f"JavaScript URL: {event.data}",
+                                ]
+                                if cves:
+                                    description_parts.append(f"CVE(s): {', '.join(cves)}")
+
+                                below_version = vuln.get("below", "")
+                                at_or_above = vuln.get("atOrAbove", "")
+                                if at_or_above and below_version:
+                                    description_parts.append(f"Affected versions: [{at_or_above} to {below_version})")
+                                elif below_version:
+                                    description_parts.append(f"Affected versions: [< {below_version}]")
+                                elif at_or_above:
+                                    description_parts.append(f"Affected versions: [>= {at_or_above}]")
+                                description = " ".join(description_parts)
+                                data = {
+                                    "description": description,
+                                    "severity": severity,
+                                    "component": component,
+                                    "url": event.parent.data["url"],
+                                }
+                                await self.emit_event(
+                                    data,
+                                    "FINDING",
+                                    parent=event,
+                                    context=f"{{module}} identified vulnerable JavaScript library {component} v{version} ({severity} severity)",
+                                )
+
+    async def filter_event(self, event):
+        url_extension = getattr(event, "url_extension", "")
+        if url_extension != "js":
+            return False, f"it is a {url_extension} URL but retirejs only accepts js URLs"
+        return True
+
+    async def execute_retirejs(self, js_file):
+        cache_dir = self.helpers.cache_dir / "retire_cache"
+        retire_dir = self.scan.helpers.tools_dir / "retirejs"
+
+        # Use the retire CLI script directly with our local node binary
+        local_node_dir = self.scan.helpers.tools_dir / "node"
+        retire_cli_script = retire_dir / "node_modules" / "retire" / "lib" / "cli.js"
+
+        command = [
+            str(local_node_dir / "bin" / "node"),
+            str(retire_cli_script),
+            "--outputformat",
+            "json",
+            "--cachedir",
+            str(cache_dir),
+            "--path",
+            js_file,
+            "--jsrepo",
+            str(self.repofile),
+        ]
+
+        proxy = self.scan.web_config.get("http_proxy")
+        if proxy:
+            command.extend(["--proxy", proxy])
+
+        self.verbose(f"Running retire.js on {js_file}")
+        self.verbose(f"retire.js command: {command}")
+
+        result = await self.run_process(command)
+        return result.stdout

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/scanner/manager.py

@@ -94,10 +94,6 @@ class ScanIngress(BaseInterceptModule):
         # special handling of URL extensions
         url_extension = getattr(event, "url_extension", None)
         if url_extension is not None:
-            if url_extension in self.scan.url_extension_httpx_only:
-                event.add_tag("httpx-only")
-                event._omit = True
-
             # blacklist by extension
             if url_extension in self.scan.url_extension_blacklist:
                 self.debug(
@@ -209,6 +205,13 @@ class ScanEgress(BaseInterceptModule):
             )
             event.internal = True
 
+        # mark special URLs (e.g. Javascript) as internal so they don't get output except when they're critical to the graph
+        if event.type.startswith("URL"):
+            extension = getattr(event, "url_extension", "")
+            if extension in self.scan.url_extension_special:
+                event.internal = True
+                self.debug(f"Making {event} internal because it is a special URL (extension {extension})")
+
         if event.type in self.scan.omitted_event_types:
             self.debug(f"Omitting {event} because its type is omitted in the config")
             event._omit = True

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/scanner/scanner.py

@@ -230,8 +230,8 @@ class Scanner:
             )
 
         # url file extensions
+        self.url_extension_special = {e.lower() for e in self.config.get("url_extension_special", [])}
         self.url_extension_blacklist = {e.lower() for e in self.config.get("url_extension_blacklist", [])}
-        self.url_extension_httpx_only = {e.lower() for e in self.config.get("url_extension_httpx_only", [])}
 
         # url querystring behavior
         self.url_querystring_remove = self.config.get("url_querystring_remove", True)

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/test/test_step_1/test_bbot_fastapi.py

@@ -22,8 +22,8 @@ def test_bbot_multiprocess(bbot_httpserver):
     queue = multiprocessing.Queue()
     events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))
     events_process.start()
-    events_process.join()
-    events = queue.get()
+    events_process.join(timeout=300)
+    events = queue.get(timeout=10)
     assert len(events) >= 3
     scan_events = [e for e in events if e["type"] == "SCAN"]
     assert len(scan_events) == 2

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/test/test_step_1/test_events.py

@@ -209,7 +209,6 @@ async def test_events(events, helpers):
     javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", parent=scan.root_event)
     assert "extension-js" in javascript_event.tags
     await scan.ingress_module.handle_event(javascript_event)
-    assert "httpx-only" in javascript_event.tags
 
     # scope distance
     event1 = scan.make_event("1.2.3.4", dummy=True)

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/test/test_step_1/test_scan.py

@@ -147,24 +147,18 @@ async def test_task_scan_handle_event_timeout(bbot_scanner):
 
 @pytest.mark.asyncio
 async def test_url_extension_handling(bbot_scanner):
-    scan = bbot_scanner(config={"url_extension_blacklist": ["css"], "url_extension_httpx_only": ["js"]})
+    scan = bbot_scanner(config={"url_extension_blacklist": ["css"]})
     await scan._prep()
     assert scan.url_extension_blacklist == {"css"}
-    assert scan.url_extension_httpx_only == {"js"}
     good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], parent=scan.root_event)
     bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], parent=scan.root_event)
-    httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], parent=scan.root_event)
     assert "blacklisted" not in bad_event.tags
-    assert "httpx-only" not in httpx_event.tags
     result = await scan.ingress_module.handle_event(good_event)
     assert result is None
     result, reason = await scan.ingress_module.handle_event(bad_event)
     assert result is False
     assert reason == "event is blacklisted"
     assert "blacklisted" in bad_event.tags
-    result = await scan.ingress_module.handle_event(httpx_event)
-    assert result is None
-    assert "httpx-only" in httpx_event.tags
 
     await scan._cleanup()
 

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/test/test_step_2/module_tests/base.py

@@ -61,6 +61,7 @@ class ModuleTestBase:
                 config=self.config,
                 whitelist=module_test_base.whitelist,
                 blacklist=module_test_base.blacklist,
+                force_start=getattr(module_test_base, "force_start", False),
             )
             self.events = []
             self.log = logging.getLogger(f"bbot.test.{module_test_base.name}")

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -24,14 +24,13 @@ class TestExcavate(ModuleTestBase):
         \\x3dwww6.test.notreal
         %0awww7.test.notreal
         \\u000awww8.test.notreal
-        # these ones shouldn't get emitted because they're .js (url_extension_httpx_only)
-        <a href="/a_relative.js">
-        <link href="/link_relative.js">
-        # these ones should
         <a href="/a_relative.txt">
         <link href="/link_relative.txt">
         <a href="mailto:bob@evilcorp.org?subject=help">Help</a>
         <li class="toctree-l3"><a class="reference internal" href="miscellaneous.html#x50-uart-driver">16x50 UART Driver</a></li>
+        # these ones should get emitted as URL_UNVERIFIED events (processed by httpx which has accept_js_url=True)
+        <a href="/a_relative.js">
+        <link href="/link_relative.js">
         """
         expect_args = {"method": "GET", "uri": "/"}
         respond_args = {"response_data": response_data}
@@ -63,8 +62,9 @@ class TestExcavate(ModuleTestBase):
         assert "www6.test.notreal" in event_data
         assert "www7.test.notreal" in event_data
         assert "www8.test.notreal" in event_data
-        assert "http://127.0.0.1:8888/a_relative.js" not in event_data
-        assert "http://127.0.0.1:8888/link_relative.js" not in event_data
+        # .js files should be emitted as URL_UNVERIFIED events (they are processed by httpx which has accept_js_url=True)
+        assert "http://127.0.0.1:8888/a_relative.js" in event_data
+        assert "http://127.0.0.1:8888/link_relative.js" in event_data
         assert "http://127.0.0.1:8888/a_relative.txt" in event_data
         assert "http://127.0.0.1:8888/link_relative.txt" in event_data
 

bbot-2.7.0.6989rc0/bbot/test/test_step_2/module_tests/test_module_retirejs.py

@@ -0,0 +1,159 @@
+from .base import ModuleTestBase
+
+
+class TestRetireJS(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "excavate", "retirejs"]
+
+    # HTML page with vulnerable JavaScript libraries
+    vulnerable_html = """<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>retire.js test page</title>
+</head>
+<body>
+  <h1>retire.js test page</h1>
+  <p>This page includes JavaScript libraries for testing.</p>
+
+  <!-- jQuery 3.4.1 -->
+  <script src="/jquery-3.4.1.min.js"></script>
+
+  <!-- Lodash 4.17.11 -->
+  <script src="/lodash.min.js"></script>
+
+  <!-- Handlebars 4.0.5 -->
+  <script src="/handlebars.min.js"></script>
+
+  <script>
+    console.log('Libraries loaded');
+  </script>
+</body>
+</html>"""
+
+    # Sample jQuery 3.4.1 content
+    jquery_content = """/*!
+ * jQuery JavaScript Library v3.4.1
+ * https://jquery.com/
+ */
+(function( global, factory ) {
+    "use strict";
+    factory( global );
+})(typeof window !== "undefined" ? window : this, function( window, noGlobal ) {
+    var jQuery = function( selector, context ) {
+        return new jQuery.fn.init( selector, context );
+    };
+    jQuery.fn = jQuery.prototype = {};
+    jQuery.fn.jquery = "3.4.1";
+    if ( typeof noGlobal === "undefined" ) {
+        window.jQuery = window.$ = jQuery;
+    }
+    return jQuery;
+});"""
+
+    # Sample Lodash 4.17.11 content
+    lodash_content = """/**
+ * @license
+ * Lodash lodash.com/license | Underscore.js 1.8.3 underscorejs.org/LICENSE
+ */
+;(function(){
+var i="4.17.11";
+var Mn={VERSION:i};
+if(typeof define=="function"&&define.amd)define(function(){return Mn});else if(typeof module=="object"&&module.exports)module.exports=Mn;else this._=Mn}());"""
+
+    # Sample Handlebars 4.0.5 content
+    handlebars_content = """/*!
+ handlebars v4.0.5
+*/
+!function(a,b){"object"==typeof exports&&"object"==typeof module?module.exports=b():"function"==typeof define&&define.amd?define([],b):"object"==typeof exports?exports.Handlebars=b():a.Handlebars=b()}(this,function(){
+var Handlebars={};
+Handlebars.VERSION="4.0.5";
+return Handlebars;
+});"""
+
+    async def setup_after_prep(self, module_test):
+        expect_args = {"uri": "/"}
+        respond_args = {"response_data": self.vulnerable_html}
+        module_test.set_expect_requests(expect_args, respond_args)
+
+        expect_args = {"uri": "/jquery-3.4.1.min.js"}
+        respond_args = {"response_data": self.jquery_content}
+        module_test.set_expect_requests(expect_args, respond_args)
+
+        expect_args = {"uri": "/lodash.min.js"}
+        respond_args = {"response_data": self.lodash_content}
+        module_test.set_expect_requests(expect_args, respond_args)
+
+        expect_args = {"uri": "/handlebars.min.js"}
+        respond_args = {"response_data": self.handlebars_content}
+        module_test.set_expect_requests(expect_args, respond_args)
+
+    def check(self, module_test, events):
+        # Check that excavate found the JavaScript URLs
+        url_unverified_events = [e for e in events if e.type == "URL_UNVERIFIED"]
+        js_url_events = [e for e in url_unverified_events if "extension-js" in e.tags]
+
+        # We should have found the JavaScript URLs
+        assert len(url_unverified_events) > 0, "No URL_UNVERIFIED events found - excavate may not be working"
+        assert len(js_url_events) >= 3, f"Expected at least 3 JavaScript URLs, found {len(js_url_events)}"
+
+        # Check for FINDING events generated by retirejs
+        finding_events = [e for e in events if e.type == "FINDING"]
+        retirejs_findings = [
+            e
+            for e in finding_events
+            if "vulnerable javascript library detected:" in e.data.get("description", "").lower()
+        ]
+
+        # We should have at least some findings from our vulnerable libraries
+        assert len(retirejs_findings) > 0, (
+            f"Expected retirejs to find vulnerabilities, but got {len(retirejs_findings)} findings"
+        )
+
+        # Check for specific expected vulnerability descriptions
+        descriptions = [finding.data.get("description", "") for finding in retirejs_findings]
+        all_descriptions = "\n".join(descriptions)
+
+        # Look for specific vulnerabilities we expect to find
+        expected_handlebars_vuln = "Vulnerable JavaScript library detected: handlebars v4.0.5 Severity: HIGH Summary: Regular Expression Denial of Service in Handlebars JavaScript URL: http://127.0.0.1:8888/handlebars.min.js CVE(s): CVE-2019-20922 Affected versions: [4.0.0 to 4.4.5)"
+        expected_jquery_vuln = "Vulnerable JavaScript library detected: jquery v3.4.1 Severity: MEDIUM Summary: Regex in its jQuery.htmlPrefilter sometimes may introduce XSS JavaScript URL: http://127.0.0.1:8888/jquery-3.4.1.min.js CVE(s): CVE-2020-11022 Affected versions: [1.2.0 to 3.5.0)"
+
+        # Verify at least one of the expected vulnerabilities is found
+        handlebars_found = expected_handlebars_vuln in all_descriptions
+        jquery_found = expected_jquery_vuln in all_descriptions
+
+        assert handlebars_found and jquery_found, (
+            f"Expected to find specific vulnerabilities but didn't find them. Found descriptions:\n{all_descriptions}"
+        )
+
+        # Basic validation of findings structure
+        for finding in retirejs_findings:
+            assert "description" in finding.data, "Finding should have description"
+            assert "url" in finding.data, "Finding should have url"
+            assert finding.parent.type == "URL_UNVERIFIED", "Parent should be URL_UNVERIFIED"
+
+
+class TestRetireJSNoExcavate(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "retirejs"]
+    force_start = True  # Allow scan to continue even if modules fail setup
+    config_overrides = {
+        "excavate": False,
+    }
+
+    def check(self, module_test, events):
+        # When excavate is disabled, retirejs should fail setup but scan should still run
+        retirejs_module = module_test.scan.modules.get("retirejs")
+
+        if retirejs_module:
+            # Check that the module exists but setup failed
+            setup_status = getattr(retirejs_module, "_setup_status", None)
+            if setup_status is not None:
+                success, error_msg = setup_status
+                assert success is False, "retirejs setup should have failed without excavate"
+                expected_error = "retirejs will not function without excavate enabled"
+                assert error_msg == expected_error, f"Expected error message '{expected_error}', but got '{error_msg}'"
+
+        # No retirejs findings should be generated since setup failed
+        retirejs_findings = [e for e in events if e.type == "FINDING" and getattr(e, "module", None) == "retirejs"]
+        assert len(retirejs_findings) == 0, "retirejs should not generate findings when setup fails"

{bbot-2.7.0.6962rc0 → bbot-2.7.0.6989rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.7.0.6962rc"
+version = "v2.7.0.6989rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",