bbot 2.6.0.6879rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl

bbot/modules/github_workflows.py

@@ -8,7 +8,7 @@ from bbot.modules.templates.github import github
 class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download a github repositories workflow logs and workflow artifacts",
         "created_date": "2024-04-29",

bbot/modules/gitlab_com.py

@@ -0,0 +1,31 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_com(GitLabBaseModule):
+    watched_events = ["SOCIAL"]
+    produced_events = [
+        "CODE_REPOSITORY",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Enumerate GitLab SaaS (gitlab.com/org) for projects and groups",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"}
+
+    # This is needed because we are consuming SOCIAL events, which aren't in scope
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        await self.handle_social(event)
+
+    async def filter_event(self, event):
+        if event.data["platform"] != "gitlab":
+            return False, "platform is not gitlab"
+        _, domain = self.helpers.split_domain(event.host)
+        if domain not in self.saas_domains:
+            return False, "gitlab instance is not gitlab.com/org"
+        return True

bbot/modules/gitlab_onprem.py

@@ -0,0 +1,84 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_onprem(GitLabBaseModule):
+    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
+    produced_events = [
+        "TECHNOLOGY",
+        "SOCIAL",
+        "CODE_REPOSITORY",
+        "FINDING",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Detect self-hosted GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    # Optional GitLab access token (only required for gitlab.com, but still
+    # supported for on-prem installations that expose private projects).
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"}
+
+    # Allow accepting events slightly beyond configured max distance so we can
+    # discover repos on neighbouring infrastructure.
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        if event.type == "HTTP_RESPONSE":
+            await self.handle_http_response(event)
+        elif event.type == "TECHNOLOGY":
+            await self.handle_technology(event)
+        elif event.type == "SOCIAL":
+            await self.handle_social(event)
+
+    async def filter_event(self, event):
+        # only accept out-of-scope SOCIAL events
+        if event.type == "HTTP_RESPONSE":
+            if event.scope_distance > self.scan.scope_search_distance:
+                return False, "event is out of scope distance"
+        elif event.type == "TECHNOLOGY":
+            if not event.data["technology"].lower().startswith("gitlab"):
+                return False, "technology is not gitlab"
+            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
+                return False, "gitlab instance is not self-hosted"
+        elif event.type == "SOCIAL":
+            if event.data["platform"] != "gitlab":
+                return False, "platform is not gitlab"
+            _, domain = self.helpers.split_domain(event.host)
+            if domain in self.saas_domains:
+                return False, "gitlab instance is not self-hosted"
+        return True
+
+    async def handle_http_response(self, event):
+        """Identify GitLab servers from HTTP responses."""
+        headers = event.data.get("header", {})
+        if "x_gitlab_meta" in headers:
+            url = event.parsed_url._replace(path="/").geturl()
+            await self.emit_event(
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
+            )
+            description = f"GitLab server at {event.host}"
+            await self.emit_event(
+                {"host": str(event.host), "description": description},
+                "FINDING",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
+            )
+
+    async def handle_technology(self, event):
+        """Enumerate projects & groups once we know a host is GitLab."""
+        base_url = self.get_base_url(event)
+
+        # Projects owned by the authenticated user (or public projects if no
+        # authentication).
+        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
+        await self.handle_projects_url(projects_url, event)
+
+        # Group enumeration.
+        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
+        await self.handle_groups_url(groups_url, event)

bbot/modules/gowitness.py

@@ -161,7 +161,6 @@ class gowitness(BaseModule):
                 key = e.data["url"]
             event_dict[key] = e
         stdin = "\n".join(list(event_dict))
-        self.hugeinfo(f"Gowitness input: {stdin}")
 
         try:
             async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
@@ -182,7 +181,6 @@ class gowitness(BaseModule):
             # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
             filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
-            self.hugewarning(event_dict)
             parent_event = event_dict[url]
             await self.emit_event(
                 webscreenshot_data,
@@ -259,9 +257,7 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 con.text_factory = self.helpers.smart_decode
                 async with con.execute("SELECT * FROM results") as cur:
-                    self.critical(f"CUR: {cur}")
                     async for row in cur:
-                        self.critical(f"SCREENSHOT: {row}")
                         row = dict(row)
                         _id = row["id"]
                         if _id not in self.screenshots_taken:
@@ -276,7 +272,6 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM network_logs") as cur:
                     async for row in cur:
-                        self.critical(f"NETWORK LOG: {row}")
                         row = dict(row)
                         url = row["url"]
                         if url not in self.connections_logged:
@@ -291,7 +286,6 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM technologies") as cur:
                     async for row in cur:
-                        self.critical(f"TECHNOLOGY: {row}")
                         _id = row["id"]
                         if _id not in self.technologies_found:
                             self.technologies_found.add(_id)

bbot/modules/graphql_introspection.py

@@ -27,7 +27,6 @@ class graphql_introspection(BaseModule):
             self.output_dir = Path(output_folder) / "graphql-schemas"
         else:
             self.output_dir = self.scan.home / "graphql-schemas"
-        self.helpers.mkdir(self.output_dir)
         return True
 
     async def filter_event(self, event):
@@ -120,7 +119,10 @@ fragment TypeRef on __Type {
             }
             response = await self.helpers.request(**request_args)
             if not response or response.status_code != 200:
-                self.debug(f"Failed to get GraphQL schema for {url} (status code {response.status_code})")
+                self.debug(
+                    f"Failed to get GraphQL schema for {url} "
+                    f"{f'(status code {response.status_code})' if response else ''}"
+                )
                 continue
             try:
                 response_json = response.json()
@@ -128,6 +130,7 @@ fragment TypeRef on __Type {
                 self.debug(f"Failed to parse JSON for {url}")
                 continue
             if response_json.get("data", {}).get("__schema", {}).get("types", []):
+                self.helpers.mkdir(self.output_dir)
                 filename = f"schema-{self.helpers.tagify(url)}.json"
                 filename = self.output_dir / filename
                 with open(filename, "w") as f:

bbot/modules/httpx.py

@@ -50,6 +50,8 @@ class httpx(BaseModule):
     _shuffle_incoming_queue = False
     _batch_size = 500
     _priority = 2
+    # accept Javascript URLs
+    accept_url_special = True
 
     async def setup(self):
         self.threads = self.config.get("threads", 50)

bbot/modules/iis_shortnames.py

@@ -116,13 +116,6 @@ class iis_shortnames(BaseModule):
 
         return duplicates
 
-    async def threaded_request(self, method, url, affirmative_status_code, c):
-        r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10)
-        if r is not None:
-            if r.status_code == affirmative_status_code:
-                return True, c
-        return None, c
-
     async def solve_valid_chars(self, method, target, affirmative_status_code):
         confirmed_chars = []
         confirmed_exts = []

bbot/modules/internal/unarchive.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from contextlib import suppress
 from bbot.modules.internal.base import BaseInternalModule
 from bbot.core.helpers.libmagic import get_magic_info, get_compression
 
@@ -62,15 +63,20 @@ class unarchive(BaseInternalModule):
                 context=f'extracted "{path}" to: {output_dir}',
             )
         else:
-            output_dir.rmdir()
+            with suppress(OSError):
+                output_dir.rmdir()
 
     async def extract_file(self, path, output_dir):
         extension, mime_type, description, confidence = get_magic_info(path)
         compression_format = get_compression(mime_type)
         cmd_list = self.compression_methods.get(compression_format, [])
         if cmd_list:
-            if not output_dir.exists():
-                self.helpers.mkdir(output_dir)
+            # output dir must not already exist
+            try:
+                output_dir.mkdir(exist_ok=False)
+            except FileExistsError:
+                self.warning(f"Destination directory {output_dir} already exists, aborting unarchive for {path}")
+                return False
             command = [s.format(filename=path, extract_dir=output_dir) for s in cmd_list]
             try:
                 await self.run_process(command, check=True)

bbot/modules/lightfuzz/lightfuzz.py

@@ -34,6 +34,7 @@ class lightfuzz(BaseModule):
         self.event_dict = {}
         self.interactsh_subdomain_tags = {}
         self.interactsh_instance = None
+        self.interactsh_domain = None
         self.disable_post = self.config.get("disable_post", False)
         self.enabled_submodules = self.config.get("enabled_submodules")
         self.interactsh_disable = self.scan.config.get("interactsh_disable", False)
@@ -51,13 +52,16 @@ class lightfuzz(BaseModule):
             self.submodules[submodule_name] = submodule_class
 
         interactsh_needed = any(submodule.uses_interactsh for submodule in self.submodules.values())
-
         if interactsh_needed and not self.interactsh_disable:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
+                if not self.interactsh_domain:
+                    self.warning("Interactsh failure: No domain returned from self.interactsh_instance.register()")
+                    self.interactsh_instance = None
             except InteractshError as e:
                 self.warning(f"Interactsh failure: {e}")
+                self.interactsh_instance = None
         return True
 
     async def interactsh_callback(self, r):

bbot/modules/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.4.7",
+        "version": "3.4.10",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/modules/output/base.py

@@ -38,11 +38,6 @@ class BaseOutputModule(BaseModule):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, (f"Omitting {event} from output because it's marked as httpx-only")
-
         # omit certain event types
         if event._omit:
             if "target" in event.tags:

bbot/modules/postman_download.py

@@ -7,7 +7,7 @@ from bbot.modules.templates.postman import postman
 class postman_download(postman):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum", "download"]
     meta = {
         "description": "Download workspaces, collections, requests from Postman",
         "created_date": "2024-09-07",

bbot/modules/retirejs.py

@@ -0,0 +1,232 @@
+import json
+from enum import IntEnum
+from bbot.modules.base import BaseModule
+
+
+class RetireJSSeverity(IntEnum):
+    NONE = 0
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+    CRITICAL = 4
+
+    @classmethod
+    def from_string(cls, severity_str):
+        try:
+            return cls[severity_str.upper()]
+        except (KeyError, AttributeError):
+            return cls.NONE
+
+
+class retirejs(BaseModule):
+    watched_events = ["URL_UNVERIFIED"]
+    produced_events = ["FINDING"]
+    flags = ["active", "safe", "web-thorough"]
+    meta = {
+        "description": "Detect vulnerable/out-of-date JavaScript libraries",
+        "created_date": "2025-08-19",
+        "author": "@liquidsec",
+    }
+    options = {
+        "version": "5.3.0",
+        "node_version": "18.19.1",
+        "severity": "medium",
+    }
+    options_desc = {
+        "version": "retire.js version",
+        "node_version": "Node.js version to install locally",
+        "severity": "Minimum severity level to report (none, low, medium, high, critical)",
+    }
+
+    deps_ansible = [
+        # Download Node.js binary (Linux x64)
+        {
+            "name": "Download Node.js binary (Linux x64)",
+            "get_url": {
+                "url": "https://nodejs.org/dist/v#{BBOT_MODULES_RETIREJS_NODE_VERSION}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "dest": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "mode": "0644",
+            },
+        },
+        # Extract Node.js binary (x64)
+        {
+            "name": "Extract Node.js binary (x64)",
+            "unarchive": {
+                "src": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz",
+                "dest": "#{BBOT_TOOLS}",
+                "remote_src": True,
+            },
+        },
+        # Remove existing node directory if it exists
+        {
+            "name": "Remove existing node directory",
+            "file": {"path": "#{BBOT_TOOLS}/node", "state": "absent"},
+        },
+        # Rename extracted directory to 'node' (x64)
+        {
+            "name": "Rename Node.js directory (x64)",
+            "command": "mv #{BBOT_TOOLS}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64 #{BBOT_TOOLS}/node",
+        },
+        # Set permissions on entire Node.js bin directory
+        {
+            "name": "Set permissions on Node.js bin directory",
+            "file": {"path": "#{BBOT_TOOLS}/node/bin", "mode": "0755", "recurse": "yes"},
+        },
+        # Make Node.js binary executable
+        {
+            "name": "Make Node.js binary executable",
+            "file": {"path": "#{BBOT_TOOLS}/node/bin/node", "mode": "0755"},
+        },
+        # Remove existing retirejs directory if it exists
+        {
+            "name": "Remove existing retirejs directory",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "absent"},
+        },
+        # Create retire.js local directory
+        {
+            "name": "Create retire.js directory in BBOT_TOOLS",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "directory", "mode": "0755"},
+        },
+        # Install retire.js locally using local Node.js
+        {
+            "name": "Install retire.js locally",
+            "shell": "cd #{BBOT_TOOLS}/retirejs && #{BBOT_TOOLS}/node/bin/node #{BBOT_TOOLS}/node/lib/node_modules/npm/bin/npm-cli.js install --prefix . retire@#{BBOT_MODULES_RETIREJS_VERSION} --no-fund --no-audit --silent --no-optional",
+            "args": {"creates": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire"},
+            "timeout": 600,
+            "ignore_errors": False,
+        },
+        # Make retire script executable
+        {
+            "name": "Make retire script executable",
+            "file": {"path": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire", "mode": "0755"},
+        },
+        # Create retire cache directory
+        {
+            "name": "Create retire cache directory",
+            "file": {"path": "#{BBOT_CACHE}/retire_cache", "state": "directory", "mode": "0755"},
+        },
+    ]
+
+    accept_url_special = True
+    scope_distance_modifier = 1
+    _module_threads = 4
+
+    async def setup(self):
+        excavate_enabled = self.scan.config.get("excavate")
+        if not excavate_enabled:
+            return None, "retirejs will not function without excavate enabled"
+
+        # Validate severity level
+        valid_severities = ["none", "low", "medium", "high", "critical"]
+        configured_severity = self.config.get("severity", "medium").lower()
+        if configured_severity not in valid_severities:
+            return (
+                False,
+                f"Invalid severity level '{configured_severity}'. Valid options are: {', '.join(valid_severities)}",
+            )
+
+        self.repofile = await self.helpers.download(
+            "https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository-v4.json", cache_hrs=24
+        )
+        if not self.repofile:
+            return False, "failed to download retire.js repository file"
+        return True
+
+    async def handle_event(self, event):
+        js_file = await self.helpers.request(event.data)
+        if js_file:
+            js_file_body = js_file.text
+            if js_file_body:
+                js_file_body_saved = self.helpers.tempfile(js_file_body, pipe=False, extension="js")
+                results = await self.execute_retirejs(js_file_body_saved)
+                if not results:
+                    self.warning("no output from retire.js")
+                    return
+                results_json = json.loads(results)
+                if results_json.get("data"):
+                    for file_result in results_json["data"]:
+                        for component_result in file_result.get("results", []):
+                            component = component_result.get("component", "unknown")
+                            version = component_result.get("version", "unknown")
+                            vulnerabilities = component_result.get("vulnerabilities", [])
+                            for vuln in vulnerabilities:
+                                severity = vuln.get("severity", "unknown")
+
+                                # Filter by minimum severity level
+                                min_severity = RetireJSSeverity.from_string(self.config.get("severity", "medium"))
+                                vuln_severity = RetireJSSeverity.from_string(severity)
+                                if vuln_severity < min_severity:
+                                    self.debug(
+                                        f"Skipping vulnerability with severity '{severity}' (below minimum '{min_severity.name.lower()}')"
+                                    )
+                                    continue
+
+                                identifiers = vuln.get("identifiers", {})
+                                summary = identifiers.get("summary", "Unknown vulnerability")
+                                cves = identifiers.get("CVE", [])
+                                description_parts = [
+                                    f"Vulnerable JavaScript library detected: {component} v{version}",
+                                    f"Severity: {severity.upper()}",
+                                    f"Summary: {summary}",
+                                    f"JavaScript URL: {event.data}",
+                                ]
+                                if cves:
+                                    description_parts.append(f"CVE(s): {', '.join(cves)}")
+
+                                below_version = vuln.get("below", "")
+                                at_or_above = vuln.get("atOrAbove", "")
+                                if at_or_above and below_version:
+                                    description_parts.append(f"Affected versions: [{at_or_above} to {below_version})")
+                                elif below_version:
+                                    description_parts.append(f"Affected versions: [< {below_version}]")
+                                elif at_or_above:
+                                    description_parts.append(f"Affected versions: [>= {at_or_above}]")
+                                description = " ".join(description_parts)
+                                data = {
+                                    "description": description,
+                                    "severity": severity,
+                                    "component": component,
+                                    "url": event.parent.data["url"],
+                                }
+                                await self.emit_event(
+                                    data,
+                                    "FINDING",
+                                    parent=event,
+                                    context=f"{{module}} identified vulnerable JavaScript library {component} v{version} ({severity} severity)",
+                                )
+
+    async def filter_event(self, event):
+        url_extension = getattr(event, "url_extension", "")
+        if url_extension != "js":
+            return False, f"it is a {url_extension} URL but retirejs only accepts js URLs"
+        return True
+
+    async def execute_retirejs(self, js_file):
+        cache_dir = self.helpers.cache_dir / "retire_cache"
+        retire_dir = self.scan.helpers.tools_dir / "retirejs"
+        local_node_dir = self.scan.helpers.tools_dir / "node"
+
+        # Use the retire binary directly with our local Node.js
+        retire_binary_path = retire_dir / "node_modules" / ".bin" / "retire"
+        command = [
+            str(local_node_dir / "bin" / "node"),
+            str(retire_binary_path),
+            "--outputformat",
+            "json",
+            "--cachedir",
+            str(cache_dir),
+            "--path",
+            js_file,
+            "--jsrepo",
+            str(self.repofile),
+        ]
+
+        proxy = self.scan.web_config.get("http_proxy")
+        if proxy:
+            command.extend(["--proxy", proxy])
+
+        self.verbose(f"Running retire.js on {js_file}")
+        self.verbose(f"retire.js command: {command}")
+
+        result = await self.run_process(command)
+        return result.stdout

bbot/modules/securitytxt.py

@@ -123,6 +123,3 @@ class securitytxt(BaseModule):
 
                         if found_url != url and self._urls is True:
                             await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags)
-
-
-# EOF

bbot/modules/subdomaincenter.py

@@ -12,25 +12,10 @@ class subdomaincenter(subdomain_enum):
     }
 
     base_url = "https://api.subdomain.center"
-    retries = 2
-
-    async def sleep(self, time_to_wait):
-        self.info(f"Sleeping for {time_to_wait} seconds to avoid rate limit")
-        await self.helpers.sleep(time_to_wait)
 
     async def request_url(self, query):
         url = f"{self.base_url}/?domain={self.helpers.quote(query)}"
-        response = None
-        status_code = 0
-        for i, _ in enumerate(range(self.retries + 1)):
-            if i > 0:
-                self.verbose(f"Retry #{i} for {query} after response code {status_code}")
-            response = await self.helpers.request(url, timeout=self.http_timeout + 30)
-            status_code = getattr(response, "status_code", 0)
-            if status_code == 429:
-                await self.sleep(20)
-            else:
-                break
+        response = await self.api_request(url)
         return response
 
     async def parse_results(self, r, query):

bbot/modules/telerik.py

@@ -204,7 +204,7 @@ class telerik(BaseModule):
             webresource = "Telerik.Web.UI.WebResource.axd?type=rau"
             result, _ = await self.test_detector(base_url, webresource)
             if result:
-                if "RadAsyncUpload handler is registered successfully" in result.text:
+                if "RadAsyncUpload handler is registered succesfully" in result.text:
                     self.verbose("Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)")
 
                     probe_data = {
@@ -263,6 +263,11 @@ class telerik(BaseModule):
                                     str(root_tool_path / "testfile.txt"),
                                     result.url,
                                 ]
+
+                                # Add proxy if set in the scan config
+                                if self.scan.http_proxy:
+                                    command.append(self.scan.http_proxy)
+
                                 output = await self.run_process(command)
                                 description = f"[CVE-2017-11317] [{str(version)}] {webresource}"
                                 if "fileInfo" in output.stdout: