bbot 2.3.0.5489rc0__py3-none-any.whl → 2.3.0.5504rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5489rc"
+__version__ = "v2.3.0.5504rc"
 
 from .scanner import Scanner, Preset

bbot/core/event/base.py

@@ -515,22 +515,25 @@ class BaseEvent:
             new_scope_distance = min(self.scope_distance, scope_distance)
         if self._scope_distance != new_scope_distance:
             # remove old scope distance tags
-            for t in list(self.tags):
-                if t.startswith("distance-"):
-                    self.remove_tag(t)
-            if self.host:
-                if scope_distance == 0:
-                    self.add_tag("in-scope")
-                    self.remove_tag("affiliate")
-                else:
-                    self.remove_tag("in-scope")
-                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
+            self.refresh_scope_tags()
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
             if parent_scope_distance is not None and self.parent is not self:
                 self.parent.scope_distance = new_scope_distance + 1
 
+    def refresh_scope_tags(self):
+        for t in list(self.tags):
+            if t.startswith("distance-"):
+                self.remove_tag(t)
+        if self.host:
+            if self.scope_distance == 0:
+                self.add_tag("in-scope")
+                self.remove_tag("affiliate")
+            else:
+                self.remove_tag("in-scope")
+                self.add_tag(f"distance-{self.scope_distance}")
+
     @property
     def scope_description(self):
         """
@@ -1352,18 +1355,22 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
         self.parsed_url = self.validators.validate_url_parsed(url)
         data["url"] = self.parsed_url.geturl()
 
-        header_dict = {}
-        for i in data.get("raw_header", "").splitlines():
-            if len(i) > 0 and ":" in i:
-                k, v = i.split(":", 1)
-                k = k.strip().lower()
-                v = v.lstrip()
-                if k in header_dict:
-                    header_dict[k].append(v)
-                else:
-                    header_dict[k] = [v]
+        if not "raw_header" in data:
+            raise ValueError("raw_header is required for HTTP_RESPONSE events")
+
+        if "header-dict" not in data:
+            header_dict = {}
+            for i in data.get("raw_header", "").splitlines():
+                if len(i) > 0 and ":" in i:
+                    k, v = i.split(":", 1)
+                    k = k.strip().lower()
+                    v = v.lstrip()
+                    if k in header_dict:
+                        header_dict[k].append(v)
+                    else:
+                        header_dict[k] = [v]
+            data["header-dict"] = header_dict
 
-        data["header-dict"] = header_dict
         # move URL to the front of the dictionary for visibility
         data = dict(data)
         new_data = {"url": data.pop("url")}
@@ -1377,6 +1384,13 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
     def _pretty_string(self):
         return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
 
+    @property
+    def raw_response(self):
+        """
+        Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.
+        """
+        return f'{self.data["raw_header"]}{self.data["body"]}'
+
     @property
     def http_status(self):
         try:

bbot/core/helpers/misc.py

@@ -559,13 +559,12 @@ def is_port(p):
     return p and p.isdigit() and 0 <= int(p) <= 65535
 
 
-def is_dns_name(d, include_local=True):
+def is_dns_name(d):
     """
     Determines if the given string is a valid DNS name.
 
     Args:
         d (str): The string to be checked.
-        include_local (bool): Consider local hostnames to be valid (hostnames without periods)
 
     Returns:
         bool: True if the string is a valid DNS name, False otherwise.
@@ -575,17 +574,12 @@ def is_dns_name(d, include_local=True):
         True
         >>> is_dns_name('localhost')
         True
-        >>> is_dns_name('localhost', include_local=False)
-        False
         >>> is_dns_name('192.168.1.1')
         False
     """
     if is_ip(d):
         return False
     d = smart_decode(d)
-    if include_local:
-        if bbot_regexes.hostname_regex.match(d):
-            return True
     if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False

bbot/core/helpers/regexes.py

@@ -39,14 +39,10 @@ _ip_range_regexes = (
 ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
 
 # dns names with periods
-_dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
+_dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.?)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
 dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
 dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
-# dns names without periods
-_hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"
-hostname_regex = re.compile(r"^" + _hostname_regex + r"$", re.I)
-
 _email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
 email_regex = re.compile(_email_regex, re.I)
 
@@ -61,14 +57,12 @@ event_uuid_regex = re.compile(_event_uuid_regex, re.I)
 
 _open_port_regexes = (
     _dns_name_regex + r":[0-9]{1,5}",
-    _hostname_regex + r":[0-9]{1,5}",
     r"\[" + _ipv6_regex + r"\]:[0-9]{1,5}",
 )
 open_port_regexes = [re.compile(r, re.I) for r in _open_port_regexes]
 
 _url_regexes = (
     r"https?://" + _dns_name_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
-    r"https?://" + _hostname_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
     r"https?://\[" + _ipv6_regex + r"\](?::[0-9]{1,5})?(?:(?:/|\?).*)?",
 )
 url_regexes = [re.compile(r, re.I) for r in _url_regexes]
@@ -83,10 +77,7 @@ event_type_regexes = OrderedDict(
         for k, regexes in (
             (
                 "DNS_NAME",
-                (
-                    r"^" + _dns_name_regex + r"$",
-                    r"^" + _hostname_regex + r"$",
-                ),
+                (r"^" + _dns_name_regex + r"$",),
             ),
             (
                 "EMAIL_ADDRESS",
@@ -140,7 +131,7 @@ select_tag_regex = re.compile(
 textarea_tag_regex = re.compile(
     r'<textarea[^>]*\bname=["\']?(\w+)["\']?[^>]*>(.*?)</textarea>', re.IGNORECASE | re.DOTALL
 )
-tag_attribute_regex = re.compile(r"<[^>]*(?:href|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>")
+tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\s\'\"\>]+)[\"\']?[^>]*>")
 
 valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+"
 

bbot/modules/dnsbrute_mutations.py

@@ -1,3 +1,5 @@
+import time
+
 from bbot.modules.base import BaseModule
 
 
@@ -40,8 +42,11 @@ class dnsbrute_mutations(BaseModule):
                 except KeyError:
                     self.found[domain] = {subdomain}
 
-    def get_parent_event(self, subdomain):
-        parent_host = self.helpers.closest_match(subdomain, self.parent_events)
+    async def get_parent_event(self, subdomain):
+        start = time.time()
+        parent_host = await self.helpers.run_in_executor(self.helpers.closest_match, subdomain, self.parent_events)
+        elapsed = time.time() - start
+        self.trace(f"{subdomain}: got closest match among {len(self.parent_events):,} parent events in {elapsed:.2f}s")
         return self.parent_events[parent_host]
 
     async def finish(self):
@@ -124,7 +129,7 @@ class dnsbrute_mutations(BaseModule):
                             self._mutation_run_counter[domain] = mutation_run = 1
                         self._mutation_run_counter[domain] += 1
                         for hostname in results:
-                            parent_event = self.get_parent_event(hostname)
+                            parent_event = await self.get_parent_event(hostname)
                             mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)
                             await self.emit_event(
                                 hostname,

bbot/modules/extractous.py

@@ -28,6 +28,7 @@ class extractous(BaseModule):
             "ica",  #  Citrix Independent Computing Architecture File
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
+            "json",  #  JSON File
             "key",  #  Private Key File
             "pub",  #  Public Key File
             "log",  #  Log File
@@ -45,6 +46,7 @@ class extractous(BaseModule):
             "pptx",  #  Microsoft PowerPoint Presentation
             "ps1",  #  PowerShell Script
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "swp",  #  Swap File (temporary file, often Vim)

bbot/modules/filedownload.py

@@ -38,6 +38,7 @@ class filedownload(BaseModule):
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
             "jar",  #  Java Archive
+            "json",  #  JSON File
             "key",  #  Private Key File
             "log",  #  Log File
             "markdown",  #  Markdown File
@@ -57,6 +58,7 @@ class filedownload(BaseModule):
             "pub",  #  Public Key File
             "raw",  #  Raw Image File Format
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "sqlite",  #  SQLite Database File

bbot/modules/github_org.py

@@ -206,11 +206,7 @@ class github_org(github):
         for k, v in json.items():
             if (
                 isinstance(v, str)
-                and (
-                    self.helpers.is_dns_name(v, include_local=False)
-                    or self.helpers.is_url(v)
-                    or self.helpers.is_email(v)
-                )
+                and (self.helpers.is_dns_name(v) and "." in v or self.helpers.is_url(v) or self.helpers.is_email(v))
                 and self.scan.in_scope(v)
             ):
                 self.verbose(f'Found in-scope key "{k}": "{v}" for {org}, it appears to be in-scope')

bbot/modules/internal/excavate.py

@@ -505,6 +505,11 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                             if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
                                 if self.excavate.in_bl(parameter_name) is False:
                                     parsed_url = urlparse(url)
+                                    if not parsed_url.hostname:
+                                        self.excavate.warning(
+                                            f"Error Parsing reconstructed URL [{url}] during parameter extraction, missing hostname"
+                                        )
+                                        continue
                                     description = f"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)"
                                     data = {
                                         "host": parsed_url.hostname,
@@ -703,7 +708,7 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                 """
             ),
         }
-        full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)")
+        full_url_regex = re.compile(r"(https?)://(\w(?:[\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)")
         full_url_regex_strict = re.compile(r"^(https?):\/\/([\w.-]+)(?::\d{1,5})?(\/[\w\/\.-]*)?(\?[^\s]+)?$")
         tag_attribute_regex = bbot_regexes.tag_attribute_regex
 

bbot/modules/trufflehog.py

@@ -3,7 +3,7 @@ from bbot.modules.base import BaseModule
 
 
 class trufflehog(BaseModule):
-    watched_events = ["CODE_REPOSITORY", "FILESYSTEM"]
+    watched_events = ["CODE_REPOSITORY", "FILESYSTEM", "HTTP_RESPONSE", "RAW_TEXT"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["passive", "safe", "code-enum"]
     meta = {
@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.87.1",
+        "version": "3.87.2",
         "config": "",
         "only_verified": True,
         "concurrency": 8,
@@ -81,12 +81,15 @@ class trufflehog(BaseModule):
         return True
 
     async def handle_event(self, event):
-        description = event.data.get("description", "")
+        description = ""
+        if isinstance(event.data, dict):
+            description = event.data.get("description", "")
+
         if event.type == "CODE_REPOSITORY":
             path = event.data["url"]
             if "git" in event.tags:
                 module = "github-experimental"
-        else:
+        elif event.type == "FILESYSTEM":
             path = event.data["path"]
             if "git" in event.tags:
                 module = "git"
@@ -96,6 +99,14 @@ class trufflehog(BaseModule):
                 module = "postman"
             else:
                 module = "filesystem"
+        elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            module = "filesystem"
+            file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
+            # write the response to a tempfile
+            # this is necessary because trufflehog doesn't yet support reading from stdin
+            # https://github.com/trufflesecurity/trufflehog/issues/162
+            path = self.helpers.tempfile(file_data, pipe=False)
+
         if event.type == "CODE_REPOSITORY":
             host = event.host
         else:
@@ -108,41 +119,32 @@ class trufflehog(BaseModule):
             verified,
             source_metadata,
         ) in self.execute_trufflehog(module, path):
-            if verified:
-                data = {
-                    "severity": "High",
-                    "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "VULNERABILITY",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
-                )
-            else:
-                data = {
-                    "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "FINDING",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found possible secret ({{event.type}}): {raw_result}',
-                )
-
-    async def execute_trufflehog(self, module, path):
+            verified_str = "Verified" if verified else "Possible"
+            finding_type = "VULNERABILITY" if verified else "FINDING"
+            data = {
+                "description": f"{verified_str} Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
+            }
+            if host:
+                data["host"] = host
+            if finding_type == "VULNERABILITY":
+                data["severity"] = "High"
+            if description:
+                data["description"] += f" Description: [{description}]"
+            data["description"] += f" Raw result: [{raw_result}]"
+            if rawv2_result:
+                data["description"] += f" RawV2 result: [{rawv2_result}]"
+            await self.emit_event(
+                data,
+                finding_type,
+                event,
+                context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',
+            )
+
+        # clean up the tempfile when we're done with it
+        if event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            path.unlink(missing_ok=True)
+
+    async def execute_trufflehog(self, module, path=None, string=None):
         command = [
             "trufflehog",
             "--json",

bbot/test/test_step_1/test_events.py

@@ -149,6 +149,7 @@ async def test_events(events, helpers):
             "title": "HTTP%20RESPONSE",
             "url": "http://www.evilcorp.com:80",
             "input": "http://www.evilcorp.com:80",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
             "location": "/asdf",
             "status_code": 301,
         },
@@ -161,7 +162,13 @@ async def test_events(events, helpers):
 
     # http response url validation
     http_response_2 = scan.make_event(
-        {"port": "80", "url": "http://evilcorp.com:80/asdf"}, "HTTP_RESPONSE", dummy=True
+        {
+            "port": "80",
+            "url": "http://evilcorp.com:80/asdf",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
+        "HTTP_RESPONSE",
+        dummy=True,
     )
     assert http_response_2.data["url"] == "http://evilcorp.com/asdf"
 
@@ -546,6 +553,10 @@ async def test_events(events, helpers):
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event)
     assert http_response.parent_id == scan.root_event.id
     assert http_response.data["input"] == "http://example.com:80"
+    assert (
+        http_response.raw_response
+        == 'HTTP/1.1 200 OK\r\nConnection: close\r\nAge: 526111\r\nCache-Control: max-age=604800\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Mon, 14 Nov 2022 17:14:27 GMT\r\nEtag: "3147526947+ident+gzip"\r\nExpires: Mon, 21 Nov 2022 17:14:27 GMT\r\nLast-Modified: Thu, 17 Oct 2019 07:18:26 GMT\r\nServer: ECS (agb/A445)\r\nVary: Accept-Encoding\r\nX-Cache: HIT\r\n\r\n<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n\n    <meta charset="utf-8" />\n    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n    <meta name="viewport" content="width=device-width, initial-scale=1" />\n    <style type="text/css">\n    body {\n        background-color: #f0f0f2;\n        margin: 0;\n        padding: 0;\n        font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n        \n    }\n    div {\n        width: 600px;\n        margin: 5em auto;\n        padding: 2em;\n        background-color: #fdfdff;\n        border-radius: 0.5em;\n        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n    }\n    a:link, a:visited {\n        color: #38488f;\n        text-decoration: none;\n    }\n    @media (max-width: 700px) {\n        div {\n            margin: 0 auto;\n            width: auto;\n        }\n    }\n    </style>    \n</head>\n\n<body>\n<div>\n    <h1>Example Domain</h1>\n    <p>This domain is for use in illustrative examples in documents. You may use this\n    domain in literature without prior coordination or asking for permission.</p>\n    <p><a href="https://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n'
+    )
     json_event = http_response.json(mode="graph")
     assert isinstance(json_event["data"], str)
     json_event = http_response.json()
@@ -906,7 +917,12 @@ def test_event_closest_host():
     assert event1.host == "evilcorp.com"
     # second event has a host + url
     event2 = scan.make_event(
-        {"method": "GET", "url": "http://www.evilcorp.com/asdf", "hash": {"header_mmh3": "1", "body_mmh3": "2"}},
+        {
+            "method": "GET",
+            "url": "http://www.evilcorp.com/asdf",
+            "hash": {"header_mmh3": "1", "body_mmh3": "2"},
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
         "HTTP_RESPONSE",
         parent=event1,
     )

bbot/test/test_step_1/test_helpers.py

@@ -122,7 +122,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert not helpers.is_dns_name("evilcorp.com:80")
     assert not helpers.is_dns_name("http://evilcorp.com:80")
     assert helpers.is_dns_name("evilcorp")
-    assert not helpers.is_dns_name("evilcorp", include_local=False)
+    assert helpers.is_dns_name("evilcorp.")
     assert helpers.is_dns_name("ドメイン.テスト")
     assert not helpers.is_dns_name("127.0.0.1")
     assert not helpers.is_dns_name("dead::beef")

bbot/test/test_step_1/test_regexes.py

@@ -267,7 +267,6 @@ def test_url_regexes():
         "http:///evilcorp.com",
         "http:// evilcorp.com",
         "http://evilcorp com",
-        "http://evilcorp.",
         "http://.com",
         "evilcorp.com",
         "http://ex..ample.com",
@@ -288,6 +287,7 @@ def test_url_regexes():
 
     good_urls = [
         "https://evilcorp.com",
+        "http://evilcorp.",
         "https://asdf.www.evilcorp.com",
         "https://asdf.www-test.evilcorp.com",
         "https://a.www-test.evilcorp.c",

bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -29,6 +29,7 @@ class TestExcavate(ModuleTestBase):
         # these ones should
         <a href="/a_relative.txt">
         <link href="/link_relative.txt">
+        <a href="mailto:bob@evilcorp.org?subject=help">Help</a>
         """
         expect_args = {"method": "GET", "uri": "/"}
         respond_args = {"response_data": response_data}
@@ -1010,3 +1011,29 @@ A href <a href='/donot_detect.js'>Click me</a>"""
         assert (
             "/donot_detect.js" not in url_events
         ), f"URL extracted from extractous text is incorrect, got {url_events}"
+
+
+class TestExcavateBadURLs(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888/"]
+    modules_overrides = ["excavate", "httpx", "hunt"]
+    config_overrides = {"interactsh_disable": True, "scope": {"report_distance": 10}}
+
+    bad_url_data = """
+<a href='mailto:bob@evilcorp.org?subject=help'>Help</a>
+<a href='https://ssl.'>Help</a>
+"""
+
+    async def setup_after_prep(self, module_test):
+        module_test.set_expect_requests({"uri": "/"}, {"response_data": self.bad_url_data})
+
+    def check(self, module_test, events):
+        log_file = module_test.scan.home / "debug.log"
+        log_text = log_file.read_text()
+        # make sure our logging is working
+        assert "Setting scan status to STARTING" in log_text
+        # make sure we don't have any URL validation errors
+        assert "Error Parsing reconstructed URL" not in log_text
+        assert "Error sanitizing event data" not in log_text
+
+        url_events = [e for e in events if e.type == "URL_UNVERIFIED"]
+        assert sorted([e.data for e in url_events]) == sorted(["https://ssl/", "http://127.0.0.1:8888/"])

bbot/test/test_step_2/module_tests/test_module_github_codesearch.py

@@ -3,17 +3,35 @@ from .base import ModuleTestBase
 
 class TestGithub_Codesearch(ModuleTestBase):
     config_overrides = {
-        "modules": {"github_codesearch": {"api_key": "asdf", "limit": 1}},
+        "modules": {
+            "github_codesearch": {"api_key": "asdf", "limit": 1},
+            "trufflehog": {"only_verified": False},
+        },
         "omit_event_types": [],
         "scope": {"report_distance": 2},
     }
-    modules_overrides = ["github_codesearch", "httpx", "secretsdb"]
+    modules_overrides = ["github_codesearch", "httpx", "trufflehog"]
 
     github_file_endpoint = (
         "/projectdiscovery/nuclei/06f242e5fce3439b7418877676810cbf57934875/v2/cmd/cve-annotate/main.go"
     )
     github_file_url = f"http://127.0.0.1:8888{github_file_endpoint}"
-    github_file_content = "-----BEGIN PGP PRIVATE KEY BLOCK-----"
+    github_file_content = """-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOBY2pd9PSQvuxqu
+WXFNVgILTWuUc721Wc2sFNvp4beowhUe1lfxaq5ZfCJcz7z4QsqFhOeks69O9UIb
+oiOTDocPDog9PHO8yZXopHm0StFZvSjjKSNuFvy/WopPTGpxUZ5boCaF1CXumY7W
+FL+jIap5faimLL9prIwaQKBwv80lAgMBAAECgYEAxvpHtgCgD849tqZYMgOTevCn
+U/kwxltoMOClB39icNA+gxj8prc6FTTMwnVq0oGmS5UskX8k1yHCqUV1AvRU9o+q
+I8L8a3F3TQKQieI/YjiUNK8A87bKkaiN65ooOnhT+I3ZjZMPR5YEyycimMp22jsv
+LyX/35J/wf1rNiBs/YECQQDvtxgmMhE+PeajXqw1w2C3Jds27hI3RPDnamEyWr/L
+KkSplbKTF6FuFDYOFdJNPrfxm1tx2MZ2cBfs+h/GnCJVAkEA75Z9w7q8obbqGBHW
+9bpuFvLjW7bbqO7HBuXYX9zQcZL6GSArFP0ba5lhgH1qsVQfxVWVyiV9/chme7xc
+ljfvkQJBAJ7MpSPQcRnRefNp6R0ok+5gFqt55PlWI1y6XS81bO7Szm+laooE0n0Q
+yIpmLE3dqY9VgquVlkupkD/9poU0s40CQD118ZVAVht1/N9n1Cj9RjiE3mYspnTT
+rCLM25Db6Gz6M0Y2xlaAB4S2uBhqE/Chj/TjW6WbsJJl0kRzsZynhMECQFYKiM1C
+T4LB26ynW00VE8z4tEWSoYt4/Vn/5wFhalVjzoSJ8Hm2qZiObRYLQ1m0X4KnkShk
+Gnl54dJHT+EhlfY=
+-----END PRIVATE KEY-----"""
 
     async def setup_before_prep(self, module_test):
         expect_args = {"method": "GET", "uri": self.github_file_endpoint}
@@ -82,5 +100,5 @@ class TestGithub_Codesearch(ModuleTestBase):
             ]
         ), "Failed to visit URL"
         assert [
-            e for e in events if e.type == "FINDING" and str(e.module) == "secretsdb"
+            e for e in events if e.type == "FINDING" and str(e.module) == "trufflehog"
         ], "Failed to find secret in repo file"

bbot/test/test_step_2/module_tests/test_module_trufflehog.py

@@ -1193,7 +1193,7 @@ class TestTrufflehog_NonVerified(TestTrufflehog):
                 or e.data["host"] == "github.com"
                 or e.data["host"] == "www.postman.com"
             )
-            and "Potential Secret Found." in e.data["description"]
+            and "Possible Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
         # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.
@@ -1240,3 +1240,35 @@ class TestTrufflehog_NonVerified(TestTrufflehog):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Failed to find blacklanternsecurity postman workspace"
+
+
+class TestTrufflehog_HTTPResponse(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "trufflehog"]
+    config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
+
+    async def setup_before_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/"}
+        respond_args = {"response_data": "https://admin:admin@internal.host.com"}
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        assert any(e.type == "FINDING" for e in events)
+
+
+class TestTrufflehog_RAWText(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888/test.pdf"]
+    modules_overrides = ["httpx", "trufflehog", "filedownload", "extractous"]
+    config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
+
+    async def setup_before_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/test.pdf"}
+        respond_args = {
+            "response_data": b"%PDF-1.4\n%\xc7\xec\x8f\xa2\n%%Invocation: path/gs -P- -dSAFER -dCompatibilityLevel=1.4 -dWriteXRefStm=false -dWriteObjStms=false -q -P- -dNOPAUSE -dBATCH -sDEVICE=pdfwrite -sstdout=? -sOutputFile=? -P- -dSAFER -dCompatibilityLevel=1.4 -dWriteXRefStm=false -dWriteObjStms=false -\n5 0 obj\n<</Length 6 0 R/Filter /FlateDecode>>\nstream\nx\x9c-\x8c\xb1\x0e\x82@\x10D\xfb\xfd\x8a-\xa1\xe0\xd8\xe5@\xe1*c\xb4\xb1\xd3lba,\xc8\x81\x82\xf1@\xe4\xfe?\x02\x92If\x92\x97\x99\x19\x90\x14#\xcdZ\xd3: |\xc2\x00\xbcP\\\xc3:\xdc\x0b\xc4\x97\xed\x0c\xe4\x01\xff2\xe36\xc5\x9c6Jk\x8d\xe2\xe0\x16\\\xeb\n\x0f\xb5E\xce\x913\x93\x15F3&\x94\xa4a\x94fD\x01\x87w9M7\xc5z3Q\x8cx\xd9'(\x15\x04\x8d\xf7\x9f\xd1\xc4qY\xb9\xb63\x8b\xef\xda\xce\xd7\xdf\xae|\xab\xa6\x1f\xbd\xb2\xbd\x0b\xe5\x05G\x81\xf3\xa4\x1f~q-\xc7endstream\nendobj\n6 0 obj\n155\nendobj\n4 0 obj\n<</Type/Page/MediaBox [0 0 595 842]\n/Rotate 0/Parent 3 0 R\n/Resources<</ProcSet[/PDF /Text]\n/Font 11 0 R\n>>\n/Contents 5 0 R\n>>\nendobj\n3 0 obj\n<< /Type /Pages /Kids [\n4 0 R\n] /Count 1\n>>\nendobj\n1 0 obj\n<</Type /Catalog /Pages 3 0 R\n/Metadata 14 0 R\n>>\nendobj\n11 0 obj\n<</R9\n9 0 R/R7\n7 0 R>>\nendobj\n9 0 obj\n<</BaseFont/YTNPVC+Courier/FontDescriptor 10 0 R/Type/Font\n/FirstChar 46/LastChar 116/Widths[ 600 600\n0 0 0 0 0 0 0 0 0 0 600 0 0 0 0 0\n600 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n0 600 0 600 600 600 0 0 600 600 0 0 600 600 600 600\n600 0 600 600 600]\n/Encoding/WinAnsiEncoding/Subtype/Type1>>\nendobj\n7 0 obj\n<</BaseFont/NXCWXT+Courier-Bold/FontDescriptor 8 0 R/Type/Font\n/FirstChar 32/LastChar 101/Widths[\n600 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n600 600 600 600 600 600 0 0 600 600 600 0 0 0 0 0\n0 0 0 0 600 0 0 0 0 0 0 0 0 0 0 0\n0 0 0 0 0 0 0 600 0 0 0 0 0 0 0 0\n0 0 0 600 600 600]\n/Encoding/WinAnsiEncoding/Subtype/Type1>>\nendobj\n10 0 obj\n<</Type/FontDescriptor/FontName/YTNPVC+Courier/FontBBox[0 -182 599 665]/Flags 33\n/Ascent 665\n/CapHeight 665\n/Descent -182\n/ItalicAngle 0\n/StemV 89\n/AvgWidth 600\n/MaxWidth 600\n/MissingWidth 600\n/XHeight 433\n/CharSet(/a/at/c/colon/d/e/h/i/l/m/n/o/p/period/r/s/slash/t)/FontFile3 12 0 R>>\nendobj\n12 0 obj\n<</Filter/FlateDecode\n/Subtype/Type1C/Length 1794>>stream\nx\x9c\x9dT{TS\xf7\x1d\xbf\x91ps\x8f\xa0\xb2\xdc\x06\x1f\xe8\xbdX[|\xa0\x85\xaa\xad\xa7\xf4\x14P\x1eG9\x05\x9c\xa2\x08\xb4\xee@\x88\xc83\x08\x04\x84\x80\x84@B\xd3\x1f84!@\x12\x08\xe0\x8b\x97S\xe9\xc4U\xf4\x06\xb5\x15\xdd:5\xc8&j=\xb2\xad:'T9\xeb\xce\xbe\xb7\xe7\xban\xbf\x80\x16\xdb\xd3\xed\x8f\x9d\x93?n\xee\xe3\xf3\xfb~\x1e\xdf\x8f\x88\x10\xcf D\"\x11\x15\xa6T\xe5\xa5+\xf2\\\xd7\xabx\x1f\x11\xbfp\x06\xbf\xc8\r\tQ\xfc\xd8\xb7\xab\xdcy\xc6\x93\xa8\xf1\x14!O7\xe4)n_H\x19\xa4\xd0\xfb3\xa8\x9d\x03\xc5^\x84X$Z\x17\x9dd]\xb6mK\xfcr\x7f\xff\x95a\xca\xdc\xe2\xbc\xf4\xb4\xdd\x05\xbe\xab\x03\xdf\\\xeb\x9bR\xec\xfb\xfc\x89o\xb8\"?=-\xc7\xd7\x0f_\x14*\xb2\x94\xb9\xd9\x8a\x9c\x82\x98\xf4\xec\x14U\xbeo\xb42G\xe9\xbby\xab\xef\x16E\x9a*+9\xef\x87w\xa7\x11\xff\xbf3\x08\x82\x90\xe6(s\xf3\xf2\x0b\x92\xe5\xa9\x8a\xdd\xe9Y\xd9o\x04\x04\x85\x12D,\xb1\x99\xf89\xb1\x95\x88#\xb6\x11\x1b\x88p\"\x82\x88$6\x11QD4\x11C\xcc!\xbc\x08\x1fb1Acq\x081\xa1'\x06E\x1bE}3>\x9cq\xc1m\x93[\x9fx\x89\xb8P\x0c\xee\x91\xee\x95\xe4\xab\xe4zRIvJ\xd6\xf3\xe3\xb3\xf9q\xc4\xc1}N:\x08\xee\xf1\x0eht\xcc\xa5Ga=\xbfN\x16D\xaa**KJ\xcc\xdaV\x96\x1e\xe9\x10\x9crR\xa5\xd1\xaaK\x1a\xf0\x7f\x98G\xb6\x9aM6\xab\xc6T\xc8\xcaAG^\xf9\xe3a\xcb\x15t\x02\xb5\xe8\xda\x8a\x0f\x155\x14\xa0\\J\xa8PJ\xa6\xdf\x17\x91\xf6\x86\xe7\xef\xe7\xc0G\xe4\xed\x88\xc1\x00\x86\x1e\x8dAi\xc5\xdb\xb7Rx\x025\x07O9\xd15\x07\xfc\xdb\xe1\x06\x9f\xf1\x112a\xc1k\xcb\x05Z\xf0\xfaf)x\x83\xf7\xdf\x9f\x80\x14\xe6\xbc6!\xd0\xacn\x87\xec\x9b\xbb\xa1\xcb\xfc\xdf\r\xf6\xf3\x0b\x1a\x19\x7f|\xf7\xf6\x13\x16\x03\x08Q\x1c,\xe6`\x90\xdb\xc5Im0\x1f\x13\xf9\x1a\x13y\x04+0\x11\xbf\x97\x88|u\xeeYu\"I?*t\x8d\xe6\xba\x03\xdb\xc8\xb6)**\x96~\x18\x00\x05\xe4\xa7[.\xee\x19F\x14H\xc7\x1f\x81\x07K/\x00O\xff\x87\xc2+\xeb\x93\xf2cv0t\"\x04\x1f\x97=\xb9\x15\x11\xb8:$\xdc\x7fE\xc8\xd0\x83\xbf\xdc\xba\xf97vJC'\x97\xc2I\xe1\x17\xf8\xdc\x1b`\xc4\xe7\n\xb3\xc8\xc2r\xadZ\xddP\xd1\xca\xde\x10\x9c\x81\xf8_E\xe9\x94\x1e\xceI=,\xe5\xf5E\xac\xb0\x01RI:p\x1c\x88\x9e\xb6>\x1f;j\xd6\x1e\xca7V\xed7\x98\x10e1\x9b\xad\xf5:\xd3^\x0b\x9b\xdb\xae2e\xa1x\xf4\xc1\x9e5\xefM\xe9\xb5\xdb\x0e\xdfq\xe9v)x\\\x82\xc3\x97\xe6\xd2\xef\xc3\n\x98)\xb3j\xcc\xa5%ZM!\x13$)4ilV\x93\xd9\xce\xd0=Y\xa7\x06\xd4W|`\xe6\xfdKwN\x14\xfd*\xb3\x95\xcdh\xdbe\x8e>\xb0\xa6^_\xa3j,6k,\xa8\x89\xea\x1d\xe8\xb89|>7\xa5\x8e\xa9-6j-\x88\xb2\x99\xcc\xad\xecu\t\xbd\xb0UkV\x97UT\x94\x1a0\xd2\x91\xf4\x9d\x8d\xdb|\xfcB\x137f4gu\x16\xb3\x1d\xc5\x1dU\x7f\xa8\xba\xa8;\xa2;Rzx\x9fU\x85\n\xa9\xc4\xf7\xd3\xde~g\xe3\xf1\xd3\xcc\x94\xad\x7f\xe2D\xe0\x8bM\x8d\xc3\x82\x80X\xd2\xaa\xad/\xc1\x03\x161\x828\x12\xe7c\xd2\x966\xac\x8e\x99\x0c\xf9m\xc2\xd7g/\x99\x9b\xfb\x99\x93M\xd6Fd\xa1\x9a4\xe62}\xf5\xc7:-\x93\xaa\x8aT\xc7!jSJ\xe7Y\x16L\x90!q9f\xd3\x18U\xec\x94\x14\x1c\xbc\xc5\x81\x07'\xc5\xf9\xe9w\xc4\xc3\xfc\xb9t\x1e\xbf\xda{b:\xa3ti\"\x98\xc8\xe1\xf0\x01\x7fE\xd4\xbe\xbdqL\x99\xbe\xaa\x12\x95SefMc\xdd\xfe\x9a_62\x9f5\x9f6v#\xca\xd9\x9f\xbd\x93\x8d\x96\xc4Z\xf2\xf6\xefD\x94\xe0\xbd6v5Kk\x83\xbf\xd8>v\xe3b\xdb\xc0U,\xc0eqTl|A$\xa26&w\xf5\x7f\xee\xfc\xe4\xe9\x99~}e\x0f\xfb\"\xc2\xd8\x90;.\xff\xf9]\xbcL&\xef\xdan\xdb\x8ca\x16-_)\xcc\x17dc\x01\xe0s\xed\xf7-'\x06\xd8N\xbb\xa5\x19K\xde\xa81\xef\xab\xd4\x1b\xb4Z&\xe1\xc3\x98\x820D-\x0euN\xfccx\xe8\x9f\xf7\xae)\x12\x0e\xb0\xb5E\xc6\xca)\x1f\xec\xec\x03\t\x1d\x88}()\xa9\xc4\xde\xbe }\x7f\x92\xf4\xe7\x0ehvQ>\xc7\xd7\xf1Oq\xd6\xbfO\xf69a\x17\xb9s0\xb6+\x1c\x8f0g\xd9R\xc1K\xf0z\xe2\x07\xb3\x87\xaev_>\x83\x15\t\x9d\x90|\xafO\")\x14\xc1}\x9c\xeb\xd0e,\xdd\xe3\x1f\x1c\x8c\xa3=2>vk\xe4\xf1s\x17\xd7r\xb0\x90\x13\xf1\xed\x10/3J\x0eJ\xe0\x95\xa5\x8f\x85\x05\xc2\xbc\xd7W\t\xb3\x84y z\x1d\xd8q\xf0\xe8?\xe5\xb2LWm\xd0U2\xf2\xec0U,Z\x82\xde\xfb]\xd9\x18\xc5\x89m\xf7n^\xf8+z\x88\x86\xe3\xacA\xd4\x8b\xc6\xc1\xd3\x8b\xc0\xc3\x01M8\x1e!?\x9a\xfd\x99\xe1Gu\xd3\xf0|G\xe5PM\x1e\xed\xb4\xb5\x1c\xa8\xeb8t\xb4\xfe\x14\xeaEvW\xe9\xec\xc5\xa5\xa3\xc4\xa5#\x97Lo\xf6\x0f\xbe\xaa\"\xefE\x0e\xae\x8cM)\xda\x9e\xc4\xbcX\xd7\x07\xe0.\x85\x83\xce\x84\xc9\xa6\xb8\xe3\xda\xd8w\xa6\xab\x02\xdc\x05\xa7\x100=\x12|7\r\x87\xef\xd3\x13\x06\xfe\xba,Bpw\x92\x93p\xbc\x01\x939\x8a\x99\xdc\xc1L\x84uS\xc3\xbb\xb2\rn\xcf\x0c\xff\x03\xc7\xf5\xb1k\x95\xa5\x07@\xbc\x83\x835\xae\x9f\xab\x81g\xe2q\xde}\xa9\xb8n\xe0\x06\xce!\xe9Q\x17\x0en\x94\x16W\xa7b\x1c\xabm\xb2\xb8\xbeT\x82\x91<1\xd0\xd9~\x1cQ]\xc72w\xb3\xc2\xf5\xbb\xd3\xf6\xe6L>\xech\xefAT\xcf\xb1\xectV\x18\xba+y\xa9\x8f\x0f\x91W\x12\xce\xc7\xa4d\x97$\xc9\x99\xfc3\x99\xad\xc9\x88\xa2G\xe5(G\x9d\xa5pyUj\x17A?x\xc9\x923\xb3SS\xbb\xb3N\xb3f\xf2tw\xe7'\xbd\x99\x9d\xc9\xae\xdc\xf3\xeao\xc5\xb2\xba\xfa\x9aZTG5\x96\x9b\xcb\xca\xab\xf4\xa5U\x8c\xf0\xe5\xbfB\xaa+?\xaeF\xfa\xf9\xfb\x1a4M\r\x07\xeb,\x07\x99I0~\xd1O\xe1u\xf5N\xe2i\xe0\xec\x7f;'\xe6<\x04p\xbc''z\xea\x18u\x80\x97\xc3\x8d\x7f\x13^\x95\xf5\xe2%767T\x99\xca\xf7\xb3`\x97<\nw\xbe!Po\x0bn\xc2JFX#Aa-\xd1'w\x9c\x8c\xffM\xfeUD\xdd\x1e\xe99\x8eW\xaeT\xa77T\xeb\xd9=\xf9\x19\x9aD\x94\x842l{Nf\xf7\xa9/\xa2\xcb\x14\x04J@z\xf5\xab?\x7fq\xf6\x83(F.Y\xf2QX,ZGm\x18\x8c\xbbg6\xd5\xd461\xe7\xc5j\x83\x1eU *N\xd1\xfd\xe9\x85\x81_\x0f\xd5\xb0\xb3\xd5V\xfe-+x7\x1ck$\x1d39\x8f>\x93\xa7g\x9f\xd1s\x16A\xfc\x07\xbe\x9e\x12\xf0\nendstream\nendobj\n8 0 obj\n<</Type/FontDescriptor/FontName/NXCWXT+Courier-Bold/FontBBox[-14 -15 617 617]/Flags 131105\n/Ascent 617\n/CapHeight 566\n/Descent -15\n/ItalicAngle 0\n/StemV 92\n/AvgWidth 600\n/MaxWidth 600\n/MissingWidth 600\n/XHeight 437\n/CharSet(/D/W/c/colon/d/e/eight/five/four/nine/one/space/three/two/zero)/FontFile3 13 0 R>>\nendobj\n13 0 obj\n<</Filter/FlateDecode\n/Subtype/Type1C/Length 1758>>stream\nx\x9c\x9d\x93{PSg\x1a\xc6O\x80\x9c\x9c\xad\xb4\"\xd9S\xd4\xb6Iv\xba\xabh\x91\x11\xa4\xad\xbbu\xb7\xd3B\xcb\xb6\x16G\xc1\x16P\xa0\x18\x03$\x84\\ AHBX\x92p1\xbc\x04\xb9$\xe1\x12 @@B@.\xca\x1dA\xb7\x8a\x80\x8e\x8b\xbb\x9d\xae\xb3\xf62\xbb\xba[;[hw\xc3\xd4\xef\x8cGg\xf6$\xe8t\xf7\xdf\xfd\xeb\x9cy\xbfs\xde\xf7\xf9~\xcf\xf3\xb2\xb0\xa0\x00\x8c\xc5b=\x1b\xab(,\x90d\x15\xecy[\x91'\xf2\x15\"\xa8\x17X\xd4\x8b\x01\xd4K\x81\xfa\x12\xea1\xf5\x98M\xf1\x82\xb1\x9a`\x16\x04\x07BpP\xc7\x8b\x9c\x0b\xa1\xc8\xb3\x05\xc1f\xa4\r\xc1\x82X\xac\xd7\xdfOi\x0e\xff01y\xd7+\xafD\xc4*\x94\x9a\x02I\x8eX-\x88\xde\x1b\x15#\x10j\x04ON\x04qY*I\x8e\\\xb0\x83y9\x95\x95\xa7P\xca\xb2\xe4\xeaC\x12\x99\xb0P%HP\xc8\x15\x82\xc3I\x02\x9f\x80\xff-\xfd\xd8\xee\xff\x1b\x80a\xd8\xe6\xb8\x93\xa2\xac\xe4\xbdQ\xd1\xfbb^\x15\xec\xff\xe5\xaf0\xec\x17X\x1c\xf6\x0e\xf6.\xb6\x1f\xdb\x82\x85b\\\xec\xa7\x18\x89=\x8f\xb1\xb0m\xd8v\xec\x05,\x84\x81\x82\x05aE\x18\xc5r\x07\x04\x04X\x03\x1e\x04&\x05^\tJ\x0bZ`\xc7\xb3\xdfg/\xe1\xb1\xb8\x86Z}\x8eZ\x05/z\xe8eQ\x89\x08\x0b\xfc\xa3\x97\xcc\xaaV\x17C\x1eh\xad\xbaf\xa3\xad\xbc\xf5\xb4\x0b\x08\x94\x89\xa3\xe8*\x14\xf8\xef\x1a\x14ALr\x00\xed\xa19h\x13\xbd\xd3L\xd0b\\\t\xa6jC\x85\xce`\xd0\x82\xd6\xf7W\x8b\xd1Z\xde`\xee\xaa&\x10F?$\xd1\xc3\x1f8\xf7\xcf\xac\xbck\t'28\x10\x91p$\xfc\x0c\xc1\x8c,\xf1\xa2j/k\x8e\x99H\x8dQ89\xad\xeb\xcc),3\x15\x97\xf3\xb2\xda\x8fY\x8f\x02A\xef\x11\xec\xa6\xf9\x87;S\xc6D\xfc\xb9\xb4\xebEk\xf0\x19\xdc\xb0\x8f9';\xbb{\xe1,\xd1\xa7r\xc9J\rU&\x03\xefd\xae\xd4\xf8\x06\xf3='q\xf4\xcf_,^\xfafb\xc8\xa4\xeb\xe17\x95\xd7\x9bjuu\x85\xb5\x15\x8d\xe5V\x93\xa3\xa2\x05\xda\xc0\xd1hon\xb4Yl\xd0\xeb\x13P\xea\x8dr\xa2\x15o\xa8\x1bah\x02aa\xdc)j\x80\xfa\x9e\xa4\x83\xf1\xfc\xa7\xf7\xd1\x81\x06\xb4\x8d%-\x06{\xb9\xed\xf4Y \x9a~\x86\x8b\xdc\xa9\xad\x89\xf0\x1bH,J\xcbL\xcbT%\xc1\x07p\xd0\x954\x939\x93y\xb5\xe86,\xc0\x85\xa6\x8b\x1e\x82[,C\xc1\x1c\x17\xd8-\xd6:\x87\xcd\xd6\x06\xed\xe009\xf4\xb6\xb2\x06\xa3E\x01\xc4\xefp\xba\x1e\x95\x90\xb3\xe0)\xeb\xcbw\x15\xb6HAFp\xa7\xde:\x9c\x1a\x93\x9e\xdb\xd4\xa3\xe4\xa9\xba\xf5\x1e\x18\x00O\x8b\xc7\xd5}\xb6w\xc0>\x0b\x1b\xc0n\xdf\xff\x0bc\xd2<\xdaO\x8eq\xd0v:p\x8d\x8e\xa0w\xd1\xecp\x9a\xa4\xc3P@$\x8a\xfe\xd4\xdb\xe6\x9c\xe2\xf5\xd8\x9aZ\xa1\x93p\x17v\xcb\xcb\xca\xcc\xa7KyQ\xea\xfc\xaat\xd8\x0f\xa9\xae\x82K\x84\xe5>\xe9\x98^\x18X\x81\x15\xb8*mK\xf7u\x06'\x95\xe0e\xa1\xcb\xc8F~M\xdb\xd8\x88\xc0\x17)a\x7f][\x07\x9c\xdd\xc6\x08o\xd5\xdb\x9f\x08\xa7\xc3\x9e\xb21\x1a4>\xaf\x1b\x19\xaf\xed&\xbb\xb9\x17\x88\x8bx.m\x8cE\x1f\xb3i\x0c\x8f\xa5?\xceEF\xf6\x04\xeeC`\xfb\x11A+\x83\xa0\xd1\xf0\xa4\x93\x12\xca\x99NZ\x83Q\x07E\xa0ph\xfb\xab\x96\x1f\t\xb7\xa2gpF\x91\xdeK\xfd\xda\xcb\xba\xc38s\xca\x17\x90v\xf4\x1d\t\xf7\xe4wR\xe7s\x86\x8e\xb7\x1f\x81#p\\\x93#NM\x91\x1f\x80}D\x14\x07b\xdco\xcc\xa5\x0e\x8bg5\x0b\x8c\x03\xb3\xed\xc3Css\xee\xcf\xe1.A\xdf]%\xd7&\xaf\xdf\xba5\xf9\xc1.\xde\xcf9\xbb3\x0e\xc6\xc7g\xdcX\xe5m$\xfe\xae\x93\x85\xaa\x99\xf6\xe8\x01\xf5\x98\xa4e\x1f\x9d0\xe8\xf5 \xdf&\xebR\xf5\xd9jk\xea\x9c\xbc/;\xd9\x8f\xb6\xec\xe6\xe4\xffw\xbcuV\xed\xc6Rt3K\xf1\t>\xedj?\xe7\xbf\x17\xdfw1%\x10\xbb}\xf2a\x9d\x8ad\x9cz\xd9\xd7\\\xbeN\xa2f\x94\xe5\x1e\x84\xaf\x88\x07\x91_\xd0!\x87\x92\x8a\xc4B\x9eX\xa6L\x03)\xa1\xecQ\xbb\xbb\x9dM\xed\xf5<\xbb\xa7\xc6b\xb5u\xb9\x06[\xce\x03q}V\x9c\x96\xa7+\xde\x19\xc3\x17\xe6\xbc\x93H\x13Q\x15\x95[\x05\x94\xf0\x1e\x07\\fk\x85\xcd\xd0\xaa\xb5\x16\x83\x14\xb4\xba*1\xe1\xc7\x85\xbes^\xf3\x86R;\x11\xf6\xaa/\xca\xdf 7\xf5\x13R\xaa*\x94\xcb\x9d\xda!3\x7f\xcal7;M\xd3\x9a>)H\xe0T\x99ZW\x9a\xaf\xce1\xc6\xc3A\x90\xd7\xa9\x1cZ[\xa5\xa5\x14\x88<\xb5Z\x9e\xf2U.\n\xbdw\xb9yp\x8a?s\xce\xfd\t\\\x85\xc5\xec\xb9\xb8s\x04\xf7_\x8bC\xbd\xa3\xf3\xdba\xbcx\\\xea\x11\x8d$w\xc43&\x06\x86'\x1f\x91\xbb\xd4\xee\xd6\x96z\x9b\x95?0\xd8k\xfb=\x10\x7f\x18\xcf?!:)I\xe3\xfb)\xbb}\xd2X\xe8[\x9f\x8d\xc9\xd4\x1aI\xbf\x84\xd3U\x8fH\xf6\xeb\xa8G.\xe1\x14\x80\xd1l\xa8\xdc@KH\\\x9ai\x1e\xda\x8a\xcf\xf8\x99:\xf4V\xbe\xa1\xa1\xdcRXC\xb89\xe7k\xba:\x98\x8d\xf0/\x91\xa1\xde_\xa4\xb1\xe7i\x1e\x8ex(\x97\xbdA \xdf\xfbW&\xc4\x1c&3\x19>\xee*\xaa\x92D\xc7\xf0.h\xb14>M`\x9b?\x81\r~\xa3\xe8kt\x1f\x9e\xdb\xad\xf2\xd8\xcf\xd44\xb4\xf0\xc6\x9c\xd3\xcd\x1e nNd\xc4\xbf\x95.\xd9\xf1\x9e\xa2\xa1[\xc6/i6\xd5\x96\x00!/P+\x92\xee\x9f@!\xdf.t\xccL\xf1\x87G\x9d\xf3p\x85@[\xf6~M\x87\xc8\xf3*\rb_\xa06D\xbc\xb6\x8e\xf6yC\x99\xe0\x863:D\xfeG\x18w\x95z\x13-\x91W\x86\xddSp\x91\xf8>\xf2\x0e\xbd\x89\xde\x14y`g\xaa;\xf3J6\x8f\xebM\xc8\x96\xa6\x1c\xde\xfe\xf2\xdf\xe3P\x18\xda\xfa\x8f?\xad_\x93\xce'\x8c\xf0\xb8\xab4\x17\t\xc9\xa5\ti\xfa\xb1\x13\xd2\x84C\x99\x8333\xe3\x03\xcb|\xae\x97v\x04-\xcf\xe7d\x1cO\xcf\xfd\xed{i\x833\xd3\xf3\xc3\xcb>\xd6\xfa\x1fP\xe8::\xeae=\xf0\xb1\x8eC\xfd\xa4\x92f\xed{s\x07\x18\xe1t\x8d\xa1V[o\xb0\x18\x80\x90\x15\xa8e\xa2\xd9\xfcO\xff\xf9\xe5\x85\xcfW\xf8\x97\x96z?\x83\xbf\xc1-\xcdm\xe5\xb4\xe8\xe6\xa1\xc1\xd7 \x1eR\x8b\xb3E\x92\x9c\xe2T8\xca\x18|7\x1aa\xb3\xa3m\xe3\x93<\x13\xdaL\xe6g\x1c\xcb\x15\x02\x91,\x1c\xbf\xbc4<\xbcx\xe3\x9c\xf8@\xab\x7f4\xe3\xf0\xb2\x9e<\xefq\x8f\x8e\xe4\xf5\x8b\xf8\x1a<K*\xcb\xce\xf6\xc8\xce\xf3\xdb\xd1U\xa6\xde?2\x9a\xe7\xf6\xd5EyL}@6\xca\x7f\xae\xb4\x99Zs\xe0\xdeg\x10\xb6\xe9\xe6Hp\xf0\xcd\xf1\xe0g1\xec?N\xf8\xb8\xce\nendstream\nendobj\n14 0 obj\n<</Type/Metadata\n/Subtype/XML/Length 1251>>stream\n<?xpacket begin='\xef\xbb\xbf' id='W5M0MpCehiHzreSzNTczkc9d'?>\n<?adobe-xap-filters esc=\"CRLF\"?>\n<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>\n<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>\n<rdf:Description rdf:about=\"\" xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 10.03.1'/>\n<rdf:Description rdf:about=\"\" xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2024-12-18T15:59:31-05:00</xmp:ModifyDate>\n<xmp:CreateDate>2024-12-18T15:59:31-05:00</xmp:CreateDate>\n<xmp:CreatorTool>GNU Enscript 1.6.6</xmp:CreatorTool></rdf:Description>\n<rdf:Description rdf:about=\"\" xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:86e4e793-f59f-11fa-0000-c8d2c052bf7e'/>\n<rdf:Description rdf:about=\"\" xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>Enscript Output</rdf:li></rdf:Alt></dc:title><dc:creator><rdf:Seq><rdf:li></rdf:li></rdf:Seq></dc:creator></rdf:Description>\n</rdf:RDF>\n</x:xmpmeta>\n                                                                        \n                                                                        \n<?xpacket end='w'?>\nendstream\nendobj\n2 0 obj\n<</Producer(GPL Ghostscript 10.03.1)\n/CreationDate(D:20241218155931-05'00')\n/ModDate(D:20241218155931-05'00')\n/Title(Enscript Output)\n/Author()\n/Creator(GNU Enscript 1.6.6)>>endobj\nxref\n0 15\n0000000000 65535 f \n0000000711 00000 n \n0000007145 00000 n \n0000000652 00000 n \n0000000510 00000 n \n0000000266 00000 n \n0000000491 00000 n \n0000001145 00000 n \n0000003652 00000 n \n0000000815 00000 n \n0000001471 00000 n \n0000000776 00000 n \n0000001773 00000 n \n0000003974 00000 n \n0000005817 00000 n \ntrailer\n<< /Size 15 /Root 1 0 R /Info 2 0 R\n/ID [<9BB34E42BF7AF21FE61720F4EBDFCCF8><9BB34E42BF7AF21FE61720F4EBDFCCF8>]\n>>\nstartxref\n7334\n%%EOF\n"
+        }
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        finding_events = [e for e in events if e.type == "FINDING"]
+        assert len(finding_events) == 1
+        assert "Possible Secret Found" in finding_events[0].data["description"]

bbot/test/test_step_2/module_tests/test_module_web_report.py

@@ -3,10 +3,11 @@ from .base import ModuleTestBase
 
 class TestWebReport(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["httpx", "wappalyzer", "badsecrets", "web_report", "secretsdb"]
+    modules_overrides = ["httpx", "wappalyzer", "badsecrets", "web_report", "trufflehog"]
+    config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
 
     async def setup_before_prep(self, module_test):
-        # secretsdb --> FINDING
+        # trufflehog --> FINDING
         # wappalyzer --> TECHNOLOGY
         # badsecrets --> VULNERABILITY
         respond_args = {"response_data": web_body}
@@ -23,7 +24,7 @@ class TestWebReport(ModuleTestBase):
 <li><strong>http://127.0.0.1:8888/</strong>"""
             in report_content
         )
-        assert """Possible secret (Asymmetric Private Key)""" in report_content
+        assert """Possible Secret Found. Detector Type: [PrivateKey]""" in report_content
         assert "<h3>TECHNOLOGY</h3>" in report_content
         assert "<p>flask</p>" in report_content
 
@@ -45,7 +46,22 @@ web_body = """
     <input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
 </div>
     </form>
-    <p>-----BEGIN PGP PRIVATE KEY BLOCK-----</p>
+    <p>-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOBY2pd9PSQvuxqu
+WXFNVgILTWuUc721Wc2sFNvp4beowhUe1lfxaq5ZfCJcz7z4QsqFhOeks69O9UIb
+oiOTDocPDog9PHO8yZXopHm0StFZvSjjKSNuFvy/WopPTGpxUZ5boCaF1CXumY7W
+FL+jIap5faimLL9prIwaQKBwv80lAgMBAAECgYEAxvpHtgCgD849tqZYMgOTevCn
+U/kwxltoMOClB39icNA+gxj8prc6FTTMwnVq0oGmS5UskX8k1yHCqUV1AvRU9o+q
+I8L8a3F3TQKQieI/YjiUNK8A87bKkaiN65ooOnhT+I3ZjZMPR5YEyycimMp22jsv
+LyX/35J/wf1rNiBs/YECQQDvtxgmMhE+PeajXqw1w2C3Jds27hI3RPDnamEyWr/L
+KkSplbKTF6FuFDYOFdJNPrfxm1tx2MZ2cBfs+h/GnCJVAkEA75Z9w7q8obbqGBHW
+9bpuFvLjW7bbqO7HBuXYX9zQcZL6GSArFP0ba5lhgH1qsVQfxVWVyiV9/chme7xc
+ljfvkQJBAJ7MpSPQcRnRefNp6R0ok+5gFqt55PlWI1y6XS81bO7Szm+laooE0n0Q
+yIpmLE3dqY9VgquVlkupkD/9poU0s40CQD118ZVAVht1/N9n1Cj9RjiE3mYspnTT
+rCLM25Db6Gz6M0Y2xlaAB4S2uBhqE/Chj/TjW6WbsJJl0kRzsZynhMECQFYKiM1C
+T4LB26ynW00VE8z4tEWSoYt4/Vn/5wFhalVjzoSJ8Hm2qZiObRYLQ1m0X4KnkShk
+Gnl54dJHT+EhlfY=
+-----END PRIVATE KEY-----</p>
 </body>
 </html>
 """

{bbot-2.3.0.5489rc0.dist-info → bbot-2.3.0.5504rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5489rc0
+Version: 2.3.0.5504rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.3.0.5489rc0.dist-info → bbot-2.3.0.5504rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=2i439UrdrmNdnUnbGMyYNiH9L1fMmHNYSRfZh7W-OK4,130
+bbot/__init__.py,sha256=GL__SZeN3u2sl0zznhow8MaFrcggtRyBSyT9lOjgRt8,130
 bbot/cli.py,sha256=SUEd4CcI-9QzFnqXpezza1sq_TNPcfDtJaSwL4MAl9g,10717
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -7,7 +7,7 @@ bbot/core/config/logger.py,sha256=FzQ7Myl0MVqBi7gpn9LOnbuL-UTXxSKpl11xuEGxS5I,10
 bbot/core/core.py,sha256=V0G3dKPN5xCbXOoFeBRkh-BZb6A3kSNA060De01LiTU,7065
 bbot/core/engine.py,sha256=uauGZgd7zAnJwNmVVPDicwHbUVCa8pV_pif49lgYeWk,29364
 bbot/core/event/__init__.py,sha256=8ut88ZUg0kbtWkOx2j3XzNr_3kTfgoM-3UdiWHFA_ag,56
-bbot/core/event/base.py,sha256=m7wA9IOmWNyLZfv3A-hmHEWYwsv0vVFrnXM3v4DxHUQ,62195
+bbot/core/event/base.py,sha256=IoU4nYZRP0VgqT7B4rlAVGS4khuXm7iihr_qd_RzjTE,62671
 bbot/core/event/helpers.py,sha256=PUN4Trq5_wpKVuhmwUQWAr40apgMXhJ9Gz-VfZ0j3lA,1554
 bbot/core/flags.py,sha256=Ltvm8Bc4D65I55HuU5bzyjO1R3yMDNpVmreGU83ZBXE,1266
 bbot/core/helpers/__init__.py,sha256=0UNwcZjNsX41hbHdo3yZPuARkYWch-okI68DScexve4,86
@@ -29,13 +29,13 @@ bbot/core/helpers/files.py,sha256=9tVr3973QvX8l6o3TweD5_MCZiQpuJVffbzW0U7Z30U,57
 bbot/core/helpers/helper.py,sha256=eD9yJUhrFUmmoXm4pyi31dulJ5W2eyS-YLGXmx7AyDQ,8439
 bbot/core/helpers/interactsh.py,sha256=VBYYH6-rWBofRsgemndK6iZNmyifOps8vgQOw2mac4k,12624
 bbot/core/helpers/libmagic.py,sha256=QMHyxjgDLb2jyjBvK1MQ-xt6WkGXhKcHu9ZP1li-sik,3460
-bbot/core/helpers/misc.py,sha256=156tWmNGZ0I1X537hQ8cAlShVq8D2dQVGz3DxaLh5d0,87734
+bbot/core/helpers/misc.py,sha256=r-FUL5zdZagmLjNd_F577IE1CrT22f5qPb69RQcEHD0,87452
 bbot/core/helpers/names_generator.py,sha256=x6nZfEPKMwv3qR_RI4U6TBNbo6PgCF4fqbldtwilvpw,10334
 bbot/core/helpers/ntlm.py,sha256=P2Xj4-GPos2iAzw4dfk0FJp6oGyycGhu2x6sLDVjYjs,2573
 bbot/core/helpers/process.py,sha256=00uRpLMFi3Pt3uT8qXwAIhsXdoa7h-ifoXh0sGYgwqs,1702
 bbot/core/helpers/ratelimiter.py,sha256=K8qFIyJPJtfdb9kSW6_lL6ahWqxR2uWyCBkDlg6uJgo,1990
 bbot/core/helpers/regex.py,sha256=yK8F6b2lOthvyNfZ62pqKXiF_LLFjblzH7HSo6hsuOE,4168
-bbot/core/helpers/regexes.py,sha256=_thQYedV5lhd2zMIrJteDs3ACZJyJoc5qB59ujN-8iU,5935
+bbot/core/helpers/regexes.py,sha256=f8Mv7yIK1yV7FvHabZadO0lvnunJuKKvzMim8UoLem0,5611
 bbot/core/helpers/url.py,sha256=1NDrvirODzzD6Mcssu-4WDNerMeMdekHCFzhRCS0m3g,5947
 bbot/core/helpers/validators.py,sha256=RBRCWpU-tKvcThx-KnEePpWTjsIxfQ_CrPEUBmiqPF0,9695
 bbot/core/helpers/web/__init__.py,sha256=pIEkL3DhjaGTSmZ7D3yKKYwWpntoLRILekV2wWsbsws,27
@@ -89,7 +89,7 @@ bbot/modules/dehashed.py,sha256=enDarOzlY84R4_ctp2fLVNLmjocaCh1j1x8nIKwEdHY,5064
 bbot/modules/digitorus.py,sha256=XQY0eAQrA7yo8S57tGncP1ARud-yG4LiWxx5VBYID34,1027
 bbot/modules/dnsbimi.py,sha256=A4cqhvhytmEEd-tY4CgFwMLbsVtMjkRY9238Aj8aVtU,6921
 bbot/modules/dnsbrute.py,sha256=Y2bSbG2IcwIJID1FSQ6Qe9fdpWwG7GIO-wVQw7MdQFM,2439
-bbot/modules/dnsbrute_mutations.py,sha256=bOJidK_oKZe87u8e9t0mEFnyuBi93UiNsQvpZYvhzZg,6939
+bbot/modules/dnsbrute_mutations.py,sha256=WuhEdbi8wWkysAVPBkEZ---9u2tBk6_x24_1CYRNA1w,7186
 bbot/modules/dnscaa.py,sha256=pyaLqHrdsVhqtd1JBZVjKKcuYT_ywUbFYkrnfXcGD5s,5014
 bbot/modules/dnscommonsrv.py,sha256=gEErfSur7Odkaif4CbXYx3OZ3FQrQESyiMGPbcDKSIg,1538
 bbot/modules/dnsdumpster.py,sha256=bqUqyvRJVtoTXbDxTZ-kgPNq4dCE9xv_msBIn_Nj5IM,3251
@@ -98,16 +98,16 @@ bbot/modules/docker_pull.py,sha256=T_xObzExDTZF-_HfgZSfrU199QgCME3rYmkVs1HigXQ,9
 bbot/modules/dockerhub.py,sha256=yHKxV-uVubAUvYrIXizSZoLUiPKArTH2mCh5FjY4sas,3486
 bbot/modules/dotnetnuke.py,sha256=rw_EchDg49VyQj5JiUh0AqUqtsuqLrhc-nwrybdzhZ8,10537
 bbot/modules/emailformat.py,sha256=RLPJW-xitYB-VT4Lp08qVzFkXx_kMyV_035JT_Yf4fM,1082
-bbot/modules/extractous.py,sha256=qE4h0reR51t9NSIcR4-VMMjc5t3H03bpW3fO0A6nb9c,4563
+bbot/modules/extractous.py,sha256=LbqfCoMXGyP19dxw0CIcbEPnyX7_eRWvuhHGHQ4wDXw,4641
 bbot/modules/ffuf_shortnames.py,sha256=9Kh0kJsw7XXpXmCkiB5eAhG4h9rSo8Y-mB3p0EDa_l0,12624
-bbot/modules/filedownload.py,sha256=x2LmLcLD1FD2L9O6p58l21WYoqyxD5HgHg6P1g1RyxU,8180
+bbot/modules/filedownload.py,sha256=Kq3oUejZcQozc6mgXNRKG7_wlRaGBsDobQ-zgQ0yo1g,8258
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=zeehQb9akBSbHW9dF4icH8Vfd8LqoTrpIvnQEEMWes8,1311
 bbot/modules/generic_ssrf.py,sha256=4JzlOXd7MMwDZEhEEXpLdkMUe8y3Qqql7ggb3eZeN54,7946
 bbot/modules/git.py,sha256=CMDarsmBemZEzZSeQTzB70XD8IRdwdG39zXpwDdgZbw,1383
 bbot/modules/git_clone.py,sha256=XFZXx0k97EMY3E5PZzdNvqQzZddOfRMaVp5ol2gk11s,2468
 bbot/modules/github_codesearch.py,sha256=6B0TjiRgDwBT8y5f45NCApjzl22NnHGxsnS1801v5G4,3634
-bbot/modules/github_org.py,sha256=O1VBn65sYJaPWBDjssyQSnlEh6XQgLEF7gKDzWj64qc,9164
+bbot/modules/github_org.py,sha256=u_skzY7rjhM3B8OvZV6YETOdIQtUtMwsVwhNcquM148,9078
 bbot/modules/github_workflows.py,sha256=vt4PbYGj2R6pi35AvyCsZar3wFYRJYGotZalDVSADus,9555
 bbot/modules/gitlab.py,sha256=9oWWpBijeHCjuFBfWW4HvNqt7bvJvrBgBjaaz_UPPnE,5964
 bbot/modules/google_playstore.py,sha256=N4QjzQag_bgDXfX17rytBiiWA-SQtYI2N0J_ZNEOdv0,3701
@@ -123,7 +123,7 @@ bbot/modules/internal/aggregate.py,sha256=csWYIt2fUp9K_CRxP3bndUMIjpNIh8rmBubp5F
 bbot/modules/internal/base.py,sha256=BXO4Hc7XKaAOaLzolF3krJX1KibPxtek2GTQUgnCHk0,387
 bbot/modules/internal/cloudcheck.py,sha256=sr1_dj4eqcD3x8MigZkjf0AxZZWSJFzvRjYGl5uxVdk,4913
 bbot/modules/internal/dnsresolve.py,sha256=1fwWChIGpSEIIkswueiIhEwIahQ7YngZ-njFK-RIsfU,15679
-bbot/modules/internal/excavate.py,sha256=IazekmFOnjrqghTf9YhBbMM4dohv27vIEvfNZXJSZvg,51529
+bbot/modules/internal/excavate.py,sha256=4buS6KL0g-spt9n3Toae-7xj2HZDw3RDAYr6Er006_4,51871
 bbot/modules/internal/speculate.py,sha256=NolqW2s8tokibc6gVM960KlrABkjhLB-7YlCdVx4O9s,9223
 bbot/modules/internetdb.py,sha256=Edg0Z84dH8dPTZMd7RlzvYBYNq8JHs_ns_ldnFxwRKo,5415
 bbot/modules/ip2location.py,sha256=yGivX9fzvwvLpnqmYCP2a8SPjTarzrZxfRluog-nkME,2628
@@ -171,7 +171,6 @@ bbot/modules/report/affiliates.py,sha256=vvus8LylqOfP-lfGid0z4FS6MwOpNuRTcSJ9aSn
 bbot/modules/report/asn.py,sha256=D0jQkcZe_gEbmSokgSisYw6QolVJI9l71ksSMlOVTfo,9687
 bbot/modules/report/base.py,sha256=hOtZF41snTSlHZmzZndmOjfmtdKPy2-tfFBAxxbHcao,105
 bbot/modules/robots.py,sha256=LGG6ixsxrlaCk-mi4Lp6kB2RB1v-25NhTAQxdQEtH8s,2172
-bbot/modules/secretsdb.py,sha256=MA6IKo5rOvC0Dzt-F4QVrSwLkcPRwWLd9FpzqYkc8u8,3082
 bbot/modules/securitytrails.py,sha256=5Jk_HTQP8FRq6A30sN19FU79uLJt7aiOsI2dxNkLDcM,1148
 bbot/modules/securitytxt.py,sha256=nwaTOnRAl2NWcEc3i_I9agB56QjqK8dHqiKRHPPkCPE,4558
 bbot/modules/shodan_dns.py,sha256=ETOyUhCiAETlGUAQhvAP47oEEPYss7fm_F_CAeCQyoI,842
@@ -191,7 +190,7 @@ bbot/modules/templates/sql.py,sha256=o-CdyyoJvHJdJBKkj3CIGXYxUta4w2AB_2Vr-k7cDDU
 bbot/modules/templates/subdomain_enum.py,sha256=SJmQKbWpymgSV_CYXDLlARhDCFxonzhhpvO_gIFaHnM,8396
 bbot/modules/templates/webhook.py,sha256=Ch7Xrq8DuIBSYaIUWsSGqg8irtDsyk6LVKhsRHTpTh0,3706
 bbot/modules/trickest.py,sha256=MRgLW0YiDWzlWdAjyqfPPLFb-a51r-Ffn_dphiJI_gA,1550
-bbot/modules/trufflehog.py,sha256=yNUJkb-zLq0_g4OHbOppMClVCwQcrAsp0At_Q7_Rq1k,8553
+bbot/modules/trufflehog.py,sha256=BUOdOgJ7WWJ-9cKYqC9dGGHDhpENZY-XxWX7i5aUnng,8627
 bbot/modules/url_manipulation.py,sha256=4J3oFkqTSJPPmbKEKAHJg2Q2w4QNKtQhiN03ZJq5VtI,4326
 bbot/modules/urlscan.py,sha256=-w_3Bm6smyG2GLQyIbnMUkKmeQVauo-V6F4_kJDYG7s,3740
 bbot/modules/viewdns.py,sha256=2SjNZNjQL1tko58tPAjP-CGYDmP-zZ1HpY-vACGa9UI,2595
@@ -250,15 +249,15 @@ bbot/test/test_step_1/test_depsinstaller.py,sha256=zr9f-wJDotD1ZvKXGEuDRWzFYMAYB
 bbot/test/test_step_1/test_dns.py,sha256=SjefP-8GGyx9q-PWotCWu4esF0dRhWJRrS-J5MGNi6w,32153
 bbot/test/test_step_1/test_docs.py,sha256=YWVGNRfzcrvDmFekX0Cq9gutQplsqvhKTpZ0XK4tWvo,82
 bbot/test/test_step_1/test_engine.py,sha256=3HkCPtYhUxiZzfA-BRHpLsyaRj9wIXKbb49BCk9dILM,4267
-bbot/test/test_step_1/test_events.py,sha256=d801YobkGGxolHx6H1qUZTo6ZedVJ1DGDxCfgdEf-0Y,50555
+bbot/test/test_step_1/test_events.py,sha256=E09r2gpXnlG580wrPPPXRa3aCT6cJprTGD1_D0AiM-E,52717
 bbot/test/test_step_1/test_files.py,sha256=5Q_3jPpMXULxDHsanSDUaj8zF8bXzKdiJZHOmoYpLhQ,699
-bbot/test/test_step_1/test_helpers.py,sha256=6WG2rqnI7Jt0Z7Dc5AyqTDcL16QM0_WJ3CXE1M-xSMc,39506
+bbot/test/test_step_1/test_helpers.py,sha256=hZfnzjtegezYOqTuo5uAaXGI2GGsfuHhglbfaPPYV-U,39482
 bbot/test/test_step_1/test_manager_deduplication.py,sha256=hZQpDXzg6zvzxFolVOcJuY-ME8NXjZUsqS70BRNXp8A,15594
 bbot/test/test_step_1/test_manager_scope_accuracy.py,sha256=JV1bQHt9EIM0GmGS4T4Brz_L2lfcwTxtNC06cfv7r64,79763
 bbot/test/test_step_1/test_modules_basic.py,sha256=hxXdsrBwme5elGQtvyvA52-KzahyQC3FlWQZ3T0EheA,19989
 bbot/test/test_step_1/test_presets.py,sha256=Y5Bbz1qKk5yiQt3LdZ_R_M4gcInCSX4XP2vm-Jpv28E,38209
 bbot/test/test_step_1/test_python_api.py,sha256=GM5Kp2AAFl92ozo1kL6axsM87F8Gdq2_mWQvRnbXW_0,5503
-bbot/test/test_step_1/test_regexes.py,sha256=34-BHzDE5qdltE-sQIzkrTmJTL49QUYoTn2uT1DZLwI,14356
+bbot/test/test_step_1/test_regexes.py,sha256=CxzMZpvza5RLVNUgzK4-ov5YTe2ib-vBv5IeG7-aeew,14356
 bbot/test/test_step_1/test_scan.py,sha256=h3JP5RXnOUH8dqqq2Q_7yLpx1LCAEvqfE1bpHL7bDS0,5756
 bbot/test/test_step_1/test_scope.py,sha256=S2nssENKJKCvgXUMyU8MFQmXHeUIz0C_sbWGkdYti2A,3063
 bbot/test/test_step_1/test_target.py,sha256=CDfcCTuhh1Z-MdcSHC3lZ94ucDI2M-xacdv6-SchqxE,19512
@@ -317,7 +316,7 @@ bbot/test/test_step_2/module_tests/test_module_dockerhub.py,sha256=9T8CFcFP32MOp
 bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py,sha256=voi1C_v7VeaRe_-yzCybO9FUxnFf9qzWkoUY66KYiGI,8114
 bbot/test/test_step_2/module_tests/test_module_emailformat.py,sha256=cKxBPnEQ4AiRKV_-hSYEE6756ypst3hi6MN0L5RTukY,461
 bbot/test/test_step_2/module_tests/test_module_emails.py,sha256=bZjtO8N3GG2_g6SUEYprAFLcsi7SlwNPJJ0nODfrWYU,944
-bbot/test/test_step_2/module_tests/test_module_excavate.py,sha256=HexVwLYEAfJn5aMrbALktoA-uv1kkAHnZ05y6DiJv6U,42147
+bbot/test/test_step_2/module_tests/test_module_excavate.py,sha256=IPV3C2G7J5cR1526xBwBnnHBK_5yXM0oKGT4bps9byE,43303
 bbot/test/test_step_2/module_tests/test_module_extractous.py,sha256=m2zlG26NGtT-dfvn0DbAKV_aeInTHGQWhCemD7ddI3g,17973
 bbot/test/test_step_2/module_tests/test_module_ffuf.py,sha256=aSB49aN77sw-2LNTDHckiEEaHAn_85xCJno1shdOwus,2964
 bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py,sha256=s8E7M9d1fhm__krM4lmteyTtSsYpVL4hn1z8ub7RVss,7608
@@ -327,7 +326,7 @@ bbot/test/test_step_2/module_tests/test_module_fullhunt.py,sha256=NblfNHQrE82j-c
 bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py,sha256=w8FHR88mRIIr0-H_tK3ymmpJwymOb_kj2B4tcFi5kic,2155
 bbot/test/test_step_2/module_tests/test_module_git.py,sha256=gyBS3vZUWAyatGlcY26mGOYeqXSqJA5pbhJWgTmLqNo,1656
 bbot/test/test_step_2/module_tests/test_module_git_clone.py,sha256=Mo0Q7bCXcrkGWJc3-u5y4sdfC13ei-qj79aKvEbnkk4,13198
-bbot/test/test_step_2/module_tests/test_module_github_codesearch.py,sha256=yWQoGV0v43Eikt8Hr0xavxyUI96ogGloUZJ8txpxflg,3774
+bbot/test/test_step_2/module_tests/test_module_github_codesearch.py,sha256=e33MO94RShvT9lEf-2uyPvp0nRCa_jYu2PB_ZffOBu4,4733
 bbot/test/test_step_2/module_tests/test_module_github_org.py,sha256=5tKO6NH4TPBeIdeTf7Bz9PUZ1pcvKsjrG0nFhc3YgT0,25458
 bbot/test/test_step_2/module_tests/test_module_github_workflows.py,sha256=o_teEaskm3H22QEKod5KJayFvvcgOQoG4eItGWv8C8E,38006
 bbot/test/test_step_2/module_tests/test_module_gitlab.py,sha256=fnwE7BWTU6EQquKdGLCiaX_LwVwvzOLev3Y9GheTLSY,11859
@@ -368,7 +367,6 @@ bbot/test/test_step_2/module_tests/test_module_postman_download.py,sha256=B_NajQ
 bbot/test/test_step_2/module_tests/test_module_python.py,sha256=6UQVXGJ1ugfNbt9l_nN0q5FVxNWlpq6j0sZcB0Nh_Pg,184
 bbot/test/test_step_2/module_tests/test_module_rapiddns.py,sha256=zXHNLnUjLO22yRwrDFCZ40sRTmFVZEj9q_dyK8w1TYM,4441
 bbot/test/test_step_2/module_tests/test_module_robots.py,sha256=8rRw4GpGE6tN_W3ohtpfWiji_bEEmD31wvxz7r1FqnI,1564
-bbot/test/test_step_2/module_tests/test_module_secretsdb.py,sha256=EmqWGnqAQHGEZxbmJJXHwIgz5SQi4R3Fq982k6sYkZM,537
 bbot/test/test_step_2/module_tests/test_module_securitytrails.py,sha256=NB8_PhWN1-2s8wporRjI6rrQeQW4inoz4Z_mBhhycXo,758
 bbot/test/test_step_2/module_tests/test_module_securitytxt.py,sha256=i9Emp1utut1LPIe502Jb3XzN4GiVBeo4ATLFR9w4rbY,2382
 bbot/test/test_step_2/module_tests/test_module_shodan_dns.py,sha256=9DpFizZguP2aFCKI1HgXzxy3sB_afA03pzIx0BDfgqg,1515
@@ -388,7 +386,7 @@ bbot/test/test_step_2/module_tests/test_module_subdomains.py,sha256=r1zCmw5ZZ_0w
 bbot/test/test_step_2/module_tests/test_module_teams.py,sha256=rhVRSUb5VJVSWM5VLeysB2yZwHLN36lR1FeznsjfUG4,1206
 bbot/test/test_step_2/module_tests/test_module_telerik.py,sha256=_EynKEnQXIV0y2YC0VzGozSCqp9VCLWYqTX8OCv85SU,7752
 bbot/test/test_step_2/module_tests/test_module_trickest.py,sha256=6mTYH6fIah-WbKnFI-_WZBwRdKFi-oeWyVtl1n0nVAU,1630
-bbot/test/test_step_2/module_tests/test_module_trufflehog.py,sha256=hPbQF0eRDvf5Riynbz9QIqo9oPzgZdvgDk_q8BlVB8c,79142
+bbot/test/test_step_2/module_tests/test_module_trufflehog.py,sha256=v7VWMButtyoYJKmKnKu-NLpMY9LPZDA-YwGcNGz7Eu8,95421
 bbot/test/test_step_2/module_tests/test_module_txt.py,sha256=R-EBfEZM0jwY2yuVyfYhoccDOl0Y2uQZSkXQ1HyinUA,247
 bbot/test/test_step_2/module_tests/test_module_url_manipulation.py,sha256=aP3nK2TQQOjk0ZeuHhHYfZm_e37qrrXbnufd7m-QeJU,1144
 bbot/test/test_step_2/module_tests/test_module_urlscan.py,sha256=H_og5fOQMLpDbEGOhcVcZcDXvodT6nfgCE6Rk8LTkas,2902
@@ -398,7 +396,7 @@ bbot/test/test_step_2/module_tests/test_module_virustotal.py,sha256=MixCr5T4-pQf
 bbot/test/test_step_2/module_tests/test_module_wafw00f.py,sha256=3a0bsM5VyIDmSS5Addfmq0HM5ztQIlPm85tnoqlw9r4,1976
 bbot/test/test_step_2/module_tests/test_module_wappalyzer.py,sha256=viPAWXrnQT2J2jYX6LNU3IEqwhJYfXiRceoLVkkLF58,936
 bbot/test/test_step_2/module_tests/test_module_wayback.py,sha256=byMJ_u6R9WmwB0p2uf01P39c9q_OqsBOyhmA2sC-XUE,547
-bbot/test/test_step_2/module_tests/test_module_web_report.py,sha256=xV7bhm-rxT2fOlvZ32rl5VQxFJlM0mDiTWBDH-hPpEE,1961
+bbot/test/test_step_2/module_tests/test_module_web_report.py,sha256=5h4yAl_z265UyQXq9V3hNEeqhJhNhBM2hshWOQ_7hH8,2928
 bbot/test/test_step_2/module_tests/test_module_websocket.py,sha256=Vkqs9C35oKlGZTCVglepE2gjcXgpDmnxwdZ-cVRrTPM,1002
 bbot/test/test_step_2/module_tests/test_module_wpscan.py,sha256=ACGnHsavQy4uRJYoosE1JD-eJFdOj50G65P2FhIqRrM,35772
 bbot/test/test_step_2/module_tests/test_module_zoomeye.py,sha256=mEaMYa9ytxSMDIR1csmK1k7F1UrbolqEwZtqGRhA-OY,1979
@@ -416,8 +414,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.3.0.5489rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.3.0.5489rc0.dist-info/METADATA,sha256=y_xqx4nhbAp0Daqx5AQ3wbpHJLw5Gi-REluIFj2P3ZE,18213
-bbot-2.3.0.5489rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-bbot-2.3.0.5489rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.3.0.5489rc0.dist-info/RECORD,,
+bbot-2.3.0.5504rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.3.0.5504rc0.dist-info/METADATA,sha256=TzSBs17LvHzt6Q5iNlPUUDkNKb6OHhKZmOFltx3s3-w,18213
+bbot-2.3.0.5504rc0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+bbot-2.3.0.5504rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.3.0.5504rc0.dist-info/RECORD,,

bbot/modules/secretsdb.py

@@ -1,78 +0,0 @@
-import re
-import yaml
-
-from .base import BaseModule
-
-
-class secretsdb(BaseModule):
-    watched_events = ["HTTP_RESPONSE"]
-    produced_events = ["FINDING"]
-    flags = ["active", "safe", "web-basic"]
-    meta = {
-        "description": "Detect common secrets with secrets-patterns-db",
-        "created_date": "2023-03-17",
-        "author": "@TheTechromancer",
-    }
-    options = {
-        "min_confidence": 99,
-        "signatures": "https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml",
-    }
-    options_desc = {
-        "min_confidence": "Only use signatures with this confidence score or higher",
-        "signatures": "File path or URL to YAML signatures",
-    }
-    deps_pip = ["pyyaml~=6.0"]
-    # accept any HTTP_RESPONSE including out-of-scope ones (such as from github_codesearch)
-    scope_distance_modifier = 3
-
-    async def setup(self):
-        self.rules = []
-        self.min_confidence = self.config.get("min_confidence", 99)
-        self.sig_file = await self.helpers.wordlist(self.config.get("signatures", ""))
-        with open(self.sig_file) as f:
-            rules_yaml = yaml.safe_load(f).get("patterns", [])
-        for r in rules_yaml:
-            r = r.get("pattern", {})
-            if not r:
-                continue
-            name = r.get("name", "").lower()
-            confidence = r.get("confidence", "")
-            if name and confidence >= self.min_confidence:
-                regex = r.get("regex", "")
-                try:
-                    compiled_regex = re.compile(regex)
-                    r["regex"] = compiled_regex
-                    self.rules.append(r)
-                except Exception:
-                    self.debug(f"Error compiling regex: r'{regex}'")
-        return True
-
-    async def handle_event(self, event):
-        resp_body = event.data.get("body", "")
-        resp_headers = event.data.get("raw_header", "")
-        all_matches = await self.helpers.run_in_executor(self.search_data, resp_body, resp_headers)
-        for matches, name in all_matches:
-            matches = [m.string[m.start() : m.end()] for m in matches]
-            description = f"Possible secret ({name}): {matches}"
-            event_data = {"host": str(event.host), "description": description}
-            parsed_url = getattr(event, "parsed_url", None)
-            if parsed_url:
-                event_data["url"] = parsed_url.geturl()
-            await self.emit_event(
-                event_data,
-                "FINDING",
-                parent=event,
-                context=f"{{module}} searched HTTP response and found {{event.type}}: {description}",
-            )
-
-    def search_data(self, resp_body, resp_headers):
-        all_matches = []
-        for r in self.rules:
-            regex = r["regex"]
-            name = r["name"]
-            for text in (resp_body, resp_headers):
-                if text:
-                    matches = list(regex.finditer(text))
-                    if matches:
-                        all_matches.append((matches, name))
-        return all_matches

bbot/test/test_step_2/module_tests/test_module_secretsdb.py

@@ -1,14 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestSecretsDB(ModuleTestBase):
-    targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["httpx", "secretsdb"]
-
-    async def setup_before_prep(self, module_test):
-        expect_args = {"method": "GET", "uri": "/"}
-        respond_args = {"response_data": "-----BEGIN PGP PRIVATE KEY BLOCK-----"}
-        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
-
-    def check(self, module_test, events):
-        assert any(e.type == "FINDING" for e in events)