bbot 2.3.0.5491rc0__tar.gz → 2.3.0.5515rc0__tar.gz

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bbot
-Version: 2.3.0.5491rc0
+Version: 2.3.0.5515rc0
 Summary: OSINT automation for hackers.
 Home-page: https://github.com/blacklanternsecurity/bbot
 License: GPL-3.0

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.0.5491rc"
+__version__ = "v2.3.0.5515rc"
 
 from .scanner import Scanner, Preset

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/core/event/base.py

@@ -515,22 +515,25 @@ class BaseEvent:
             new_scope_distance = min(self.scope_distance, scope_distance)
         if self._scope_distance != new_scope_distance:
             # remove old scope distance tags
-            for t in list(self.tags):
-                if t.startswith("distance-"):
-                    self.remove_tag(t)
-            if self.host:
-                if scope_distance == 0:
-                    self.add_tag("in-scope")
-                    self.remove_tag("affiliate")
-                else:
-                    self.remove_tag("in-scope")
-                    self.add_tag(f"distance-{new_scope_distance}")
             self._scope_distance = new_scope_distance
+            self.refresh_scope_tags()
             # apply recursively to parent events
             parent_scope_distance = getattr(self.parent, "scope_distance", None)
             if parent_scope_distance is not None and self.parent is not self:
                 self.parent.scope_distance = new_scope_distance + 1
 
+    def refresh_scope_tags(self):
+        for t in list(self.tags):
+            if t.startswith("distance-"):
+                self.remove_tag(t)
+        if self.host:
+            if self.scope_distance == 0:
+                self.add_tag("in-scope")
+                self.remove_tag("affiliate")
+            else:
+                self.remove_tag("in-scope")
+                self.add_tag(f"distance-{self.scope_distance}")
+
     @property
     def scope_description(self):
         """
@@ -587,7 +590,7 @@ class BaseEvent:
                         if t in ("spider-danger", "spider-max"):
                             self.add_tag(t)
         elif not self._dummy:
-            log.warning(f"Tried to set invalid parent on {self}: (got: {parent})")
+            log.warning(f"Tried to set invalid parent on {self}: (got: {repr(parent)} ({type(parent)}))")
 
     @property
     def parent_id(self):
@@ -1042,6 +1045,9 @@ class DictPathEvent(DictEvent):
         blob = None
         try:
             self._data_path = Path(data["path"])
+            # prepend the scan's home dir if the path is relative
+            if not self._data_path.is_absolute():
+                self._data_path = self.scan.home / self._data_path
             if self._data_path.is_file():
                 self.add_tag("file")
                 if file_blobs:
@@ -1352,18 +1358,22 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
         self.parsed_url = self.validators.validate_url_parsed(url)
         data["url"] = self.parsed_url.geturl()
 
-        header_dict = {}
-        for i in data.get("raw_header", "").splitlines():
-            if len(i) > 0 and ":" in i:
-                k, v = i.split(":", 1)
-                k = k.strip().lower()
-                v = v.lstrip()
-                if k in header_dict:
-                    header_dict[k].append(v)
-                else:
-                    header_dict[k] = [v]
+        if not "raw_header" in data:
+            raise ValueError("raw_header is required for HTTP_RESPONSE events")
+
+        if "header-dict" not in data:
+            header_dict = {}
+            for i in data.get("raw_header", "").splitlines():
+                if len(i) > 0 and ":" in i:
+                    k, v = i.split(":", 1)
+                    k = k.strip().lower()
+                    v = v.lstrip()
+                    if k in header_dict:
+                        header_dict[k].append(v)
+                    else:
+                        header_dict[k] = [v]
+            data["header-dict"] = header_dict
 
-        data["header-dict"] = header_dict
         # move URL to the front of the dictionary for visibility
         data = dict(data)
         new_data = {"url": data.pop("url")}
@@ -1377,6 +1387,13 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
     def _pretty_string(self):
         return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
 
+    @property
+    def raw_response(self):
+        """
+        Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.
+        """
+        return f'{self.data["raw_header"]}{self.data["body"]}'
+
     @property
     def http_status(self):
         try:

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/core/helpers/depsinstaller/installer.py

@@ -20,11 +20,43 @@ log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
 
 class DepsInstaller:
+    CORE_DEPS = {
+        # core BBOT dependencies in the format of binary: package_name
+        # each one will only be installed if the binary is not found
+        "unzip": "unzip",
+        "zipinfo": "unzip",
+        "curl": "curl",
+        "git": "git",
+        "make": "make",
+        "gcc": "gcc",
+        "bash": "bash",
+        "which": "which",
+        "unrar": "unrar-free",
+        "tar": "tar",
+        # debian why are you like this
+        "7z": [
+            {
+                "name": "Install 7zip (Debian)",
+                "package": {"name": ["p7zip-full"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Debian'",
+            },
+            {
+                "name": "Install 7zip (Non-Debian)",
+                "package": {"name": ["p7zip"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] != 'Debian'",
+            },
+        ],
+    }
+
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.preset = self.parent_helper.preset
         self.core = self.preset.core
 
+        self.os_platform = os_platform()
+
         # respect BBOT's http timeout
         self.web_config = self.parent_helper.config.get("web", {})
         http_timeout = self.web_config.get("http_timeout", 30)
@@ -202,28 +234,32 @@ class DepsInstaller:
         """
         Install packages with the OS's default package manager (apt, pacman, dnf, etc.)
         """
-        packages_str = ",".join(packages)
+        args, kwargs = self._make_apt_ansible_args(packages)
+        success, err = self.ansible_run(module="package", args=args, **kwargs)
+        if success:
+            log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"')
+        else:
+            log.warning(
+                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
+            )
+            for p in packages:
+                log.warning(f" - {p}")
+        return success
+
+    def _make_apt_ansible_args(self, packages):
+        packages_str = ",".join(sorted(packages))
         log.info(f"Installing the following OS packages: {packages_str}")
         args = {"name": packages_str, "state": "present"}  # , "update_cache": True, "cache_valid_time": 86400}
         kwargs = {}
         # don't sudo brew
-        if os_platform() != "darwin":
+        if self.os_platform != "darwin":
             kwargs = {
                 "ansible_args": {
                     "ansible_become": True,
                     "ansible_become_method": "sudo",
                 }
             }
-        success, err = self.ansible_run(module="package", args=args, **kwargs)
-        if success:
-            log.info(f'Successfully installed OS packages "{packages_str}"')
-        else:
-            log.warning(
-                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
-            )
-            for p in packages:
-                log.warning(f" - {p}")
-        return success
+        return args, kwargs
 
     def shell(self, module, commands):
         tasks = []
@@ -269,7 +305,7 @@ class DepsInstaller:
             for task in tasks:
                 if "package" in task:
                     # special case for macos
-                    if os_platform() == "darwin":
+                    if self.os_platform == "darwin":
                         # don't sudo brew
                         task["become"] = False
                         # brew doesn't support update_cache
@@ -292,8 +328,8 @@ class DepsInstaller:
             },
             module=module,
             module_args=module_args,
-            quiet=not self.ansible_debug,
-            verbosity=(3 if self.ansible_debug else 0),
+            quiet=True,
+            verbosity=0,
             cancel_callback=lambda: None,
         )
 
@@ -303,7 +339,7 @@ class DepsInstaller:
         err = ""
         for e in res.events:
             if self.ansible_debug and not success:
-                log.debug(json.dumps(e, indent=4))
+                log.debug(json.dumps(e, indent=2))
             if e["event"] == "runner_on_failed":
                 err = e["event_data"]["res"]["msg"]
                 break
@@ -347,26 +383,30 @@ class DepsInstaller:
 
     def install_core_deps(self):
         to_install = set()
+        to_install_friendly = set()
+        playbook = []
         self._install_sudo_askpass()
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
-        # command: package_name
-        core_deps = {
-            "unzip": "unzip",
-            "zipinfo": "unzip",
-            "curl": "curl",
-            "git": "git",
-            "make": "make",
-            "gcc": "gcc",
-            "bash": "bash",
-            "which": "which",
-        }
-        for command, package_name in core_deps.items():
+        for command, package_name_or_playbook in self.CORE_DEPS.items():
             if not self.parent_helper.which(command):
-                to_install.add(package_name)
+                to_install_friendly.add(command)
+                if isinstance(package_name_or_playbook, str):
+                    to_install.add(package_name_or_playbook)
+                else:
+                    playbook.extend(package_name_or_playbook)
         if to_install:
+            playbook.append(
+                {
+                    "name": "Install Core BBOT Dependencies",
+                    "package": {"name": list(to_install), "state": "present"},
+                    "become": True,
+                }
+            )
+        if playbook:
+            log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()
-            self.apt_install(list(to_install))
+            self.ansible_run(tasks=playbook)
 
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup:

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/core/helpers/misc.py

@@ -559,13 +559,12 @@ def is_port(p):
     return p and p.isdigit() and 0 <= int(p) <= 65535
 
 
-def is_dns_name(d, include_local=True):
+def is_dns_name(d):
     """
     Determines if the given string is a valid DNS name.
 
     Args:
         d (str): The string to be checked.
-        include_local (bool): Consider local hostnames to be valid (hostnames without periods)
 
     Returns:
         bool: True if the string is a valid DNS name, False otherwise.
@@ -575,17 +574,12 @@ def is_dns_name(d, include_local=True):
         True
         >>> is_dns_name('localhost')
         True
-        >>> is_dns_name('localhost', include_local=False)
-        False
         >>> is_dns_name('192.168.1.1')
         False
     """
     if is_ip(d):
         return False
     d = smart_decode(d)
-    if include_local:
-        if bbot_regexes.hostname_regex.match(d):
-            return True
     if bbot_regexes.dns_name_validation_regex.match(d):
         return True
     return False

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/core/helpers/regexes.py

@@ -39,14 +39,10 @@ _ip_range_regexes = (
 ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
 
 # dns names with periods
-_dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
+_dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.?)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
 dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
 dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
 
-# dns names without periods
-_hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?"
-hostname_regex = re.compile(r"^" + _hostname_regex + r"$", re.I)
-
 _email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
 email_regex = re.compile(_email_regex, re.I)
 
@@ -61,14 +57,12 @@ event_uuid_regex = re.compile(_event_uuid_regex, re.I)
 
 _open_port_regexes = (
     _dns_name_regex + r":[0-9]{1,5}",
-    _hostname_regex + r":[0-9]{1,5}",
     r"\[" + _ipv6_regex + r"\]:[0-9]{1,5}",
 )
 open_port_regexes = [re.compile(r, re.I) for r in _open_port_regexes]
 
 _url_regexes = (
     r"https?://" + _dns_name_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
-    r"https?://" + _hostname_regex + r"(?::[0-9]{1,5})?(?:(?:/|\?).*)?",
     r"https?://\[" + _ipv6_regex + r"\](?::[0-9]{1,5})?(?:(?:/|\?).*)?",
 )
 url_regexes = [re.compile(r, re.I) for r in _url_regexes]
@@ -83,10 +77,7 @@ event_type_regexes = OrderedDict(
         for k, regexes in (
             (
                 "DNS_NAME",
-                (
-                    r"^" + _dns_name_regex + r"$",
-                    r"^" + _hostname_regex + r"$",
-                ),
+                (r"^" + _dns_name_regex + r"$",),
             ),
             (
                 "EMAIL_ADDRESS",
@@ -140,7 +131,7 @@ select_tag_regex = re.compile(
 textarea_tag_regex = re.compile(
     r'<textarea[^>]*\bname=["\']?(\w+)["\']?[^>]*>(.*?)</textarea>', re.IGNORECASE | re.DOTALL
 )
-tag_attribute_regex = re.compile(r"<[^>]*(?:href|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>")
+tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?(?!mailto:)([^\s\'\"\>]+)[\"\']?[^>]*>")
 
 valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+"
 

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/dnsbrute_mutations.py

@@ -1,3 +1,5 @@
+import time
+
 from bbot.modules.base import BaseModule
 
 
@@ -40,8 +42,11 @@ class dnsbrute_mutations(BaseModule):
                 except KeyError:
                     self.found[domain] = {subdomain}
 
-    def get_parent_event(self, subdomain):
-        parent_host = self.helpers.closest_match(subdomain, self.parent_events)
+    async def get_parent_event(self, subdomain):
+        start = time.time()
+        parent_host = await self.helpers.run_in_executor(self.helpers.closest_match, subdomain, self.parent_events)
+        elapsed = time.time() - start
+        self.trace(f"{subdomain}: got closest match among {len(self.parent_events):,} parent events in {elapsed:.2f}s")
         return self.parent_events[parent_host]
 
     async def finish(self):
@@ -124,7 +129,7 @@ class dnsbrute_mutations(BaseModule):
                             self._mutation_run_counter[domain] = mutation_run = 1
                         self._mutation_run_counter[domain] += 1
                         for hostname in results:
-                            parent_event = self.get_parent_event(hostname)
+                            parent_event = await self.get_parent_event(hostname)
                             mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)
                             await self.emit_event(
                                 hostname,

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/extractous.py

@@ -28,6 +28,7 @@ class extractous(BaseModule):
             "ica",  #  Citrix Independent Computing Architecture File
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
+            "json",  #  JSON File
             "key",  #  Private Key File
             "pub",  #  Public Key File
             "log",  #  Log File
@@ -45,6 +46,7 @@ class extractous(BaseModule):
             "pptx",  #  Microsoft PowerPoint Presentation
             "ps1",  #  PowerShell Script
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "swp",  #  Swap File (temporary file, often Vim)

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/filedownload.py

@@ -38,6 +38,7 @@ class filedownload(BaseModule):
             "indd",  #  Adobe InDesign Document
             "ini",  #  Initialization File
             "jar",  #  Java Archive
+            "json",  #  JSON File
             "key",  #  Private Key File
             "log",  #  Log File
             "markdown",  #  Markdown File
@@ -57,6 +58,7 @@ class filedownload(BaseModule):
             "pub",  #  Public Key File
             "raw",  #  Raw Image File Format
             "rdp",  #  Remote Desktop Protocol File
+            "rsa",  #  RSA Private Key File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
             "sqlite",  #  SQLite Database File
@@ -177,7 +179,9 @@ class filedownload(BaseModule):
         if extension:
             filename = f"{filename}.{extension}"
             orig_filename = f"{orig_filename}.{extension}"
-        return orig_filename, self.download_dir / filename, base_url
+        file_destination = self.download_dir / filename
+        file_destination = self.helpers.truncate_filename(file_destination)
+        return orig_filename, file_destination, base_url
 
     async def report(self):
         if self.files_downloaded > 0:

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/github_org.py

@@ -206,11 +206,7 @@ class github_org(github):
         for k, v in json.items():
             if (
                 isinstance(v, str)
-                and (
-                    self.helpers.is_dns_name(v, include_local=False)
-                    or self.helpers.is_url(v)
-                    or self.helpers.is_email(v)
-                )
+                and (self.helpers.is_dns_name(v) and "." in v or self.helpers.is_url(v) or self.helpers.is_email(v))
                 and self.scan.in_scope(v)
             ):
                 self.verbose(f'Found in-scope key "{k}": "{v}" for {org}, it appears to be in-scope')

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/gowitness.py

@@ -142,6 +142,9 @@ class gowitness(BaseModule):
             url = screenshot["url"]
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
+            filename = filename.relative_to(self.scan.home)
+            # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
+            filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
             parent_event = event_dict[url]
             await self.emit_event(

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/internal/excavate.py

@@ -505,6 +505,11 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                             if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
                                 if self.excavate.in_bl(parameter_name) is False:
                                     parsed_url = urlparse(url)
+                                    if not parsed_url.hostname:
+                                        self.excavate.warning(
+                                            f"Error Parsing reconstructed URL [{url}] during parameter extraction, missing hostname"
+                                        )
+                                        continue
                                     description = f"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)"
                                     data = {
                                         "host": parsed_url.hostname,
@@ -703,7 +708,7 @@ class excavate(BaseInternalModule, BaseInterceptModule):
                 """
             ),
         }
-        full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)")
+        full_url_regex = re.compile(r"(https?)://(\w(?:[\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)")
         full_url_regex_strict = re.compile(r"^(https?):\/\/([\w.-]+)(?::\d{1,5})?(\/[\w\/\.-]*)?(\?[^\s]+)?$")
         tag_attribute_regex = bbot_regexes.tag_attribute_regex
 

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/modules/trufflehog.py

@@ -3,7 +3,7 @@ from bbot.modules.base import BaseModule
 
 
 class trufflehog(BaseModule):
-    watched_events = ["CODE_REPOSITORY", "FILESYSTEM"]
+    watched_events = ["CODE_REPOSITORY", "FILESYSTEM", "HTTP_RESPONSE", "RAW_TEXT"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["passive", "safe", "code-enum"]
     meta = {
@@ -81,12 +81,15 @@ class trufflehog(BaseModule):
         return True
 
     async def handle_event(self, event):
-        description = event.data.get("description", "")
+        description = ""
+        if isinstance(event.data, dict):
+            description = event.data.get("description", "")
+
         if event.type == "CODE_REPOSITORY":
             path = event.data["url"]
             if "git" in event.tags:
                 module = "github-experimental"
-        else:
+        elif event.type == "FILESYSTEM":
             path = event.data["path"]
             if "git" in event.tags:
                 module = "git"
@@ -96,6 +99,14 @@ class trufflehog(BaseModule):
                 module = "postman"
             else:
                 module = "filesystem"
+        elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            module = "filesystem"
+            file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
+            # write the response to a tempfile
+            # this is necessary because trufflehog doesn't yet support reading from stdin
+            # https://github.com/trufflesecurity/trufflehog/issues/162
+            path = self.helpers.tempfile(file_data, pipe=False)
+
         if event.type == "CODE_REPOSITORY":
             host = event.host
         else:
@@ -108,41 +119,32 @@ class trufflehog(BaseModule):
             verified,
             source_metadata,
         ) in self.execute_trufflehog(module, path):
-            if verified:
-                data = {
-                    "severity": "High",
-                    "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "VULNERABILITY",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
-                )
-            else:
-                data = {
-                    "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
-                    "host": host,
-                }
-                if description:
-                    data["description"] += f" Description: [{description}]"
-                data["description"] += f" Raw result: [{raw_result}]"
-                if rawv2_result:
-                    data["description"] += f" RawV2 result: [{rawv2_result}]"
-                await self.emit_event(
-                    data,
-                    "FINDING",
-                    event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found possible secret ({{event.type}}): {raw_result}',
-                )
-
-    async def execute_trufflehog(self, module, path):
+            verified_str = "Verified" if verified else "Possible"
+            finding_type = "VULNERABILITY" if verified else "FINDING"
+            data = {
+                "description": f"{verified_str} Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]",
+            }
+            if host:
+                data["host"] = host
+            if finding_type == "VULNERABILITY":
+                data["severity"] = "High"
+            if description:
+                data["description"] += f" Description: [{description}]"
+            data["description"] += f" Raw result: [{raw_result}]"
+            if rawv2_result:
+                data["description"] += f" RawV2 result: [{rawv2_result}]"
+            await self.emit_event(
+                data,
+                finding_type,
+                event,
+                context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',
+            )
+
+        # clean up the tempfile when we're done with it
+        if event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
+            path.unlink(missing_ok=True)
+
+    async def execute_trufflehog(self, module, path=None, string=None):
         command = [
             "trufflehog",
             "--json",

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/test/test_step_1/test_events.py

@@ -149,6 +149,7 @@ async def test_events(events, helpers):
             "title": "HTTP%20RESPONSE",
             "url": "http://www.evilcorp.com:80",
             "input": "http://www.evilcorp.com:80",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
             "location": "/asdf",
             "status_code": 301,
         },
@@ -161,7 +162,13 @@ async def test_events(events, helpers):
 
     # http response url validation
     http_response_2 = scan.make_event(
-        {"port": "80", "url": "http://evilcorp.com:80/asdf"}, "HTTP_RESPONSE", dummy=True
+        {
+            "port": "80",
+            "url": "http://evilcorp.com:80/asdf",
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
+        "HTTP_RESPONSE",
+        dummy=True,
     )
     assert http_response_2.data["url"] == "http://evilcorp.com/asdf"
 
@@ -546,6 +553,10 @@ async def test_events(events, helpers):
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event)
     assert http_response.parent_id == scan.root_event.id
     assert http_response.data["input"] == "http://example.com:80"
+    assert (
+        http_response.raw_response
+        == 'HTTP/1.1 200 OK\r\nConnection: close\r\nAge: 526111\r\nCache-Control: max-age=604800\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Mon, 14 Nov 2022 17:14:27 GMT\r\nEtag: "3147526947+ident+gzip"\r\nExpires: Mon, 21 Nov 2022 17:14:27 GMT\r\nLast-Modified: Thu, 17 Oct 2019 07:18:26 GMT\r\nServer: ECS (agb/A445)\r\nVary: Accept-Encoding\r\nX-Cache: HIT\r\n\r\n<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n\n    <meta charset="utf-8" />\n    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n    <meta name="viewport" content="width=device-width, initial-scale=1" />\n    <style type="text/css">\n    body {\n        background-color: #f0f0f2;\n        margin: 0;\n        padding: 0;\n        font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n        \n    }\n    div {\n        width: 600px;\n        margin: 5em auto;\n        padding: 2em;\n        background-color: #fdfdff;\n        border-radius: 0.5em;\n        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n    }\n    a:link, a:visited {\n        color: #38488f;\n        text-decoration: none;\n    }\n    @media (max-width: 700px) {\n        div {\n            margin: 0 auto;\n            width: auto;\n        }\n    }\n    </style>    \n</head>\n\n<body>\n<div>\n    <h1>Example Domain</h1>\n    <p>This domain is for use in illustrative examples in documents. You may use this\n    domain in literature without prior coordination or asking for permission.</p>\n    <p><a href="https://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n'
+    )
     json_event = http_response.json(mode="graph")
     assert isinstance(json_event["data"], str)
     json_event = http_response.json()
@@ -906,7 +917,12 @@ def test_event_closest_host():
     assert event1.host == "evilcorp.com"
     # second event has a host + url
     event2 = scan.make_event(
-        {"method": "GET", "url": "http://www.evilcorp.com/asdf", "hash": {"header_mmh3": "1", "body_mmh3": "2"}},
+        {
+            "method": "GET",
+            "url": "http://www.evilcorp.com/asdf",
+            "hash": {"header_mmh3": "1", "body_mmh3": "2"},
+            "raw_header": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.evilcorp.com/asdf\r\n\r\n",
+        },
         "HTTP_RESPONSE",
         parent=event1,
     )

{bbot-2.3.0.5491rc0 → bbot-2.3.0.5515rc0}/bbot/test/test_step_1/test_helpers.py

@@ -122,7 +122,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert not helpers.is_dns_name("evilcorp.com:80")
     assert not helpers.is_dns_name("http://evilcorp.com:80")
     assert helpers.is_dns_name("evilcorp")
-    assert not helpers.is_dns_name("evilcorp", include_local=False)
+    assert helpers.is_dns_name("evilcorp.")
     assert helpers.is_dns_name("ドメイン.テスト")
     assert not helpers.is_dns_name("127.0.0.1")
     assert not helpers.is_dns_name("dead::beef")