bbot 2.6.0.6879rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.6.0.6879rc"
+__version__ = "v2.7.2.7254rc"
 
 from .scanner import Scanner, Preset
 

bbot/core/engine.py

@@ -636,7 +636,7 @@ class EngineServer(EngineBase):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/flags.py

@@ -6,6 +6,7 @@ flag_descriptions = {
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ class BloomFilter:
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""

bbot/core/helpers/dns/dns.py

@@ -38,7 +38,6 @@ class DNSHelper(EngineClient):
         _wildcard_cache (dict): Cache for wildcard detection results.
         _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.
         resolver_file (Path): File containing system's current resolver nameservers.
-        filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True.
 
     Args:
         parent_helper: The parent helper object with configuration details and utilities.

bbot/core/helpers/dns/engine.py

@@ -86,8 +86,6 @@ class DNSEngine(EngineServer):
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
-        self.filter_bad_ptrs = self.dns_config.get("filter_ptrs", True)
-
     async def resolve(self, query, **kwargs):
         """Resolve DNS names and IP addresses to their corresponding results.
 

bbot/core/helpers/files.py

@@ -9,7 +9,7 @@ from .misc import rm_at_exit
 log = logging.getLogger("bbot.core.helpers.files")
 
 
-def tempfile(self, content, pipe=True):
+def tempfile(self, content, pipe=True, extension=None):
     """
     Creates a temporary file or named pipe and populates it with content.
 
@@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True):
         >>> tempfile(["Another", "temp", "file"], pipe=False)
         '/home/user/.bbot/temp/someotherfile'
     """
-    filename = self.temp_filename()
+    filename = self.temp_filename(extension)
     rm_at_exit(filename)
     try:
         if type(content) not in (set, list, tuple):

bbot/core/helpers/git.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def sanitize_git_repo(repo_folder: Path):
+    # sanitizing the git config is infeasible since there are too many different ways to do evil things
+    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    config_file = repo_folder / ".git" / "config"
+    if config_file.exists():
+        config_file.rename(repo_folder / "git_config_original")
+    # move the index file
+    index_file = repo_folder / ".git" / "index"
+    if index_file.exists():
+        index_file.rename(repo_folder / "git_index_original")
+    # move the hooks folder
+    hooks_folder = repo_folder / ".git" / "hooks"
+    if hooks_folder.exists():
+        hooks_folder.rename(repo_folder / "git_hooks_original")

bbot/core/helpers/misc.py

@@ -17,6 +17,7 @@ from unidecode import unidecode  # noqa F401
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
+from .git import *  # noqa F401
 from .url import *  # noqa F401
 from ... import errors
 from . import regexes as bbot_regexes

bbot/core/helpers/ntlm.py

@@ -17,11 +17,9 @@ class StrStruct(object):
         self.alloc = alloc
         self.offset = offset
         self.raw = raw[offset : offset + length]
-        self.utf16 = False
 
         if len(self.raw) >= 2 and self.raw[1] == "\0":
             self.string = self.raw.decode("utf-16")
-            self.utf16 = True
         else:
             self.string = self.raw
 

bbot/core/helpers/regex.py

@@ -65,7 +65,7 @@ class RegexHelper:
 
         while tasks:  # While there are tasks pending
             # Wait for the first task to complete
-            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+            done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
 
             for task in done:
                 result = task.result()

bbot/core/modules.py

@@ -512,60 +512,6 @@ class ModuleLoader:
                         # then we have a module
                         return value
 
-    def recommend_dependencies(self, modules):
-        """
-        Returns a dictionary containing missing dependencies and their suggested resolutions
-
-        Needs work. For this we should probably be building a dependency graph
-        """
-        resolve_choices = {}
-        # step 1: build a dictionary containing event types and their associated modules
-        # {"IP_ADDRESS": set("masscan", "ipneighbor", ...)}
-        watched = {}
-        produced = {}
-        for modname in modules:
-            preloaded = self._preloaded.get(modname)
-            if preloaded:
-                for event_type in preloaded.get("watched_events", []):
-                    self.add_or_create(watched, event_type, modname)
-                for event_type in preloaded.get("produced_events", []):
-                    self.add_or_create(produced, event_type, modname)
-        watched_all = {}
-        produced_all = {}
-        for modname, preloaded in self.preloaded().items():
-            if preloaded:
-                for event_type in preloaded.get("watched_events", []):
-                    self.add_or_create(watched_all, event_type, modname)
-                for event_type in preloaded.get("produced_events", []):
-                    self.add_or_create(produced_all, event_type, modname)
-
-        # step 2: check to see if there are missing dependencies
-        for modname in modules:
-            preloaded = self._preloaded.get(modname)
-            module_type = preloaded.get("type", "unknown")
-            if module_type != "scan":
-                continue
-            watched_events = preloaded.get("watched_events", [])
-            missing_deps = {e: not self.check_dependency(e, modname, produced) for e in watched_events}
-            if all(missing_deps.values()):
-                for event_type in watched_events:
-                    if event_type == "SCAN":
-                        continue
-                    choices = produced_all.get(event_type, [])
-                    choices = set(choices)
-                    with suppress(KeyError):
-                        choices.remove(modname)
-                    if event_type not in resolve_choices:
-                        resolve_choices[event_type] = {}
-                    deps = resolve_choices[event_type]
-                    self.add_or_create(deps, "required_by", modname)
-                    for c in choices:
-                        choice_type = self._preloaded.get(c, {}).get("type", "unknown")
-                        if choice_type == "scan":
-                            self.add_or_create(deps, "recommended", c)
-
-        return resolve_choices
-
     def check_dependency(self, event_type, modname, produced):
         if event_type not in produced:
             return False

bbot/defaults.yml

@@ -187,8 +187,10 @@ url_extension_blacklist:
   - mov
   - flv
   - webm
-# Distribute URLs with these extensions only to httpx (these are omitted from output)
-url_extension_httpx_only:
+
+# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`
+# They are also excluded from output. If you want to see them in output, remove them from this list.
+url_extension_special:
   - js
 
 # These url extensions are almost always static, so we exclude them from modules that fuzz things

bbot/modules/apkpure.py

@@ -6,7 +6,7 @@ from bbot.modules.base import BaseModule
 class apkpure(BaseModule):
     watched_events = ["MOBILE_APP"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download android applications from apkpure.com",
         "created_date": "2024-10-11",

bbot/modules/base.py

@@ -53,6 +53,8 @@ class BaseModule:
 
         in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.
 
+        accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False.
+
         options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default.
 
         options_desc (Dict): Descriptions for options, e.g., {"api_key": "API Key"}. Empty dict by default.
@@ -97,7 +99,7 @@ class BaseModule:
     scope_distance_modifier = 0
     target_only = False
     in_scope_only = False
-
+    accept_url_special = False
     _module_threads = 1
     _batch_size = 1
 
@@ -785,10 +787,14 @@ class BaseModule:
             if "target" not in event.tags:
                 return False, "it did not meet target_only filter criteria"
 
-        # exclude certain URLs (e.g. javascript):
-        # TODO: revisit this after httpx rework
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, "its extension was listed in url_extension_httpx_only"
+        # limit js URLs to modules that opt in to receive them
+        if (not self.accept_url_special) and event.type.startswith("URL"):
+            extension = getattr(event, "url_extension", "")
+            if extension in self.scan.url_extension_special:
+                return (
+                    False,
+                    f"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs",
+                )
 
         return True, "precheck succeeded"
 

bbot/modules/dnsbimi.py

@@ -39,7 +39,7 @@ import re
 # Handle "v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;"
 # Handle "v=BIMI1;l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem"
 # Handle "v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem;"
-_bimi_regex = r"^v=(?P<v>BIMI1);* *(l=(?P<l>https*://[^;]*|)|);*( *a=((?P<a>https://[^;]*|)|);*)*$"
+_bimi_regex = r"^v=(?P<v>BIMI1);\s?(?:l=(?P<l>https?://[^;\s]{1,255})?)?;?(?:\s?a=(?P<a>https://[^;\s]{1,255})?;?)?$"
 bimi_regex = re.compile(_bimi_regex, re.I)
 
 
@@ -140,6 +140,3 @@ class dnsbimi(BaseModule):
 
     async def handle_event(self, event):
         await self.inspectBIMI(event, event.host)
-
-
-# EOF

bbot/modules/dnsdumpster.py

@@ -1,4 +1,4 @@
-import re
+import json
 
 from bbot.modules.templates.subdomain_enum import subdomain_enum
 
@@ -15,78 +15,61 @@ class dnsdumpster(subdomain_enum):
 
     base_url = "https://dnsdumpster.com"
 
+    async def setup(self):
+        self.apikey_regex = self.helpers.re.compile(r'<form[^>]*data-form-id="mainform"[^>]*hx-headers=\'([^\']*)\'')
+        return True
+
     async def query(self, domain):
         ret = []
-        # first, get the CSRF tokens
+        # first, get the JWT token from the main page
         res1 = await self.api_request(self.base_url)
         status_code = getattr(res1, "status_code", 0)
-        if status_code in [429]:
-            self.verbose(f'Too many requests "{status_code}"')
-            return ret
-        elif status_code not in [200]:
+        if status_code not in [200]:
             self.verbose(f'Bad response code "{status_code}" from DNSDumpster')
             return ret
-        else:
-            self.debug(f'Valid response code "{status_code}" from DNSDumpster')
-
-        html = self.helpers.beautifulsoup(res1.content, "html.parser")
-        if html is False:
-            self.verbose("BeautifulSoup returned False")
-            return ret
 
-        csrftoken = None
-        csrfmiddlewaretoken = None
+        # Extract JWT token from the form's hx-headers attribute using regex
+        jwt_token = None
         try:
-            for cookie in res1.headers.get("set-cookie", "").split(";"):
-                try:
-                    k, v = cookie.split("=", 1)
-                except ValueError:
-                    self.verbose("Error retrieving cookie")
-                    return ret
-                if k == "csrftoken":
-                    csrftoken = str(v)
-            csrfmiddlewaretoken = html.find("input", {"name": "csrfmiddlewaretoken"}).attrs.get("value", None)
-        except AttributeError:
-            pass
+            # Look for the form with data-form-id="mainform" and extract hx-headers
+            form_match = await self.helpers.re.search(self.apikey_regex, res1.text)
+            if form_match:
+                headers_json = form_match.group(1)
+                headers_data = json.loads(headers_json)
+                jwt_token = headers_data.get("Authorization")
+        except (AttributeError, json.JSONDecodeError, KeyError):
+            self.log.warning("Error obtaining JWT token")
+            return ret
 
-        # Abort if we didn't get the tokens
-        if not csrftoken or not csrfmiddlewaretoken:
-            self.verbose("Error obtaining CSRF tokens")
+        # Abort if we didn't get the JWT token
+        if not jwt_token:
+            self.verbose("Error obtaining JWT token")
             self.errorState = True
             return ret
         else:
-            self.debug("Successfully obtained CSRF tokens")
+            self.debug("Successfully obtained JWT token")
 
         if self.scan.stopping:
-            return
+            return ret
 
-        # Otherwise, do the needful
-        subdomains = set()
+        # Query the API with the JWT token
         res2 = await self.api_request(
-            f"{self.base_url}/",
+            "https://api.dnsdumpster.com/htmld/",
             method="POST",
-            cookies={"csrftoken": csrftoken},
-            data={
-                "csrfmiddlewaretoken": csrfmiddlewaretoken,
-                "targetip": str(domain).lower(),
-                "user": "free",
-            },
+            data={"target": str(domain).lower()},
             headers={
-                "origin": "https://dnsdumpster.com",
-                "referer": "https://dnsdumpster.com/",
+                "Authorization": jwt_token,
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Origin": "https://dnsdumpster.com",
+                "Referer": "https://dnsdumpster.com/",
+                "HX-Request": "true",
+                "HX-Target": "results",
+                "HX-Current-URL": "https://dnsdumpster.com/",
             },
         )
         status_code = getattr(res2, "status_code", 0)
         if status_code not in [200]:
-            self.verbose(f'Bad response code "{status_code}" from DNSDumpster')
-            return ret
-        html = self.helpers.beautifulsoup(res2.content, "html.parser")
-        if html is False:
-            self.verbose("BeautifulSoup returned False")
+            self.verbose(f'Bad response code "{status_code}" from DNSDumpster API')
             return ret
-        escaped_domain = re.escape(domain)
-        match_pattern = re.compile(r"^[\w\.-]+\." + escaped_domain + r"$")
-        for subdomain in html.findAll(text=match_pattern):
-            subdomains.add(str(subdomain).strip().lower())
 
-        return list(subdomains)
+        return await self.scan.extract_in_scope_hostnames(res2.text)

bbot/modules/dnstlsrpt.py

@@ -44,20 +44,17 @@ class dnstlsrpt(BaseModule):
         "emit_emails": True,
         "emit_raw_dns_records": False,
         "emit_urls": True,
-        "emit_vulnerabilities": True,
     }
     options_desc = {
         "emit_emails": "Emit EMAIL_ADDRESS events",
         "emit_raw_dns_records": "Emit RAW_DNS_RECORD events",
         "emit_urls": "Emit URL_UNVERIFIED events",
-        "emit_vulnerabilities": "Emit VULNERABILITY events",
     }
 
     async def setup(self):
         self.emit_emails = self.config.get("emit_emails", True)
         self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False)
         self.emit_urls = self.config.get("emit_urls", True)
-        self.emit_vulnerabilities = self.config.get("emit_vulnerabilities", True)
         return await super().setup()
 
     def _incoming_dedup_hash(self, event):
@@ -139,6 +136,3 @@ class dnstlsrpt(BaseModule):
                                                     tags=tags.append(f"tlsrpt-record-{key}"),
                                                     parent=event,
                                                 )
-
-
-# EOF

bbot/modules/docker_pull.py

@@ -8,7 +8,7 @@ from bbot.modules.base import BaseModule
 class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",

bbot/modules/emailformat.py

@@ -15,13 +15,29 @@ class emailformat(BaseModule):
 
     base_url = "https://www.email-format.com"
 
+    async def setup(self):
+        self.cfemail_regex = self.helpers.re.compile(r'data-cfemail="([0-9a-z]+)"')
+        return True
+
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
         url = f"{self.base_url}/d/{self.helpers.quote(query)}/"
         r = await self.api_request(url)
         if not r:
             return
-        for email in await self.helpers.re.extract_emails(r.text):
+
+        encrypted_emails = await self.helpers.re.findall(self.cfemail_regex, r.text)
+
+        for enc in encrypted_emails:
+            enc_len = len(enc)
+
+            if enc_len < 2 or enc_len % 2 != 0:
+                continue
+
+            key = int(enc[:2], 16)
+
+            email = "".join([chr(int(enc[i : i + 2], 16) ^ key) for i in range(2, enc_len, 2)]).lower()
+
             if email.endswith(query):
                 await self.emit_event(
                     email,

bbot/modules/filedownload.py

@@ -14,7 +14,7 @@ class filedownload(BaseModule):
 
     watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"]
     produced_events = ["FILESYSTEM"]
-    flags = ["active", "safe", "web-basic"]
+    flags = ["active", "safe", "web-basic", "download"]
     meta = {
         "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
         "created_date": "2023-10-11",

bbot/modules/git_clone.py

@@ -6,7 +6,7 @@ from bbot.modules.templates.github import github
 class git_clone(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Clone code github repositories",
         "created_date": "2024-03-08",
@@ -24,44 +24,69 @@ class git_clone(github):
 
     async def setup(self):
         output_folder = self.config.get("output_folder")
-        if output_folder:
-            self.output_dir = Path(output_folder) / "git_repos"
-        else:
-            self.output_dir = self.scan.temp_dir / "git_repos"
+        self.output_dir = Path(output_folder) / "git_repos" if output_folder else self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 
     async def filter_event(self, event):
-        if event.type == "CODE_REPOSITORY":
-            if "git" not in event.tags:
-                return False, "event is not a git repository"
+        if event.type == "CODE_REPOSITORY" and "git" not in event.tags:
+            return False, "event is not a git repository"
         return True
 
     async def handle_event(self, event):
-        repo_url = event.data.get("url")
-        repo_path = await self.clone_git_repository(repo_url)
-        if repo_path:
-            self.verbose(f"Cloned {repo_url} to {repo_path}")
-            codebase_event = self.make_event({"path": str(repo_path)}, "FILESYSTEM", tags=["git"], parent=event)
+        repository_url = event.data.get("url")
+        repository_path = await self.clone_git_repository(repository_url)
+        if repository_path:
+            self.verbose(f"Cloned {repository_url} to {repository_path}")
+            codebase_event = self.make_event({"path": str(repository_path)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
                 codebase_event,
-                context=f"{{module}} downloaded git repo at {repo_url} to {{event.type}}: {repo_path}",
+                context=f"{{module}} cloned git repository at {repository_url} to {{event.type}}: {repository_path}",
             )
 
     async def clone_git_repository(self, repository_url):
         owner = repository_url.split("/")[-2]
         folder = self.output_dir / owner
         self.helpers.mkdir(folder)
-        if self.api_key:
-            url = repository_url.replace("https://github.com", f"https://user:{self.api_key}@github.com")
-        else:
-            url = repository_url
-        command = ["git", "-C", folder, "clone", url]
+
+        command = ["git", "-C", folder, "clone", repository_url]
+        env = {"GIT_TERMINAL_PROMPT": "0"}
+
         try:
-            output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, check=True)
+            hostname = self.helpers.urlparse(repository_url).hostname
+            if hostname and self.api_key:
+                _, domain = self.helpers.split_domain(hostname)
+                # only use the api key if the domain is github.com
+                if domain == "github.com":
+                    env["GIT_HELPER"] = (
+                        f'!f() {{ case "$1" in get) '
+                        f"echo username=x-access-token; "
+                        f"echo password={self.api_key};; "
+                        f'esac; }}; f "$@"'
+                    )
+                    command = (
+                        command[:1]
+                        + [
+                            "-c",
+                            "credential.helper=",
+                            "-c",
+                            "credential.useHttpPath=true",
+                            "--config-env=credential.helper=GIT_HELPER",
+                        ]
+                        + command[1:]
+                    )
+
+            output = await self.run_process(command, env=env, check=True)
         except CalledProcessError as e:
-            self.debug(f"Error cloning {url}. STDERR: {repr(e.stderr)}")
+            self.debug(f"Error cloning {repository_url}. STDERR: {repr(e.stderr)}")
             return
 
         folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
-        return folder / folder_name
+        repo_folder = folder / folder_name
+
+        # sanitize the repo
+        # this moves the git config, index file, and hooks folder out of the .git folder to prevent nasty things
+        # Note: the index file can be regenerated by running "git checkout HEAD -- ."
+        self.helpers.sanitize_git_repo(repo_folder)
+
+        return repo_folder

bbot/modules/gitdumper.py

@@ -1,5 +1,4 @@
 import asyncio
-import regex as re
 from pathlib import Path
 from subprocess import CalledProcessError
 from bbot.modules.base import BaseModule
@@ -8,7 +7,7 @@ from bbot.modules.base import BaseModule
 class gitdumper(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download a leaked .git folder recursively or by fuzzing common names",
         "created_date": "2025-02-11",
@@ -35,7 +34,6 @@ class gitdumper(BaseModule):
         else:
             self.output_dir = self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
-        self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
         self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
         self.obj_regex = self.helpers.re.compile(r"[a-f0-9]{40}")
         self.pack_regex = self.helpers.re.compile(r"pack-([a-f0-9]{40})\.pack")
@@ -131,7 +129,6 @@ class gitdumper(BaseModule):
         else:
             result = await self.git_fuzz(repo_url, repo_folder)
         if result:
-            await self.sanitize_config(repo_folder)
             await self.git_checkout(repo_folder)
             codebase_event = self.make_event({"path": str(repo_folder)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
@@ -251,15 +248,6 @@ class gitdumper(BaseModule):
             self.debug(f"Unable to download git files to {folder}")
             return False
 
-    async def sanitize_config(self, folder):
-        config_file = folder / ".git/config"
-        if config_file.exists():
-            with config_file.open("r", encoding="utf-8", errors="ignore") as file:
-                content = file.read()
-                sanitized = await self.helpers.re.sub(self.unsafe_regex, r"# \g<0>", content)
-            with config_file.open("w", encoding="utf-8") as file:
-                file.write(sanitized)
-
     async def git_catfile(self, hash, option="-t", folder=Path()):
         command = ["git", "cat-file", option, hash]
         try:
@@ -270,8 +258,10 @@ class gitdumper(BaseModule):
         return output.stdout
 
     async def git_checkout(self, folder):
+        self.helpers.sanitize_git_repo(folder)
         self.verbose(f"Running git checkout to reconstruct the git repository at {folder}")
-        command = ["git", "checkout", "."]
+        # we do "checkout head -- ." because the sanitization deletes the index file, and it needs to be reconstructed
+        command = ["git", "checkout", "HEAD", "--", "."]
         try:
             await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
         except CalledProcessError as e: