bbot 2.3.2.5915rc0__py3-none-any.whl → 2.3.2.5927rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,4 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.3.2.5915rc"
+__version__ = "v2.3.2.5927rc"
 
 from .scanner import Scanner, Preset

bbot/core/helpers/regex.py

@@ -31,6 +31,10 @@ class RegexHelper:
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs)
 
+    async def sub(self, compiled_regex, *args, **kwargs):
+        self.ensure_compiled_regex(compiled_regex)
+        return await self.parent_helper.run_in_executor(compiled_regex.sub, *args, **kwargs)
+
     async def findall(self, compiled_regex, *args, **kwargs):
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.findall, *args, **kwargs)

bbot/core/helpers/web/client.py

@@ -83,6 +83,7 @@ class BBOTAsyncClient(httpx.AsyncClient):
         # TODO: re-enable this
         if self._target.in_scope(str(request.url)):
             for hk, hv in self._web_config.get("http_headers", {}).items():
+                hv = str(hv)
                 # don't clobber headers
                 if hk not in request.headers:
                     request.headers[hk] = hv

bbot/modules/git.py

@@ -5,7 +5,7 @@ from bbot.modules.base import BaseModule
 
 class git(BaseModule):
     watched_events = ["URL"]
-    produced_events = ["FINDING"]
+    produced_events = ["FINDING", "CODE_REPOSITORY"]
     flags = ["active", "safe", "web-basic", "code-enum"]
     meta = {
         "description": "Check for exposed .git repositories",
@@ -37,3 +37,10 @@ class git(BaseModule):
                         event,
                         context="{module} detected {event.type}: {description}",
                     )
+                    await self.emit_event(
+                        {"url": url.rstrip("config")},
+                        "CODE_REPOSITORY",
+                        event,
+                        tags="git_directory",
+                        context="{module} detected {event.type}: {description}",
+                    )

bbot/modules/gitdumper.py

@@ -0,0 +1,270 @@
+import regex as re
+from pathlib import Path
+from subprocess import CalledProcessError
+from bbot.modules.base import BaseModule
+
+
+class gitdumper(BaseModule):
+    watched_events = ["CODE_REPOSITORY"]
+    produced_events = ["FILESYSTEM"]
+    flags = ["passive", "safe", "slow", "code-enum"]
+    meta = {
+        "description": "Download a leaked .git folder recursively or by fuzzing common names",
+        "created_date": "2025-02-11",
+        "author": "@domwhewell-sage",
+    }
+    options = {
+        "output_folder": "",
+        "fuzz_tags": False,
+        "max_semanic_version": 10,
+    }
+    options_desc = {
+        "output_folder": "Folder to download repositories to",
+        "fuzz_tags": "Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version",
+        "max_semanic_version": "Maximum version number to fuzz for (default < v10.10.10)",
+    }
+
+    scope_distance_modifier = 2
+
+    async def setup(self):
+        self.urls_downloaded = set()
+        output_folder = self.config.get("output_folder")
+        if output_folder:
+            self.output_dir = Path(output_folder) / "git_repos"
+        else:
+            self.output_dir = self.scan.home / "git_repos"
+        self.helpers.mkdir(self.output_dir)
+        self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
+        self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
+        self.obj_regex = self.helpers.re.compile(r"[a-f0-9]{40}")
+        self.pack_regex = self.helpers.re.compile(r"pack-([a-f0-9]{40})\.pack")
+        self.git_files = [
+            "HEAD",
+            "description",
+            "config",
+            "COMMIT_EDITMSG",
+            "index",
+            "packed-refs",
+            "info/refs",
+            "info/exclude",
+            "refs/stash",
+            "refs/wip/index/refs/heads/master",
+            "refs/wip/wtree/refs/heads/master",
+            "logs/HEAD",
+            "objects/info/packs",
+        ]
+        self.info("Compiling fuzz list with common branch names")
+        branch_names = [
+            "bugfix",
+            "daily",
+            "dev",
+            "develop",
+            "development",
+            "feat",
+            "feature",
+            "fix",
+            "hotfix",
+            "integration",
+            "issue",
+            "main",
+            "master",
+            "ng",
+            "prod",
+            "production",
+            "qa",
+            "quickfix",
+            "release",
+            "stable",
+            "stage",
+            "staging",
+            "test",
+            "testing",
+            "wip",
+        ]
+        url_patterns = [
+            "logs/refs/heads/{branch}",
+            "logs/refs/remotes/origin/{branch}",
+            "refs/remotes/origin/{branch}",
+            "refs/heads/{branch}",
+        ]
+        for branch in branch_names:
+            for pattern in url_patterns:
+                self.git_files.append(pattern.format(branch=branch))
+        self.fuzz_tags = self.config.get("fuzz_tags", "10")
+        self.max_semanic_version = self.config.get("max_semanic_version", "10")
+        if self.fuzz_tags:
+            self.info("Adding symantec version tags to fuzz list")
+            for major in range(self.max_semanic_version):
+                for minor in range(self.max_semanic_version):
+                    for patch in range(self.max_semanic_version):
+                        self.verbose(f"{major}.{minor}.{patch}")
+                        self.git_files.append(f"refs/tags/{major}.{minor}.{patch}")
+                        self.verbose(f"v{major}.{minor}.{patch}")
+                        self.git_files.append(f"refs/tags/v{major}.{minor}.{patch}")
+        else:
+            self.info("Adding symantec version tags to fuzz list (v0.0.1, 0.0.1, v1.0.0, 1.0.0)")
+            for path in ["refs/tags/v0.0.1", "refs/tags/0.0.1", "refs/tags/v1.0.0", "refs/tags/1.0.0"]:
+                self.git_files.append(path)
+        return await super().setup()
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "git-directory" not in event.tags:
+                return False, "event is not a leaked .git directory"
+        return True
+
+    async def handle_event(self, event):
+        repo_url = event.data.get("url")
+        self.info(f"Processing leaked .git directory at {repo_url}")
+        repo_folder = self.output_dir / self.helpers.tagify(repo_url)
+        self.helpers.mkdir(repo_folder)
+        dir_listing = await self.directory_listing_enabled(repo_url)
+        if dir_listing:
+            urls = await self.recursive_dir_list(dir_listing)
+            result = await self.download_files(urls, repo_folder)
+        else:
+            result = await self.git_fuzz(repo_url, repo_folder)
+        if result:
+            await self.sanitize_config(repo_folder)
+            await self.git_checkout(repo_folder)
+            codebase_event = self.make_event({"path": str(repo_folder)}, "FILESYSTEM", tags=["git"], parent=event)
+            await self.emit_event(
+                codebase_event,
+                context=f"{{module}} cloned git repo at {repo_url} to {{event.type}}: {str(repo_folder)}",
+            )
+        else:
+            self.helpers.rm_rf(repo_folder)
+
+    async def directory_listing_enabled(self, repo_url):
+        response = await self.helpers.request(repo_url)
+        if "<title>Index of" in getattr(response, "text", ""):
+            self.info(f"Directory listing enabled at {repo_url}")
+            return response
+        return None
+
+    async def recursive_dir_list(self, dir_listing):
+        file_list = []
+        soup = self.helpers.beautifulsoup(dir_listing.text, "html.parser")
+        links = soup.find_all("a")
+        for link in links:
+            href = link["href"]
+            if href == "../" or href == "/":
+                continue
+            if href.endswith("/"):
+                folder_url = self.helpers.urljoin(str(dir_listing.url), href)
+                response = await self.helpers.request(folder_url)
+                if getattr(response, "status_code", 0) == 200:
+                    file_list.extend(await self.recursive_dir_list(response))
+            else:
+                file_url = self.helpers.urljoin(str(dir_listing.url), href)
+                # Ensure the file is in the same domain as the directory listing
+                if file_url.startswith(str(dir_listing.url)):
+                    url = self.helpers.urlparse(file_url)
+                    file_list.append(url)
+        return file_list
+
+    async def git_fuzz(self, repo_url, repo_folder):
+        self.info("Directory listing not enabled, fuzzing common git files")
+        url_list = []
+        for file in self.git_files:
+            url_list.append(self.helpers.urlparse(self.helpers.urljoin(repo_url, file)))
+        result = await self.download_files(url_list, repo_folder)
+        if result:
+            await self.download_current_branch(repo_url, repo_folder)
+            await self.download_git_objects(repo_url, repo_folder)
+            await self.download_git_packs(repo_url, repo_folder)
+            return True
+        else:
+            return False
+
+    async def download_current_branch(self, repo_url, repo_folder):
+        for branch in await self.regex_files(self.ref_regex, file=repo_folder / ".git/HEAD"):
+            await self.download_files(
+                [self.helpers.urlparse(self.helpers.urljoin(repo_url, f"refs/heads/{branch}"))], repo_folder
+            )
+
+    async def download_git_objects(self, url, folder):
+        for object in await self.regex_files(self.obj_regex, folder=folder):
+            await self.download_object(object, url, folder)
+
+    async def download_git_packs(self, url, folder):
+        url_list = []
+        for sha1 in await self.regex_files(self.pack_regex, file=folder / ".git/objects/info/packs"):
+            url_list.append(self.helpers.urlparse(self.helpers.urljoin(url, f"objects/pack/pack-{sha1}.idx")))
+            url_list.append(self.helpers.urlparse(self.helpers.urljoin(url, f"objects/pack/pack-{sha1}.pack")))
+        if url_list:
+            await self.download_files(url_list, folder)
+
+    async def regex_files(self, regex, folder=Path(), file=Path(), files=[]):
+        results = []
+        if folder:
+            if folder.is_dir():
+                for file_path in folder.rglob("*"):
+                    if file_path.is_file():
+                        results.extend(await self.regex_file(regex, file_path))
+        if files:
+            for file in files:
+                results.extend(await self.regex_file(regex, file))
+        if file:
+            results.extend(await self.regex_file(regex, file))
+        return results
+
+    async def regex_file(self, regex, file=Path()):
+        if file.exists() and file.is_file():
+            with file.open("r", encoding="utf-8", errors="ignore") as file:
+                content = file.read()
+                matches = await self.helpers.re.findall(regex, content)
+                if matches:
+                    return matches
+        return []
+
+    async def download_object(self, object, repo_url, repo_folder):
+        await self.download_files(
+            [self.helpers.urlparse(self.helpers.urljoin(repo_url, f"objects/{object[:2]}/{object[2:]}"))], repo_folder
+        )
+        output = await self.git_catfile(object, option="-p", folder=repo_folder)
+        for obj in await self.helpers.re.findall(self.obj_regex, output):
+            await self.download_object(obj, repo_url, repo_folder)
+
+    async def download_files(self, urls, folder):
+        for url in urls:
+            git_index = url.path.find(".git")
+            file_url = url.geturl()
+            filename = folder / url.path[git_index:]
+            self.helpers.mkdir(filename.parent)
+            if hash(str(file_url)) not in self.urls_downloaded:
+                self.verbose(f"Downloading {file_url} to {filename}")
+                await self.helpers.download(file_url, filename=filename, warn=False)
+                self.urls_downloaded.add(hash(str(file_url)))
+        if any(folder.rglob("*")):
+            return True
+        else:
+            self.debug(f"Unable to download git files to {folder}")
+            return False
+
+    async def sanitize_config(self, folder):
+        config_file = folder / ".git/config"
+        if config_file.exists():
+            with config_file.open("r", encoding="utf-8", errors="ignore") as file:
+                content = file.read()
+                sanitized = await self.helpers.re.sub(self.unsafe_regex, r"# \g<0>", content)
+            with config_file.open("w", encoding="utf-8") as file:
+                file.write(sanitized)
+
+    async def git_catfile(self, hash, option="-t", folder=Path()):
+        command = ["git", "cat-file", option, hash]
+        try:
+            output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
+        except CalledProcessError:
+            return ""
+
+        return output.stdout
+
+    async def git_checkout(self, folder):
+        self.verbose(f"Running git checkout to reconstruct the git repository at {folder}")
+        command = ["git", "checkout", "."]
+        try:
+            await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
+        except CalledProcessError as e:
+            # Still emit the event even if the checkout fails
+            self.debug(f"Error running git checkout in {folder}. STDERR: {repr(e.stderr)}")

bbot/test/test_step_2/module_tests/test_module_gitdumper.py

@@ -0,0 +1,385 @@
+from pathlib import Path
+from .base import ModuleTestBase
+
+
+class TestGitDumper_Dirlisting(ModuleTestBase):
+    targets = [
+        "http://127.0.0.1:8888/test",
+    ]
+
+    modules_overrides = ["git", "gitdumper", "httpx"]
+
+    index_html = """<html>
+        <head>
+            <title>Index of /.git</title>
+        </head>
+        <body>
+            <h1>Index of /.git</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='/test/.git/branches/'>&lt;branches&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/COMMIT_EDITMSG'>COMMIT_EDITMSG</a></td><td>157B</td></tr>
+                <tr><td><a href='/test/.git/config'>config</a></td><td>157B</td></tr>
+                <tr><td><a href='/test/.git/description'>description</a></td><td>73B</td></tr>
+                <tr><td><a href='/test/.git/HEAD'>HEAD</a></td><td>23B</td></tr>
+                <tr><td><a href='/test/.git/hooks/'>&lt;hooks&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/info/'>&lt;info&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/'>&lt;objects&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/index'>index</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/refs/'>&lt;refs&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/logs/'>&lt;logs&gt;</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    info_index = """<html>
+        <head>
+            <title>Index of /.git/info</title>
+        </head>
+        <body>
+            <h1>Index of /.git/info</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/info/exclude'>exclude</a></td><td>240B</td></tr>
+                <tr><td><a href='http://exclude.com/excludeme'>excludeme</a></td><td>0B</td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    objects_index = """<html>
+        <head>
+            <title>Index of /.git/objects</title>
+        </head>
+        <body>
+            <h1>Index of /.git/objects</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/05/'>&lt;05&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/34/'>&lt;34&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/c2/'>&lt;c2&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/pack/'>&lt;pack&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/info/'>&lt;info&gt;</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    objects_o5_index = """<html>
+        <head>
+            <title>Index of /.git/objects/05</title>
+        </head>
+        <body>
+            <h1>Index of /.git/objects/05</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/05/27e6bd2d76b45e2933183f1b506c7ac49f5872'>27e6bd2d76b45e2933183f1b506c7ac49f5872</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    objects_34_index = """<html>
+        <head>
+            <title>Index of /.git/objects/34</title>
+        </head>
+        <body>
+            <h1>Index of /.git/objects/34</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/34/dc86f0247798892a89553e7c5c2d5aa06c2c5b'>dc86f0247798892a89553e7c5c2d5aa06c2c5b</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    objects_c2_index = """<html>
+        <head>
+            <title>Index of /.git/objects/c2</title>
+        </head>
+        <body>
+            <h1>Index of /.git/objects/c2</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/objects/c2/69d751b8e2fd0be0d0dc7a6437a4dce4ec0200'>69d751b8e2fd0be0d0dc7a6437a4dce4ec0200</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    refs_index = """<html>
+        <head>
+            <title>Index of /.git/refs</title>
+        </head>
+        <body>
+            <h1>Index of /.git/refs</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/refs/heads/'>&lt;heads&gt;</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/refs/tags/'>&lt;tags&gt;</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>
+    """
+
+    refs_heads_index = """<html>
+        <head>
+            <title>Index of /.git/refs/heads</title>
+        </head>
+        <body>
+            <h1>Index of /.git/refs/heads</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/refs/heads/master'>master</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>
+    """
+
+    logs_index = """<html>
+        <head>
+            <title>Index of /.git/logs</title>
+        </head>
+        <body>
+            <h1>Index of /.git/logs</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/logs/HEAD'>HEAD</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/logs/refs/'>&lt;tags&gt;</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>
+    """
+
+    logs_refs_index = """<html>
+        <head>
+            <title>Index of /.git/logs/refs</title>
+        </head>
+        <body>
+            <h1>Index of /.git/logs/refs</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/logs/refs/heads/'>&lt;heads&gt;</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>
+    """
+
+    logs_refs_heads_index = """<html>
+        <head>
+            <title>Index of /.git/logs/refs/heads</title>
+        </head>
+        <body>
+            <h1>Index of /.git/logs/refs/heads</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+                <tr><td><a href='/test/.git/logs/refs/heads/master'>master</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>
+    """
+
+    empty_index = """<html>
+        <head>
+            <title>Index of /.git/...</title>
+        </head>
+        <body>
+            <h1>Index of /.git/...</h1>
+            <table>
+                <tr><th>Name</th><th>Size</th></tr>
+                <tr><td><a href='../'>[..]</a></td><td></td></tr>
+            </table>
+        </body>
+    </html>"""
+
+    git_head = "ref: refs/heads/master"
+
+    refs_head = "34dc86f0247798892a89553e7c5c2d5aa06c2c5b"
+
+    logs_head = "0000000000000000000000000000000000000000 34dc86f0247798892a89553e7c5c2d5aa06c2c5b Test <test@test.com> 1738516534 +0000	commit (initial): Initial commit"
+
+    logs_master_head = "0000000000000000000000000000000000000000 34dc86f0247798892a89553e7c5c2d5aa06c2c5b Test <test@test.com> 1738516534 +0000	commit (initial): Initial commit"
+
+    git_description = "Unnamed repository; edit this file 'description' to name the repository."
+
+    git_commit_editmsg = "Initial commit"
+
+    git_config = """[core]
+    repositoryformatversion = 0
+    filemode = true
+    bare = false
+    logallrefupdates = true"""
+
+    git_exclude = """# git ls-files --others --exclude-from=.git/info/exclude
+    # Lines that start with '#' are comments.
+    # For a project mostly in C, the following would be a good set of
+    # exclude patterns (uncomment them if you want to use them):
+    # *.[oa]
+    # *~"""
+
+    filebytes_gitindex = b"DIRC\x00\x00\x00\x02\x00\x00\x00\x01g\x9f\xbe\x04\x14\xfcb\xd1g\x9f\xbe\x04\x14\xfcb\xd1\x00\x00\x08 \x00\x04aD\x00\x00\x81\xa4\x00\x00\x03\xe8\x00\x00\x03\xe8\x00\x00\x00\x0f\x05'\xe6\xbd-v\xb4^)3\x18?\x1bPlz\xc4\x9fXr\x00\x08test.txt\x00\x00TREE\x00\x00\x00\x19\x001 0\n\xc2i\xd7Q\xb8\xe2\xfd\x0b\xe0\xd0\xdczd7\xa4\xdc\xe4\xec\x02\x00\xe8m|iw\xbb\xd6\x88;f\xdbW\x10yY\xd2\xb0G\xcfJ"
+    filebytes_27e6bd2d76b45e2933183f1b506c7ac49f5872 = (
+        b"x\x01K\xca\xc9OR04e\x08\xc9\xc8,V\x00\xa2D\x85\x92\xd4\xe2\x12.\x00U\xab\x07%"
+    )
+    filebytes_dc86f0247798892a89553e7c5c2d5aa06c2c5b = b"x\x01\x9d\x8dK\n\x021\x10D]\xe7\x14\xbd\x17\x86\xce?\x82\x88\x0b7\x9e\xc0u\xa6\xd3:\x81\xc4\xc0\x18\x99\xeb\x1b\x98\x1bX\xbbzP\xaf\xa8\xd5\x9a;\xc8\xa0\x0f}e\x06R\xee\x94\xbc\x95s`\xf5L83&L\xe4\xa33\xdaG\x93\x88\r\x13*D\x11\xbf}i+\xdcZ\x85\xc7\xc2\x1b\x97\x02\xe7\xd4\xea\xb4\xed\xe5\xfa\x89/\x9e\xa8\xd5\x0bH\xaf\x83\x95\xcej\x03G\x1c\x11\x83\x8e\xcf\xce\xff\xad\xc5\xfd\x9d{\x8e\x05v\x8d\xf8\x01\xfaF<\x05"
+    filebytes_69d751b8e2fd0be0d0dc7a6437a4dce4ec0200 = b"x\x01+)JMU06c040031Q(I-.\xd1+\xa9(a`U\x7f\xb6W\xb7lK\x9c\xa6\xb1\x84\xbdt@N\xd5\x91\xf9\x11E\x00*\x05\x0e\x8c"
+
+    async def setup_after_prep(self, module_test):
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/"}, respond_args={"response_data": self.index_html}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/index"}, respond_args={"response_data": self.filebytes_gitindex}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/COMMIT_EDITMSG"}, respond_args={"response_data": self.git_commit_editmsg}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/config"}, respond_args={"response_data": self.git_config}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/branches/"}, respond_args={"response_data": self.empty_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/description"}, respond_args={"response_data": self.git_description}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/HEAD"}, respond_args={"response_data": self.git_head}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/hooks/"}, respond_args={"response_data": self.empty_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/info/"}, respond_args={"response_data": self.info_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/info/exclude"}, respond_args={"response_data": self.git_exclude}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/"}, respond_args={"response_data": self.objects_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/05/"}, respond_args={"response_data": self.objects_o5_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/05/27e6bd2d76b45e2933183f1b506c7ac49f5872"},
+            respond_args={"response_data": self.filebytes_27e6bd2d76b45e2933183f1b506c7ac49f5872},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/34/"}, respond_args={"response_data": self.objects_34_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/34/dc86f0247798892a89553e7c5c2d5aa06c2c5b"},
+            respond_args={"response_data": self.filebytes_dc86f0247798892a89553e7c5c2d5aa06c2c5b},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/c2/"}, respond_args={"response_data": self.objects_c2_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/c2/69d751b8e2fd0be0d0dc7a6437a4dce4ec0200"},
+            respond_args={"response_data": self.filebytes_69d751b8e2fd0be0d0dc7a6437a4dce4ec0200},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/info/"}, respond_args={"response_data": self.empty_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/pack/"}, respond_args={"response_data": self.empty_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/refs/"}, respond_args={"response_data": self.refs_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/refs/heads/"}, respond_args={"response_data": self.refs_heads_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/refs/heads/master"}, respond_args={"response_data": self.refs_head}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/refs/tags/"}, respond_args={"response_data": self.empty_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/"}, respond_args={"response_data": self.logs_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/refs/"}, respond_args={"response_data": self.logs_refs_index}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/refs/heads/"},
+            respond_args={"response_data": self.logs_refs_heads_index},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/refs/heads/master"},
+            respond_args={"response_data": self.logs_master_head},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/HEAD"}, respond_args={"response_data": self.logs_head}
+        )
+
+    def check(self, module_test, events):
+        assert any(
+            e.type == "CODE_REPOSITORY"
+            and "git-directory" in e.tags
+            and e.data["url"] == "http://127.0.0.1:8888/test/.git/"
+            for e in events
+        )
+        filesystem_events = [
+            e
+            for e in events
+            if e.type == "FILESYSTEM" and "http-127-0-0-1-8888-test-git" in e.data["path"] and "git" in e.tags
+        ]
+        assert 1 == len(filesystem_events), "Failed to git clone CODE_REPOSITORY"
+        filesystem_event = filesystem_events[0]
+        folder = Path(filesystem_event.data["path"])
+        assert folder.is_dir(), "Destination folder doesn't exist"
+        with open(folder / "test.txt") as f:
+            content = f.read()
+            assert content == "This is a test\n", "File content doesn't match"
+
+
+class TestGitDumper_NoDirlisting(TestGitDumper_Dirlisting):
+    async def setup_after_prep(self, module_test):
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/index"}, respond_args={"response_data": self.filebytes_gitindex}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/COMMIT_EDITMSG"}, respond_args={"response_data": self.git_commit_editmsg}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/config"}, respond_args={"response_data": self.git_config}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/description"}, respond_args={"response_data": self.git_description}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/HEAD"}, respond_args={"response_data": self.git_head}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/info/exclude"}, respond_args={"response_data": self.git_exclude}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/05/27e6bd2d76b45e2933183f1b506c7ac49f5872"},
+            respond_args={"response_data": self.filebytes_27e6bd2d76b45e2933183f1b506c7ac49f5872},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/34/dc86f0247798892a89553e7c5c2d5aa06c2c5b"},
+            respond_args={"response_data": self.filebytes_dc86f0247798892a89553e7c5c2d5aa06c2c5b},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/objects/c2/69d751b8e2fd0be0d0dc7a6437a4dce4ec0200"},
+            respond_args={"response_data": self.filebytes_69d751b8e2fd0be0d0dc7a6437a4dce4ec0200},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/refs/heads/master"}, respond_args={"response_data": self.refs_head}
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/refs/heads/master"},
+            respond_args={"response_data": self.logs_master_head},
+        )
+        module_test.set_expect_requests(
+            expect_args={"uri": "/test/.git/logs/HEAD"}, respond_args={"response_data": self.logs_head}
+        )

{bbot-2.3.2.5915rc0.dist-info → bbot-2.3.2.5927rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.3.2.5915rc0
+Version: 2.3.2.5927rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.3.2.5915rc0.dist-info → bbot-2.3.2.5927rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=ieE73Fq94Re3L1L6yRrfMcyZnoY5aJWs2V9feNleIYw,130
+bbot/__init__.py,sha256=Km7IhIhm1aKivSPyzhmNO0CUdTkPtzm21T_bank57aQ,130
 bbot/cli.py,sha256=hrzJX07sK3psSQWa461BXFuOxgCA94iztsw8syLdpNw,10830
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -34,12 +34,12 @@ bbot/core/helpers/names_generator.py,sha256=x6nZfEPKMwv3qR_RI4U6TBNbo6PgCF4fqbld
 bbot/core/helpers/ntlm.py,sha256=P2Xj4-GPos2iAzw4dfk0FJp6oGyycGhu2x6sLDVjYjs,2573
 bbot/core/helpers/process.py,sha256=00uRpLMFi3Pt3uT8qXwAIhsXdoa7h-ifoXh0sGYgwqs,1702
 bbot/core/helpers/ratelimiter.py,sha256=fQp5mKfqfCkDkZzgntDu4NWlRsWSMCto0V8vaV8-34k,2115
-bbot/core/helpers/regex.py,sha256=yK8F6b2lOthvyNfZ62pqKXiF_LLFjblzH7HSo6hsuOE,4168
+bbot/core/helpers/regex.py,sha256=YCyvK78piK8p4YMTPH9hD1mILkZtPzLC0lNRf0vsj9g,4371
 bbot/core/helpers/regexes.py,sha256=whzgjomHsXVvKtvXLS037EMPxT9RSOxdWukpUHY8Pfc,5818
 bbot/core/helpers/url.py,sha256=1NDrvirODzzD6Mcssu-4WDNerMeMdekHCFzhRCS0m3g,5947
 bbot/core/helpers/validators.py,sha256=RBRCWpU-tKvcThx-KnEePpWTjsIxfQ_CrPEUBmiqPF0,9695
 bbot/core/helpers/web/__init__.py,sha256=pIEkL3DhjaGTSmZ7D3yKKYwWpntoLRILekV2wWsbsws,27
-bbot/core/helpers/web/client.py,sha256=UTjNnsAz-bPO6GEc1lEm_ZhatwaVOIGG7nRIn7KdZAQ,3562
+bbot/core/helpers/web/client.py,sha256=12lKbyWJQpVKBnKGCuMNNUn47ifs1fsB02nebceX75Y,3591
 bbot/core/helpers/web/engine.py,sha256=mzXpYmlB1pvNSXs8FuliGMv7myUwAcQWTtnMHqblMHA,8875
 bbot/core/helpers/web/ssl_context.py,sha256=aWVgl-d0HoE8B4EBKNxaa5UAzQmx79DjDByfBw9tezo,356
 bbot/core/helpers/web/web.py,sha256=sesBGwqXRmePwQIScfg8PbrlCPMArQVj55E1aJsWctI,23685
@@ -103,8 +103,9 @@ bbot/modules/filedownload.py,sha256=TOxftfxguaRSEKI8oG79XVRQqUGg1_IhYDYl_Jw9eYc,
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=zeehQb9akBSbHW9dF4icH8Vfd8LqoTrpIvnQEEMWes8,1311
 bbot/modules/generic_ssrf.py,sha256=KFdcHpUV9-Z7oN7emzbirimsNc2xZ_1IFqnsfIkEbcM,9196
-bbot/modules/git.py,sha256=CMDarsmBemZEzZSeQTzB70XD8IRdwdG39zXpwDdgZbw,1383
+bbot/modules/git.py,sha256=zmHeI0bn181T1P8C55HSebkdVGLTpzGxPc-LRqiHrbc,1723
 bbot/modules/git_clone.py,sha256=XFZXx0k97EMY3E5PZzdNvqQzZddOfRMaVp5ol2gk11s,2468
+bbot/modules/gitdumper.py,sha256=XBYt6oSXm09FJVdH37zrn9T1Nhqc0zK4KLugMevedOw,11531
 bbot/modules/github_codesearch.py,sha256=a-r2vE9N9WyBpFUiKCsg0TK4Qn7DaEGyVRTUKzkDLWA,3641
 bbot/modules/github_org.py,sha256=WM18vJCHuOHJJ5rPzQzQ3Pmp7XPPuaMeVgNfW-FlO0k,8938
 bbot/modules/github_workflows.py,sha256=RDtzR0DC2sqiWzMtiqlrCSwtZHWL2MoIJBKd6LVTAdI,9720
@@ -333,6 +334,7 @@ bbot/test/test_step_2/module_tests/test_module_fullhunt.py,sha256=NblfNHQrE82j-c
 bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py,sha256=ZhfZpH0QTl6_YftGoZzZk6_2x0ZDnWjZ7vNZMTibBHw,3228
 bbot/test/test_step_2/module_tests/test_module_git.py,sha256=gyBS3vZUWAyatGlcY26mGOYeqXSqJA5pbhJWgTmLqNo,1656
 bbot/test/test_step_2/module_tests/test_module_git_clone.py,sha256=Mo0Q7bCXcrkGWJc3-u5y4sdfC13ei-qj79aKvEbnkk4,13198
+bbot/test/test_step_2/module_tests/test_module_gitdumper.py,sha256=ya_eQUQk0344G7iqBYMls2z5H-bYM87rydbz-ACR2Ng,17461
 bbot/test/test_step_2/module_tests/test_module_github_codesearch.py,sha256=M50xBiGG2EuPGXDJU6uFsSUE-fhqZl3CzYtNdszW7LM,4735
 bbot/test/test_step_2/module_tests/test_module_github_org.py,sha256=5tKO6NH4TPBeIdeTf7Bz9PUZ1pcvKsjrG0nFhc3YgT0,25458
 bbot/test/test_step_2/module_tests/test_module_github_workflows.py,sha256=o_teEaskm3H22QEKod5KJayFvvcgOQoG4eItGWv8C8E,38006
@@ -423,8 +425,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.3.2.5915rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.3.2.5915rc0.dist-info/METADATA,sha256=1C-ozr_Q04_hBUMFFgChTaoKR3mumIIDZcz5LeBJKqU,18224
-bbot-2.3.2.5915rc0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-bbot-2.3.2.5915rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.3.2.5915rc0.dist-info/RECORD,,
+bbot-2.3.2.5927rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.3.2.5927rc0.dist-info/METADATA,sha256=7enWDVSsYM9dt9R4TIdCBoPZJttapvE95IOSAwYndCM,18224
+bbot-2.3.2.5927rc0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+bbot-2.3.2.5927rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.3.2.5927rc0.dist-info/RECORD,,