bbot 2.7.0.7116rc0__py3-none-any.whl → 2.7.1__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.7.0.7116rc"
+__version__ = "v2.7.1"
 
 from .scanner import Scanner, Preset
 

bbot/core/helpers/git.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def sanitize_git_repo(repo_folder: Path):
+    # sanitizing the git config is infeasible since there are too many different ways to do evil things
+    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    config_file = repo_folder / ".git" / "config"
+    if config_file.exists():
+        config_file.rename(repo_folder / "git_config_original")
+    # move the index file
+    index_file = repo_folder / ".git" / "index"
+    if index_file.exists():
+        index_file.rename(repo_folder / "git_index_original")
+    # move the hooks folder
+    hooks_folder = repo_folder / ".git" / "hooks"
+    if hooks_folder.exists():
+        hooks_folder.rename(repo_folder / "git_hooks_original")

bbot/core/helpers/misc.py

@@ -17,6 +17,7 @@ from unidecode import unidecode  # noqa F401
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
+from .git import *  # noqa F401
 from .url import *  # noqa F401
 from ... import errors
 from . import regexes as bbot_regexes

bbot/modules/git_clone.py

@@ -82,4 +82,11 @@ class git_clone(github):
             return
 
         folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
-        return folder / folder_name
+        repo_folder = folder / folder_name
+
+        # sanitize the repo
+        # this moves the git config, index file, and hooks folder out of the .git folder to prevent nasty things
+        # Note: the index file can be regenerated by running "git checkout HEAD -- ."
+        self.helpers.sanitize_git_repo(repo_folder)
+
+        return repo_folder

bbot/modules/gitdumper.py

@@ -1,5 +1,4 @@
 import asyncio
-import regex as re
 from pathlib import Path
 from subprocess import CalledProcessError
 from bbot.modules.base import BaseModule
@@ -35,7 +34,6 @@ class gitdumper(BaseModule):
         else:
             self.output_dir = self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
-        self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
         self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
         self.obj_regex = self.helpers.re.compile(r"[a-f0-9]{40}")
         self.pack_regex = self.helpers.re.compile(r"pack-([a-f0-9]{40})\.pack")
@@ -131,7 +129,6 @@ class gitdumper(BaseModule):
         else:
             result = await self.git_fuzz(repo_url, repo_folder)
         if result:
-            await self.sanitize_config(repo_folder)
             await self.git_checkout(repo_folder)
             codebase_event = self.make_event({"path": str(repo_folder)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
@@ -251,15 +248,6 @@ class gitdumper(BaseModule):
             self.debug(f"Unable to download git files to {folder}")
             return False
 
-    async def sanitize_config(self, folder):
-        config_file = folder / ".git/config"
-        if config_file.exists():
-            with config_file.open("r", encoding="utf-8", errors="ignore") as file:
-                content = file.read()
-                sanitized = await self.helpers.re.sub(self.unsafe_regex, r"# \g<0>", content)
-            with config_file.open("w", encoding="utf-8") as file:
-                file.write(sanitized)
-
     async def git_catfile(self, hash, option="-t", folder=Path()):
         command = ["git", "cat-file", option, hash]
         try:
@@ -270,8 +258,10 @@ class gitdumper(BaseModule):
         return output.stdout
 
     async def git_checkout(self, folder):
+        self.helpers.sanitize_git_repo(folder)
         self.verbose(f"Running git checkout to reconstruct the git repository at {folder}")
-        command = ["git", "checkout", "."]
+        # we do "checkout head -- ." because the sanitization deletes the index file, and it needs to be reconstructed
+        command = ["git", "checkout", "HEAD", "--", "."]
         try:
             await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
         except CalledProcessError as e:

bbot/modules/graphql_introspection.py

@@ -119,7 +119,10 @@ fragment TypeRef on __Type {
             }
             response = await self.helpers.request(**request_args)
             if not response or response.status_code != 200:
-                self.debug(f"Failed to get GraphQL schema for {url} (status code {response.status_code})")
+                self.debug(
+                    f"Failed to get GraphQL schema for {url} "
+                    f"{f'(status code {response.status_code})' if response else ''}"
+                )
                 continue
             try:
                 response_json = response.json()

bbot/modules/internal/unarchive.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from contextlib import suppress
 from bbot.modules.internal.base import BaseInternalModule
 from bbot.core.helpers.libmagic import get_magic_info, get_compression
 
@@ -62,15 +63,20 @@ class unarchive(BaseInternalModule):
                 context=f'extracted "{path}" to: {output_dir}',
             )
         else:
-            output_dir.rmdir()
+            with suppress(OSError):
+                output_dir.rmdir()
 
     async def extract_file(self, path, output_dir):
         extension, mime_type, description, confidence = get_magic_info(path)
         compression_format = get_compression(mime_type)
         cmd_list = self.compression_methods.get(compression_format, [])
         if cmd_list:
-            if not output_dir.exists():
-                self.helpers.mkdir(output_dir)
+            # output dir must not already exist
+            try:
+                output_dir.mkdir(exist_ok=False)
+            except FileExistsError:
+                self.warning(f"Destination directory {output_dir} already exists, aborting unarchive for {path}")
+                return False
             command = [s.format(filename=path, extract_dir=output_dir) for s in cmd_list]
             try:
                 await self.run_process(command, check=True)

{bbot-2.7.0.7116rc0.dist-info → bbot-2.7.1.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: bbot
-Version: 2.7.0.7116rc0
+Version: 2.7.1
 Summary: OSINT automation for hackers.
 License: GPL-3.0
+License-File: LICENSE
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool
 Author: TheTechromancer
 Requires-Python: >=3.9,<4.0
@@ -14,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Security
 Requires-Dist: ansible-core (>=2.15.13,<3.0.0)
 Requires-Dist: ansible-runner (>=2.3.2,<3.0.0)
@@ -21,7 +23,7 @@ Requires-Dist: beautifulsoup4 (>=4.12.2,<5.0.0)
 Requires-Dist: cachetools (>=5.3.2,<7.0.0)
 Requires-Dist: cloudcheck (>=7.2.11,<8.0.0)
 Requires-Dist: deepdiff (>=8.0.0,<9.0.0)
-Requires-Dist: dnspython (>=2.4.2,<3.0.0)
+Requires-Dist: dnspython (>=2.7.0,<2.8.0)
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: idna (>=3.4,<4.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)

{bbot-2.7.0.7116rc0.dist-info → bbot-2.7.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=hCaX7J67VgSZ4CbKAR8ugvcyo4dTahWLYP_RZXyjtvw,163
+bbot/__init__.py,sha256=0He7I9F9dBrSF3yjP1vDZiz713pG2Dc70VjA71aOxzs,156
 bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -26,10 +26,11 @@ bbot/core/helpers/dns/engine.py,sha256=0VP53V8QDOlNZgZk2LPGetDQCuJxqFtAsWWw-jBqw
 bbot/core/helpers/dns/helpers.py,sha256=aQroIuz5TxrCZ4zoplOaqLj3ZNgOgDRKn0xM8GKz2dA,8505
 bbot/core/helpers/dns/mock.py,sha256=FCPrihu6O4kun38IH70RfktsXIKKfe0Qx5PMzZVUdsY,2588
 bbot/core/helpers/files.py,sha256=vWxx5AfH8khboawBdUi-KYvbjpybSMLGZpixylitGMQ,5811
+bbot/core/helpers/git.py,sha256=q2y25H9wow1-R7TMT4BBSVFzJpBzfGblAMxy9hGOCvw,757
 bbot/core/helpers/helper.py,sha256=u-q_Ka9pY1atvC-FChxYpURM7b3_0gaCNIHSG__Wi74,8538
 bbot/core/helpers/interactsh.py,sha256=VBYYH6-rWBofRsgemndK6iZNmyifOps8vgQOw2mac4k,12624
 bbot/core/helpers/libmagic.py,sha256=QMHyxjgDLb2jyjBvK1MQ-xt6WkGXhKcHu9ZP1li-sik,3460
-bbot/core/helpers/misc.py,sha256=kQzxGBvD87nyEIXiT8JjIpPh5KKC_rKyHrOVoPG14cw,89035
+bbot/core/helpers/misc.py,sha256=9O0y76FZi4E8Frbip086Rip3dUS52y_qf03iwcv-5aM,89067
 bbot/core/helpers/names_generator.py,sha256=zmo4MyuOnAYjiUDiORhq9T9bHmA_gW72Y2kHMAqVENU,10594
 bbot/core/helpers/ntlm.py,sha256=BspNjNyKiWEqdqG3gYzYFyLsnCuzWyLOsguv1yBWLs0,2516
 bbot/core/helpers/process.py,sha256=00uRpLMFi3Pt3uT8qXwAIhsXdoa7h-ifoXh0sGYgwqs,1702
@@ -104,8 +105,8 @@ bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,
 bbot/modules/fullhunt.py,sha256=2ntu1yBh51N4e_l-kpXc1UBoVVcxEE2JPkyaMYCuUb4,1336
 bbot/modules/generic_ssrf.py,sha256=KFdcHpUV9-Z7oN7emzbirimsNc2xZ_1IFqnsfIkEbcM,9196
 bbot/modules/git.py,sha256=zmHeI0bn181T1P8C55HSebkdVGLTpzGxPc-LRqiHrbc,1723
-bbot/modules/git_clone.py,sha256=XIPPY2qubN-UxzFXNjEpA2tDObJKrNy24PvqAb22cD8,3449
-bbot/modules/gitdumper.py,sha256=giwXMPlmgC1gxyZKTGlF-0xplt4MBbDiXi07-ghH02o,12039
+bbot/modules/git_clone.py,sha256=SwtCnOpVqEgSMfqaN54NUpS2jYZWt4Fk8Y_TqUIO724,3764
+bbot/modules/gitdumper.py,sha256=mzlEJuWLlZIWXj-0V5kC8qTVLEvVtbrPColCXQGFEoQ,11588
 bbot/modules/github_codesearch.py,sha256=a-r2vE9N9WyBpFUiKCsg0TK4Qn7DaEGyVRTUKzkDLWA,3641
 bbot/modules/github_org.py,sha256=WM18vJCHuOHJJ5rPzQzQ3Pmp7XPPuaMeVgNfW-FlO0k,8938
 bbot/modules/github_usersearch.py,sha256=G8knkQBJsn7EKcMhcEaFPiB_Y5S96e2VaseBubsqOyk,3407
@@ -113,7 +114,7 @@ bbot/modules/github_workflows.py,sha256=xKntAFDeGuE4MqbEmhJyYXKbzoSh9tWYlHNlnF37
 bbot/modules/gitlab.py,sha256=9oWWpBijeHCjuFBfWW4HvNqt7bvJvrBgBjaaz_UPPnE,5964
 bbot/modules/google_playstore.py,sha256=N4QjzQag_bgDXfX17rytBiiWA-SQtYI2N0J_ZNEOdv0,3701
 bbot/modules/gowitness.py,sha256=hMhCz4O1sDJCzCzRIcmu0uNDgDDf9JzkFBwL1WuUum0,13144
-bbot/modules/graphql_introspection.py,sha256=xcrdW9lUJhxgIxJyZxfFQSk9vQxyBpvQmMoKWhRLdUA,4152
+bbot/modules/graphql_introspection.py,sha256=Y-MqXrN6qmXTv2T6t7hJ-SU3R2guZQRWkrrCLC56bAc,4239
 bbot/modules/hackertarget.py,sha256=IsKs9PtxUHdLJKZydlRdW_loBE2KphQYi3lKDAd4odc,1029
 bbot/modules/host_header.py,sha256=uDjwidMdeNPMRfzQ2YW4REEGsZqnGOZHbOS6GgdNd9s,7686
 bbot/modules/httpx.py,sha256=tlQ6NKw8FJ6rGaNI1BnwKqjxZFn1MZeItGZgNab_Ydo,8177
@@ -127,7 +128,7 @@ bbot/modules/internal/cloudcheck.py,sha256=IYVetq8YE--yio5DhxukjgshJwZ3EohIu6ZVw
 bbot/modules/internal/dnsresolve.py,sha256=1fwWChIGpSEIIkswueiIhEwIahQ7YngZ-njFK-RIsfU,15679
 bbot/modules/internal/excavate.py,sha256=L8tGdfdvxrvfskC1Ms9UtSy-gxudnQcW7Iv5tHNAbW4,63728
 bbot/modules/internal/speculate.py,sha256=ua35Da-f0-fnK0oXtx4DeGJAT19bfqnmLfetSUfJnIk,9262
-bbot/modules/internal/unarchive.py,sha256=sA6KYQnhkyHq0mHwhRESHy9wkaRE43PjPkShWW0mOvM,3763
+bbot/modules/internal/unarchive.py,sha256=tORk083jgbJAHdNLSLHlR1YtP0TrBBrTPWF67bnW1wk,4041
 bbot/modules/ip2location.py,sha256=yGivX9fzvwvLpnqmYCP2a8SPjTarzrZxfRluog-nkME,2628
 bbot/modules/ipneighbor.py,sha256=b_0IhorihFLtXJZEz57EGXjXW30gIOEzzVgz2GFvM3A,1591
 bbot/modules/ipstack.py,sha256=j_S8WMNqQuSQgBT7AX4tO70fgbWuRYrpsS3tVsu_hn4,2200
@@ -463,8 +464,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.7.0.7116rc0.dist-info/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.7.0.7116rc0.dist-info/METADATA,sha256=jmNWdCkwN6WmatECa30xRK1jP_vq0LwOZjo6hMsefQA,18347
-bbot-2.7.0.7116rc0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-bbot-2.7.0.7116rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.7.0.7116rc0.dist-info/RECORD,,
+bbot-2.7.1.dist-info/METADATA,sha256=Tz9f2regAqwJ-kacWp_m4st1veVZr1VWa9QB1FfMh7Q,18412
+bbot-2.7.1.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+bbot-2.7.1.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.7.1.dist-info/licenses/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.7.1.dist-info/RECORD,,

{bbot-2.7.0.7116rc0.dist-info → bbot-2.7.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.0
 Root-Is-Purelib: true
 Tag: py3-none-any