bbot 2.7.1.7212rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.7.1.7212rc"
+__version__ = "v2.7.2.7254rc"
 
 from .scanner import Scanner, Preset
 

bbot/core/flags.py

@@ -6,6 +6,7 @@ flag_descriptions = {
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/modules/apkpure.py

@@ -6,7 +6,7 @@ from bbot.modules.base import BaseModule
 class apkpure(BaseModule):
     watched_events = ["MOBILE_APP"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download android applications from apkpure.com",
         "created_date": "2024-10-11",

bbot/modules/docker_pull.py

@@ -8,7 +8,7 @@ from bbot.modules.base import BaseModule
 class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",

bbot/modules/filedownload.py

@@ -14,7 +14,7 @@ class filedownload(BaseModule):
 
     watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"]
     produced_events = ["FILESYSTEM"]
-    flags = ["active", "safe", "web-basic"]
+    flags = ["active", "safe", "web-basic", "download"]
     meta = {
         "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
         "created_date": "2023-10-11",

bbot/modules/git_clone.py

@@ -6,7 +6,7 @@ from bbot.modules.templates.github import github
 class git_clone(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Clone code github repositories",
         "created_date": "2024-03-08",

bbot/modules/gitdumper.py

@@ -7,7 +7,7 @@ from bbot.modules.base import BaseModule
 class gitdumper(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download a leaked .git folder recursively or by fuzzing common names",
         "created_date": "2025-02-11",

bbot/modules/github_workflows.py

@@ -8,7 +8,7 @@ from bbot.modules.templates.github import github
 class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download a github repositories workflow logs and workflow artifacts",
         "created_date": "2024-04-29",

bbot/modules/gitlab_com.py

@@ -0,0 +1,31 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_com(GitLabBaseModule):
+    watched_events = ["SOCIAL"]
+    produced_events = [
+        "CODE_REPOSITORY",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Enumerate GitLab SaaS (gitlab.com/org) for projects and groups",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"}
+
+    # This is needed because we are consuming SOCIAL events, which aren't in scope
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        await self.handle_social(event)
+
+    async def filter_event(self, event):
+        if event.data["platform"] != "gitlab":
+            return False, "platform is not gitlab"
+        _, domain = self.helpers.split_domain(event.host)
+        if domain not in self.saas_domains:
+            return False, "gitlab instance is not gitlab.com/org"
+        return True

bbot/modules/gitlab_onprem.py

@@ -0,0 +1,84 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_onprem(GitLabBaseModule):
+    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
+    produced_events = [
+        "TECHNOLOGY",
+        "SOCIAL",
+        "CODE_REPOSITORY",
+        "FINDING",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Detect self-hosted GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    # Optional GitLab access token (only required for gitlab.com, but still
+    # supported for on-prem installations that expose private projects).
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"}
+
+    # Allow accepting events slightly beyond configured max distance so we can
+    # discover repos on neighbouring infrastructure.
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        if event.type == "HTTP_RESPONSE":
+            await self.handle_http_response(event)
+        elif event.type == "TECHNOLOGY":
+            await self.handle_technology(event)
+        elif event.type == "SOCIAL":
+            await self.handle_social(event)
+
+    async def filter_event(self, event):
+        # only accept out-of-scope SOCIAL events
+        if event.type == "HTTP_RESPONSE":
+            if event.scope_distance > self.scan.scope_search_distance:
+                return False, "event is out of scope distance"
+        elif event.type == "TECHNOLOGY":
+            if not event.data["technology"].lower().startswith("gitlab"):
+                return False, "technology is not gitlab"
+            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
+                return False, "gitlab instance is not self-hosted"
+        elif event.type == "SOCIAL":
+            if event.data["platform"] != "gitlab":
+                return False, "platform is not gitlab"
+            _, domain = self.helpers.split_domain(event.host)
+            if domain in self.saas_domains:
+                return False, "gitlab instance is not self-hosted"
+        return True
+
+    async def handle_http_response(self, event):
+        """Identify GitLab servers from HTTP responses."""
+        headers = event.data.get("header", {})
+        if "x_gitlab_meta" in headers:
+            url = event.parsed_url._replace(path="/").geturl()
+            await self.emit_event(
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
+            )
+            description = f"GitLab server at {event.host}"
+            await self.emit_event(
+                {"host": str(event.host), "description": description},
+                "FINDING",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
+            )
+
+    async def handle_technology(self, event):
+        """Enumerate projects & groups once we know a host is GitLab."""
+        base_url = self.get_base_url(event)
+
+        # Projects owned by the authenticated user (or public projects if no
+        # authentication).
+        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
+        await self.handle_projects_url(projects_url, event)
+
+        # Group enumeration.
+        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
+        await self.handle_groups_url(groups_url, event)

bbot/modules/postman_download.py

@@ -7,7 +7,7 @@ from bbot.modules.templates.postman import postman
 class postman_download(postman):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum", "download"]
     meta = {
         "description": "Download workspaces, collections, requests from Postman",
         "created_date": "2024-09-07",

bbot/modules/templates/gitlab.py

@@ -0,0 +1,98 @@
+from bbot.modules.base import BaseModule
+
+
+class GitLabBaseModule(BaseModule):
+    """Common functionality for interacting with GitLab instances.
+
+    This template is intended to be inherited by two concrete modules:
+    1. ``gitlab_com``   – Handles public SaaS instances (gitlab.com / gitlab.org).
+    2. ``gitlab_onprem`` – Handles self-hosted, on-premises GitLab servers.
+
+    Both child modules share identical behaviour when talking to the GitLab
+    REST API; they only differ in which events they are willing to accept.
+    """
+
+    # domains owned by GitLab
+    saas_domains = ["gitlab.com", "gitlab.org"]
+
+    async def setup(self):
+        if self.options.get("api_key") is not None:
+            await self.require_api_key()
+        return True
+
+    async def handle_social(self, event):
+        """Enumerate projects belonging to a user or group profile."""
+        username = event.data.get("profile_name", "")
+        if not username:
+            return
+        base_url = self.get_base_url(event)
+        urls = [
+            # User-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
+            # Group-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
+        ]
+        for url in urls:
+            await self.handle_projects_url(url, event)
+
+    async def handle_projects_url(self, projects_url, event):
+        for project in await self.gitlab_json_request(projects_url):
+            project_url = project.get("web_url", "")
+            if project_url:
+                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
+                await self.emit_event(
+                    code_event,
+                    context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}",
+                )
+            namespace = project.get("namespace", {})
+            if namespace:
+                await self.handle_namespace(namespace, event)
+
+    async def handle_groups_url(self, groups_url, event):
+        for group in await self.gitlab_json_request(groups_url):
+            await self.handle_namespace(group, event)
+
+    async def gitlab_json_request(self, url):
+        """Helper that performs an HTTP request and safely returns JSON list."""
+        response = await self.api_request(url)
+        if response is not None:
+            try:
+                json_data = response.json()
+            except Exception:
+                return []
+            if json_data and isinstance(json_data, list):
+                return json_data
+        return []
+
+    async def handle_namespace(self, namespace, event):
+        namespace_name = namespace.get("path", "")
+        namespace_url = namespace.get("web_url", "")
+        namespace_path = namespace.get("full_path", "")
+
+        if not (namespace_name and namespace_url and namespace_path):
+            return
+
+        namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
+
+        social_event = self.make_event(
+            {
+                "platform": "gitlab",
+                "profile_name": namespace_path,
+                "url": namespace_url,
+            },
+            "SOCIAL",
+            parent=event,
+        )
+        await self.emit_event(
+            social_event,
+            context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
+        )
+
+    # ------------------------------------------------------------------
+    # Utility helpers
+    # ------------------------------------------------------------------
+    def get_base_url(self, event):
+        base_url = event.data.get("url", "")
+        if not base_url:
+            base_url = f"https://{event.host}"
+        return self.helpers.urlparse(base_url)._replace(path="/").geturl()

bbot/test/test_step_2/module_tests/test_module_gitlab_com.py

@@ -0,0 +1,66 @@
+from .base import ModuleTestBase
+
+
+class TestGitlab_Com(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["gitlab_com", "httpx", "social", "excavate"]
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
+        module_test.httpx_mock.add_response(
+            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
+            json=[
+                {
+                    "id": 55490429,
+                    "description": None,
+                    "name": "Veilid",
+                    "name_with_namespace": "Veilid / Veilid",
+                    "path": "veilid",
+                    "path_with_namespace": "veilidgroup/veilid",
+                    "created_at": "2024-03-03T05:22:53.169Z",
+                    "default_branch": "master",
+                    "tag_list": [],
+                    "topics": [],
+                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
+                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
+                    "web_url": "https://gitlab.org/veilidgroup/veilid",
+                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
+                    "forks_count": 0,
+                    "avatar_url": None,
+                    "star_count": 0,
+                    "last_activity_at": "2024-03-03T05:22:53.097Z",
+                    "namespace": {
+                        "id": 66882294,
+                        "name": "veilidgroup",
+                        "path": "veilidgroup",
+                        "kind": "group",
+                        "full_path": "veilidgroup",
+                        "parent_id": None,
+                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
+                        "web_url": "https://gitlab.org/groups/veilidgroup",
+                    },
+                },
+            ],
+        )
+
+    def check(self, module_test, events):
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "gitlab"
+                and e.data["profile_name"] == "veilidgroup"
+                and e.data["url"] == "https://gitlab.org/veilidgroup"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
+                and str(e.module) == "gitlab_com"
+            ]
+        )

bbot/test/test_step_2/module_tests/{test_module_gitlab.py → test_module_gitlab_onprem.py}

@@ -1,10 +1,10 @@
 from .base import ModuleTestBase
 
 
-class TestGitlab(ModuleTestBase):
+class TestGitlab_OnPrem(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx"]
-    config_overrides = {"modules": {"gitlab": {"api_key": "asdf"}}}
+    modules_overrides = ["gitlab_onprem", "httpx"]
+    config_overrides = {"modules": {"gitlab_onprem": {"api_key": "asdf"}}}
 
     async def setup_before_prep(self, module_test):
         module_test.httpserver.expect_request("/").respond_with_data(headers={"X-Gitlab-Meta": "asdf"})
@@ -179,7 +179,7 @@ class TestGitlab(ModuleTestBase):
                 and e.data["platform"] == "gitlab"
                 and e.data["profile_name"] == "bbotgroup"
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup"
-                and str(e.module) == "gitlab"
+                and str(e.module) == "gitlab_onprem"
             ]
         )
         assert 1 == len(
@@ -209,68 +209,3 @@ class TestGitlab(ModuleTestBase):
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup/bbot3"
             ]
         )
-
-
-class TestGitlabDotOrg(ModuleTestBase):
-    targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx", "social", "excavate"]
-
-    async def setup_before_prep(self, module_test):
-        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
-        module_test.httpx_mock.add_response(
-            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
-            json=[
-                {
-                    "id": 55490429,
-                    "description": None,
-                    "name": "Veilid",
-                    "name_with_namespace": "Veilid / Veilid",
-                    "path": "veilid",
-                    "path_with_namespace": "veilidgroup/veilid",
-                    "created_at": "2024-03-03T05:22:53.169Z",
-                    "default_branch": "master",
-                    "tag_list": [],
-                    "topics": [],
-                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
-                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
-                    "web_url": "https://gitlab.org/veilidgroup/veilid",
-                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
-                    "forks_count": 0,
-                    "avatar_url": None,
-                    "star_count": 0,
-                    "last_activity_at": "2024-03-03T05:22:53.097Z",
-                    "namespace": {
-                        "id": 66882294,
-                        "name": "veilidgroup",
-                        "path": "veilidgroup",
-                        "kind": "group",
-                        "full_path": "veilidgroup",
-                        "parent_id": None,
-                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
-                        "web_url": "https://gitlab.org/groups/veilidgroup",
-                    },
-                },
-            ],
-        )
-
-    def check(self, module_test, events):
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "SOCIAL"
-                and e.data["platform"] == "gitlab"
-                and e.data["profile_name"] == "veilidgroup"
-                and e.data["url"] == "https://gitlab.org/veilidgroup"
-            ]
-        )
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "CODE_REPOSITORY"
-                and "git" in e.tags
-                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
-                and str(e.module) == "gitlab"
-            ]
-        )

{bbot-2.7.1.7212rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bbot
-Version: 2.7.1.7212rc0
+Version: 2.7.2.7254rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 License-File: LICENSE

{bbot-2.7.1.7212rc0.dist-info → bbot-2.7.2.7254rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bbot/__init__.py,sha256=VSc_ku85NcucmahEOo7gG1JKRDfMFITM_ihdRv4OvPs,163
+bbot/__init__.py,sha256=EqXFKA9XrSnBr9t8uzsLmLvzeOMSNL0LWOUMh9RieY4,163
 bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
@@ -9,7 +9,7 @@ bbot/core/engine.py,sha256=DxlbxUWU1x20DTIsVsYXWuR5Z8eYJRmP-SOLyvO4Eek,29362
 bbot/core/event/__init__.py,sha256=pRi5lC9YBKGxx6ZgrnE4shqKYYdqKR1Ps6tDw2WKGOw,113
 bbot/core/event/base.py,sha256=1jUgd3I3TDITKoobh92ir_tIm38EN1ZbhoaX1W9fKts,67125
 bbot/core/event/helpers.py,sha256=MohOCVBjkn_K1p4Ipgx-MKliZtV6l4NJPq3YgagkvSM,6507
-bbot/core/flags.py,sha256=Ltvm8Bc4D65I55HuU5bzyjO1R3yMDNpVmreGU83ZBXE,1266
+bbot/core/flags.py,sha256=3bT3BYActmxomZLusjPROzrLhVlRJKLOCluZBPVauEg,1336
 bbot/core/helpers/__init__.py,sha256=cpOGLKIgA3vdHYqsOtx63BFO_qbtwCmez2amFPu6YTs,111
 bbot/core/helpers/async_helpers.py,sha256=bVHEUIOZo8iCmuovLYb3oNLPdLFUoEyc6wZIIvtELVs,4399
 bbot/core/helpers/bloom.py,sha256=fTMdgnrJPC9fzNXwBg1GYbz1P05YPhjxy9aUg6-z3a4,2605
@@ -57,7 +57,7 @@ bbot/logger.py,sha256=wE-532v5FyKuSSoTdyW1xSfaOnLZB1axAJnB-uW2xrI,2745
 bbot/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bbot/modules/ajaxpro.py,sha256=daE1yQoCsSI5c4dh3YKwRSggTISNjWgrK7qTPidk7cU,3764
 bbot/modules/anubisdb.py,sha256=JCy2YCfa0e_VawpzNmcPXAosKUthmYGutireJ0gMDws,1916
-bbot/modules/apkpure.py,sha256=h26zh-nv1G9IvvTwywbFcARiZPfHUhjIUGVl0Achbek,2555
+bbot/modules/apkpure.py,sha256=q15cOGsj7AfRnOZQAahJefnD1YGzsYWqkFz0RXs3RPk,2567
 bbot/modules/aspnet_bin_exposure.py,sha256=X16mK8tff7-LOcbS4fUschkDRdVVz9r2N8Ta2EZzw2Y,3737
 bbot/modules/azure_realm.py,sha256=pP2PUlLy0K9KKaE8aNcznWjDW3PKHvnMejdOSc-o4ms,1612
 bbot/modules/azure_tenant.py,sha256=qBn7CUA_hth2PqW55XZVjYxIw20xLYrMntXc6mYpmKU,5366
@@ -92,25 +92,26 @@ bbot/modules/dnscaa.py,sha256=pyaLqHrdsVhqtd1JBZVjKKcuYT_ywUbFYkrnfXcGD5s,5014
 bbot/modules/dnscommonsrv.py,sha256=wrCRTlqVuxFIScWH0Cb0UQAVk0TWxgVc5fo5awl3R24,1568
 bbot/modules/dnsdumpster.py,sha256=x4_1ZcPRAKDiCWMt7x4bbfcar2-VN6fLjWx0ijPUEmY,2775
 bbot/modules/dnstlsrpt.py,sha256=v8V72RBsawmDPrMrTcKXEyoFt9bgbfm-cpoPYgKEKLQ,6238
-bbot/modules/docker_pull.py,sha256=zNQcQdS-JWM2-TbQ_iyjeGA9CKcpuXdeO5ucoJgzZNY,9189
+bbot/modules/docker_pull.py,sha256=G1B4mjZBFIfxlbmi_Q02-Y9N53moRKpjQ7Li_aU_MYM,9201
 bbot/modules/dockerhub.py,sha256=JQkujjqvQRzQuvHjQ7JbFs_VlJj8dLRPRObAkBgUQhc,3493
 bbot/modules/dotnetnuke.py,sha256=zipcHyNYr2FEecStb1Yrm938ps01RvHV8NnyqAvnGGc,10537
 bbot/modules/emailformat.py,sha256=Koi2aSng-FSRJVhpbFaclrqZxo4lQoPMcUMn_qXTfVE,1518
 bbot/modules/extractous.py,sha256=VSGKmHPAA_4r62jaN8Yqi3JcjehjxpI2lhe8i2j786s,4648
 bbot/modules/ffuf.py,sha256=94TJ5xvqKwH0JaWmC_t1dLTpRsO8HEy4lnbsu8LF_HY,14965
 bbot/modules/ffuf_shortnames.py,sha256=y5vnypLPN-KrjpmoG5zlqcX8VwfcLBpNg1yQI7bP9Hg,18737
-bbot/modules/filedownload.py,sha256=5MctNWSYyjoXPshRXbltsn92KDAr9fsLqbPGP4eK7Es,8903
+bbot/modules/filedownload.py,sha256=ri5mUNm-kTXE3ZsamsA_5jE29SMrGcdW5MO7doh8vsM,8915
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=2ntu1yBh51N4e_l-kpXc1UBoVVcxEE2JPkyaMYCuUb4,1336
 bbot/modules/generic_ssrf.py,sha256=KFdcHpUV9-Z7oN7emzbirimsNc2xZ_1IFqnsfIkEbcM,9196
 bbot/modules/git.py,sha256=zmHeI0bn181T1P8C55HSebkdVGLTpzGxPc-LRqiHrbc,1723
-bbot/modules/git_clone.py,sha256=SwtCnOpVqEgSMfqaN54NUpS2jYZWt4Fk8Y_TqUIO724,3764
-bbot/modules/gitdumper.py,sha256=mzlEJuWLlZIWXj-0V5kC8qTVLEvVtbrPColCXQGFEoQ,11588
+bbot/modules/git_clone.py,sha256=DzYQ5p_ZmezxSXKamjcxTVB0RNIywehs_UvtnFFVW2g,3776
+bbot/modules/gitdumper.py,sha256=F1qpErD3dv4qvWODgLi8MWMENfJxrl12OAVSbusiF34,11600
 bbot/modules/github_codesearch.py,sha256=a-r2vE9N9WyBpFUiKCsg0TK4Qn7DaEGyVRTUKzkDLWA,3641
 bbot/modules/github_org.py,sha256=WM18vJCHuOHJJ5rPzQzQ3Pmp7XPPuaMeVgNfW-FlO0k,8938
 bbot/modules/github_usersearch.py,sha256=G8knkQBJsn7EKcMhcEaFPiB_Y5S96e2VaseBubsqOyk,3407
-bbot/modules/github_workflows.py,sha256=xKntAFDeGuE4MqbEmhJyYXKbzoSh9tWYlHNlnF37PYA,10040
-bbot/modules/gitlab.py,sha256=9oWWpBijeHCjuFBfWW4HvNqt7bvJvrBgBjaaz_UPPnE,5964
+bbot/modules/github_workflows.py,sha256=cj9YLoW01v5Iey6s89zShlrAF80TEBhFcEWyrvdNe38,10052
+bbot/modules/gitlab_com.py,sha256=WBNGw4ec-xd_Iz8yxJcxEgTOpsBPxfn5pDU1DtONFgs,1051
+bbot/modules/gitlab_onprem.py,sha256=OwbYeldAUCQvFiYAIikX1-waHii1F0cMPLAtqc4pyHs,3622
 bbot/modules/google_playstore.py,sha256=N4QjzQag_bgDXfX17rytBiiWA-SQtYI2N0J_ZNEOdv0,3701
 bbot/modules/gowitness.py,sha256=hMhCz4O1sDJCzCzRIcmu0uNDgDDf9JzkFBwL1WuUum0,13144
 bbot/modules/graphql_introspection.py,sha256=Y-MqXrN6qmXTv2T6t7hJ-SU3R2guZQRWkrrCLC56bAc,4239
@@ -181,7 +182,7 @@ bbot/modules/pgp.py,sha256=Xu2M9WEIlwTm5-Lv29g7BblI05tD9Dl0XsYSeY6UURs,2065
 bbot/modules/portfilter.py,sha256=3iu4xqCsHafhVMbA32Mw6K_7Yn576Rz6GxXMevZQEpM,1752
 bbot/modules/portscan.py,sha256=emhNhnFYBVMnVm7IZKmzHJRCKRwVpF3S9dtOV6H9iYA,13091
 bbot/modules/postman.py,sha256=vo761Nzu3kPBzfCY3KJcvsGEsjImaa7iA2z-LyASBDc,4589
-bbot/modules/postman_download.py,sha256=OXC4hHInwD2Ps-BURc2uM_Z7PCl95KPdcxSI34Vzdcc,3604
+bbot/modules/postman_download.py,sha256=nR7Zpy8DsF1xwPskWysr4i9Bhn0JQq9E__A-UcjjDM8,3616
 bbot/modules/rapiddns.py,sha256=uONESr0B5pv9cSAr7lF4WWV31APUhXyHexvI04rUcyk,787
 bbot/modules/reflected_parameters.py,sha256=RjS-4C-XC9U-jC9J7AYNqwn6I-O2y3LvTRhB68dpgKI,3281
 bbot/modules/report/affiliates.py,sha256=vvus8LylqOfP-lfGid0z4FS6MwOpNuRTcSJ9aSnybp4,1713
@@ -203,6 +204,7 @@ bbot/modules/subdomainradar.py,sha256=YlRNMtNGLpa13KZ7aksAMVZdSjxe1tkywU5RXlwXpP
 bbot/modules/telerik.py,sha256=kWi498zihl02gHaS7AvyAxlEAZvmfKgKMSTAG8CS62A,19108
 bbot/modules/templates/bucket.py,sha256=muLPpfAGtcNhL0tLU-qHTlTNIz4yncRcVjdZMqVRtUI,7153
 bbot/modules/templates/github.py,sha256=lrV1EYPqjtPkJsS0fQfqmLvGchNo_fO3A75W9-03gxY,2531
+bbot/modules/templates/gitlab.py,sha256=XOwCaYO77ISbVPnjzws2M1klueTnJbXRef-ZsHUtwvA,3895
 bbot/modules/templates/postman.py,sha256=MIpz2q_r6LP0kIEgByo7oX5qHhMZLOhr7oKzJI9Beec,6959
 bbot/modules/templates/shodan.py,sha256=MXBvlmfw3jZFqT47v10UkqMSyQR-zBIxMJmK7PWw6uw,1174
 bbot/modules/templates/sql.py,sha256=o-CdyyoJvHJdJBKkj3CIGXYxUta4w2AB_2Vr-k7cDDU,3553
@@ -369,7 +371,8 @@ bbot/test/test_step_2/module_tests/test_module_github_codesearch.py,sha256=M50xB
 bbot/test/test_step_2/module_tests/test_module_github_org.py,sha256=5tKO6NH4TPBeIdeTf7Bz9PUZ1pcvKsjrG0nFhc3YgT0,25458
 bbot/test/test_step_2/module_tests/test_module_github_usersearch.py,sha256=IIQ0tYZjQN8_L8u_N4m8Nz3kbB4IyBp95tYCPcQeScg,5264
 bbot/test/test_step_2/module_tests/test_module_github_workflows.py,sha256=o_teEaskm3H22QEKod5KJayFvvcgOQoG4eItGWv8C8E,38006
-bbot/test/test_step_2/module_tests/test_module_gitlab.py,sha256=fnwE7BWTU6EQquKdGLCiaX_LwVwvzOLev3Y9GheTLSY,11859
+bbot/test/test_step_2/module_tests/test_module_gitlab_com.py,sha256=fGnjYyMvMZE2hu0Fms9H8rMnPPN6_uynDDDEmcVE9-8,2753
+bbot/test/test_step_2/module_tests/test_module_gitlab_onprem.py,sha256=Soo72Ppt5hYWVUIxMYGnBGPL47EnVDPbTsEHUziKimg,9173
 bbot/test/test_step_2/module_tests/test_module_google_playstore.py,sha256=uTRqpAGI9HI-rOk_6jdV44OoSqi0QQQ3aTVzvuV0dtc,3034
 bbot/test/test_step_2/module_tests/test_module_gowitness.py,sha256=8kSeBowX4eejMW791mIaFqP9SDn1l2EDRJatvmZVWug,6500
 bbot/test/test_step_2/module_tests/test_module_graphql_introspection.py,sha256=qac8DJ_exe6Ra4UgRvVMSdgBhLIZP9lmXyKhi9RPOK8,1241
@@ -460,8 +463,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.7.1.7212rc0.dist-info/METADATA,sha256=7rDrwHdnOGfp7sZS3vDHbnlFYUnX_d7gwDpya7NkQY4,18420
-bbot-2.7.1.7212rc0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-bbot-2.7.1.7212rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.7.1.7212rc0.dist-info/licenses/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.7.1.7212rc0.dist-info/RECORD,,
+bbot-2.7.2.7254rc0.dist-info/METADATA,sha256=j16BPGJz_5QPp264U8u4wYIhVBbbx9gsdcRpODo0nFo,18420
+bbot-2.7.2.7254rc0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+bbot-2.7.2.7254rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.7.2.7254rc0.dist-info/licenses/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.7.2.7254rc0.dist-info/RECORD,,

bbot/modules/gitlab.py

@@ -1,141 +0,0 @@
-from bbot.modules.base import BaseModule
-
-
-class gitlab(BaseModule):
-    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
-    produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"]
-    flags = ["active", "safe", "code-enum"]
-    meta = {
-        "description": "Detect GitLab instances and query them for repositories",
-        "created_date": "2024-03-11",
-        "author": "@TheTechromancer",
-    }
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Gitlab access token"}
-
-    scope_distance_modifier = 2
-
-    async def setup(self):
-        await self.require_api_key()
-        return True
-
-    async def filter_event(self, event):
-        # only accept out-of-scope SOCIAL events
-        if event.type == "HTTP_RESPONSE":
-            if event.scope_distance > self.scan.scope_search_distance:
-                return False, "event is out of scope distance"
-        elif event.type == "TECHNOLOGY":
-            if not event.data["technology"].lower().startswith("gitlab"):
-                return False, "technology is not gitlab"
-            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
-                return False, "gitlab instance is not self-hosted"
-        elif event.type == "SOCIAL":
-            if event.data["platform"] != "gitlab":
-                return False, "platform is not gitlab"
-        return True
-
-    async def handle_event(self, event):
-        if event.type == "HTTP_RESPONSE":
-            await self.handle_http_response(event)
-        elif event.type == "TECHNOLOGY":
-            await self.handle_technology(event)
-        elif event.type == "SOCIAL":
-            await self.handle_social(event)
-
-    async def handle_http_response(self, event):
-        # identify gitlab instances from HTTP responses
-        # HTTP_RESPONSE --> TECHNOLOGY
-        # HTTP_RESPONSE --> FINDING
-        headers = event.data.get("header", {})
-        if "x_gitlab_meta" in headers:
-            url = event.parsed_url._replace(path="/").geturl()
-            await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url},
-                "TECHNOLOGY",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
-            )
-            description = f"GitLab server at {event.host}"
-            await self.emit_event(
-                {"host": str(event.host), "description": description},
-                "FINDING",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: {description}",
-            )
-
-    async def handle_technology(self, event):
-        # retrieve gitlab groups from gitlab instances
-        # TECHNOLOGY --> SOCIAL
-        # TECHNOLOGY --> URL
-        # TECHNOLOGY --> CODE_REPOSITORY
-        base_url = self.get_base_url(event)
-        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
-        await self.handle_projects_url(projects_url, event)
-        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
-        await self.handle_groups_url(groups_url, event)
-
-    async def handle_social(self, event):
-        # retrieve repositories from gitlab user
-        # SOCIAL --> CODE_REPOSITORY
-        # SOCIAL --> SOCIAL
-        username = event.data.get("profile_name", "")
-        if not username:
-            return
-        base_url = self.get_base_url(event)
-        urls = [
-            # group
-            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
-            # user
-            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
-        ]
-        for url in urls:
-            await self.handle_projects_url(url, event)
-
-    async def handle_projects_url(self, projects_url, event):
-        for project in await self.gitlab_json_request(projects_url):
-            project_url = project.get("web_url", "")
-            if project_url:
-                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
-                await self.emit_event(
-                    code_event, context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}"
-                )
-            namespace = project.get("namespace", {})
-            if namespace:
-                await self.handle_namespace(namespace, event)
-
-    async def handle_groups_url(self, groups_url, event):
-        for group in await self.gitlab_json_request(groups_url):
-            await self.handle_namespace(group, event)
-
-    async def gitlab_json_request(self, url):
-        response = await self.api_request(url)
-        if response is not None:
-            try:
-                json = response.json()
-            except Exception:
-                return []
-            if json and isinstance(json, list):
-                return json
-        return []
-
-    async def handle_namespace(self, namespace, event):
-        namespace_name = namespace.get("path", "")
-        namespace_url = namespace.get("web_url", "")
-        namespace_path = namespace.get("full_path", "")
-        if namespace_name and namespace_url and namespace_path:
-            namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
-            social_event = self.make_event(
-                {"platform": "gitlab", "profile_name": namespace_path, "url": namespace_url},
-                "SOCIAL",
-                parent=event,
-            )
-            await self.emit_event(
-                social_event,
-                context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
-            )
-
-    def get_base_url(self, event):
-        base_url = event.data.get("url", "")
-        if not base_url:
-            base_url = f"https://{event.host}"
-        return self.helpers.urlparse(base_url)._replace(path="/").geturl()