bbot 2.7.1.7212rc0__tar.gz → 2.7.2__tar.gz

{bbot-2.7.1.7212rc0 → bbot-2.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bbot
-Version: 2.7.1.7212rc0
+Version: 2.7.2
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 License-File: LICENSE

{bbot-2.7.1.7212rc0 → bbot-2.7.2}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.7.1.7212rc"
+__version__ = "v2.7.2"
 
 from .scanner import Scanner, Preset
 

bbot-2.7.2/bbot/modules/gitlab_com.py

@@ -0,0 +1,31 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_com(GitLabBaseModule):
+    watched_events = ["SOCIAL"]
+    produced_events = [
+        "CODE_REPOSITORY",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Enumerate GitLab SaaS (gitlab.com/org) for projects and groups",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"}
+
+    # This is needed because we are consuming SOCIAL events, which aren't in scope
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        await self.handle_social(event)
+
+    async def filter_event(self, event):
+        if event.data["platform"] != "gitlab":
+            return False, "platform is not gitlab"
+        _, domain = self.helpers.split_domain(event.host)
+        if domain not in self.saas_domains:
+            return False, "gitlab instance is not gitlab.com/org"
+        return True

bbot-2.7.2/bbot/modules/gitlab_onprem.py

@@ -0,0 +1,84 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_onprem(GitLabBaseModule):
+    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
+    produced_events = [
+        "TECHNOLOGY",
+        "SOCIAL",
+        "CODE_REPOSITORY",
+        "FINDING",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Detect self-hosted GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    # Optional GitLab access token (only required for gitlab.com, but still
+    # supported for on-prem installations that expose private projects).
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"}
+
+    # Allow accepting events slightly beyond configured max distance so we can
+    # discover repos on neighbouring infrastructure.
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        if event.type == "HTTP_RESPONSE":
+            await self.handle_http_response(event)
+        elif event.type == "TECHNOLOGY":
+            await self.handle_technology(event)
+        elif event.type == "SOCIAL":
+            await self.handle_social(event)
+
+    async def filter_event(self, event):
+        # only accept out-of-scope SOCIAL events
+        if event.type == "HTTP_RESPONSE":
+            if event.scope_distance > self.scan.scope_search_distance:
+                return False, "event is out of scope distance"
+        elif event.type == "TECHNOLOGY":
+            if not event.data["technology"].lower().startswith("gitlab"):
+                return False, "technology is not gitlab"
+            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
+                return False, "gitlab instance is not self-hosted"
+        elif event.type == "SOCIAL":
+            if event.data["platform"] != "gitlab":
+                return False, "platform is not gitlab"
+            _, domain = self.helpers.split_domain(event.host)
+            if domain in self.saas_domains:
+                return False, "gitlab instance is not self-hosted"
+        return True
+
+    async def handle_http_response(self, event):
+        """Identify GitLab servers from HTTP responses."""
+        headers = event.data.get("header", {})
+        if "x_gitlab_meta" in headers:
+            url = event.parsed_url._replace(path="/").geturl()
+            await self.emit_event(
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
+            )
+            description = f"GitLab server at {event.host}"
+            await self.emit_event(
+                {"host": str(event.host), "description": description},
+                "FINDING",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
+            )
+
+    async def handle_technology(self, event):
+        """Enumerate projects & groups once we know a host is GitLab."""
+        base_url = self.get_base_url(event)
+
+        # Projects owned by the authenticated user (or public projects if no
+        # authentication).
+        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
+        await self.handle_projects_url(projects_url, event)
+
+        # Group enumeration.
+        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
+        await self.handle_groups_url(groups_url, event)

bbot-2.7.2/bbot/modules/templates/gitlab.py

@@ -0,0 +1,98 @@
+from bbot.modules.base import BaseModule
+
+
+class GitLabBaseModule(BaseModule):
+    """Common functionality for interacting with GitLab instances.
+
+    This template is intended to be inherited by two concrete modules:
+    1. ``gitlab_com``   – Handles public SaaS instances (gitlab.com / gitlab.org).
+    2. ``gitlab_onprem`` – Handles self-hosted, on-premises GitLab servers.
+
+    Both child modules share identical behaviour when talking to the GitLab
+    REST API; they only differ in which events they are willing to accept.
+    """
+
+    # domains owned by GitLab
+    saas_domains = ["gitlab.com", "gitlab.org"]
+
+    async def setup(self):
+        if self.options.get("api_key") is not None:
+            await self.require_api_key()
+        return True
+
+    async def handle_social(self, event):
+        """Enumerate projects belonging to a user or group profile."""
+        username = event.data.get("profile_name", "")
+        if not username:
+            return
+        base_url = self.get_base_url(event)
+        urls = [
+            # User-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
+            # Group-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
+        ]
+        for url in urls:
+            await self.handle_projects_url(url, event)
+
+    async def handle_projects_url(self, projects_url, event):
+        for project in await self.gitlab_json_request(projects_url):
+            project_url = project.get("web_url", "")
+            if project_url:
+                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
+                await self.emit_event(
+                    code_event,
+                    context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}",
+                )
+            namespace = project.get("namespace", {})
+            if namespace:
+                await self.handle_namespace(namespace, event)
+
+    async def handle_groups_url(self, groups_url, event):
+        for group in await self.gitlab_json_request(groups_url):
+            await self.handle_namespace(group, event)
+
+    async def gitlab_json_request(self, url):
+        """Helper that performs an HTTP request and safely returns JSON list."""
+        response = await self.api_request(url)
+        if response is not None:
+            try:
+                json_data = response.json()
+            except Exception:
+                return []
+            if json_data and isinstance(json_data, list):
+                return json_data
+        return []
+
+    async def handle_namespace(self, namespace, event):
+        namespace_name = namespace.get("path", "")
+        namespace_url = namespace.get("web_url", "")
+        namespace_path = namespace.get("full_path", "")
+
+        if not (namespace_name and namespace_url and namespace_path):
+            return
+
+        namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
+
+        social_event = self.make_event(
+            {
+                "platform": "gitlab",
+                "profile_name": namespace_path,
+                "url": namespace_url,
+            },
+            "SOCIAL",
+            parent=event,
+        )
+        await self.emit_event(
+            social_event,
+            context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
+        )
+
+    # ------------------------------------------------------------------
+    # Utility helpers
+    # ------------------------------------------------------------------
+    def get_base_url(self, event):
+        base_url = event.data.get("url", "")
+        if not base_url:
+            base_url = f"https://{event.host}"
+        return self.helpers.urlparse(base_url)._replace(path="/").geturl()

bbot-2.7.2/bbot/test/test_step_2/module_tests/test_module_gitlab_com.py

@@ -0,0 +1,66 @@
+from .base import ModuleTestBase
+
+
+class TestGitlab_Com(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["gitlab_com", "httpx", "social", "excavate"]
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
+        module_test.httpx_mock.add_response(
+            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
+            json=[
+                {
+                    "id": 55490429,
+                    "description": None,
+                    "name": "Veilid",
+                    "name_with_namespace": "Veilid / Veilid",
+                    "path": "veilid",
+                    "path_with_namespace": "veilidgroup/veilid",
+                    "created_at": "2024-03-03T05:22:53.169Z",
+                    "default_branch": "master",
+                    "tag_list": [],
+                    "topics": [],
+                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
+                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
+                    "web_url": "https://gitlab.org/veilidgroup/veilid",
+                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
+                    "forks_count": 0,
+                    "avatar_url": None,
+                    "star_count": 0,
+                    "last_activity_at": "2024-03-03T05:22:53.097Z",
+                    "namespace": {
+                        "id": 66882294,
+                        "name": "veilidgroup",
+                        "path": "veilidgroup",
+                        "kind": "group",
+                        "full_path": "veilidgroup",
+                        "parent_id": None,
+                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
+                        "web_url": "https://gitlab.org/groups/veilidgroup",
+                    },
+                },
+            ],
+        )
+
+    def check(self, module_test, events):
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "gitlab"
+                and e.data["profile_name"] == "veilidgroup"
+                and e.data["url"] == "https://gitlab.org/veilidgroup"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
+                and str(e.module) == "gitlab_com"
+            ]
+        )

bbot-2.7.1.7212rc0/bbot/test/test_step_2/module_tests/test_module_gitlab.py → bbot-2.7.2/bbot/test/test_step_2/module_tests/test_module_gitlab_onprem.py

@@ -1,10 +1,10 @@
 from .base import ModuleTestBase
 
 
-class TestGitlab(ModuleTestBase):
+class TestGitlab_OnPrem(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx"]
-    config_overrides = {"modules": {"gitlab": {"api_key": "asdf"}}}
+    modules_overrides = ["gitlab_onprem", "httpx"]
+    config_overrides = {"modules": {"gitlab_onprem": {"api_key": "asdf"}}}
 
     async def setup_before_prep(self, module_test):
         module_test.httpserver.expect_request("/").respond_with_data(headers={"X-Gitlab-Meta": "asdf"})
@@ -179,7 +179,7 @@ class TestGitlab(ModuleTestBase):
                 and e.data["platform"] == "gitlab"
                 and e.data["profile_name"] == "bbotgroup"
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup"
-                and str(e.module) == "gitlab"
+                and str(e.module) == "gitlab_onprem"
             ]
         )
         assert 1 == len(
@@ -209,68 +209,3 @@ class TestGitlab(ModuleTestBase):
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup/bbot3"
             ]
         )
-
-
-class TestGitlabDotOrg(ModuleTestBase):
-    targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx", "social", "excavate"]
-
-    async def setup_before_prep(self, module_test):
-        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
-        module_test.httpx_mock.add_response(
-            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
-            json=[
-                {
-                    "id": 55490429,
-                    "description": None,
-                    "name": "Veilid",
-                    "name_with_namespace": "Veilid / Veilid",
-                    "path": "veilid",
-                    "path_with_namespace": "veilidgroup/veilid",
-                    "created_at": "2024-03-03T05:22:53.169Z",
-                    "default_branch": "master",
-                    "tag_list": [],
-                    "topics": [],
-                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
-                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
-                    "web_url": "https://gitlab.org/veilidgroup/veilid",
-                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
-                    "forks_count": 0,
-                    "avatar_url": None,
-                    "star_count": 0,
-                    "last_activity_at": "2024-03-03T05:22:53.097Z",
-                    "namespace": {
-                        "id": 66882294,
-                        "name": "veilidgroup",
-                        "path": "veilidgroup",
-                        "kind": "group",
-                        "full_path": "veilidgroup",
-                        "parent_id": None,
-                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
-                        "web_url": "https://gitlab.org/groups/veilidgroup",
-                    },
-                },
-            ],
-        )
-
-    def check(self, module_test, events):
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "SOCIAL"
-                and e.data["platform"] == "gitlab"
-                and e.data["profile_name"] == "veilidgroup"
-                and e.data["url"] == "https://gitlab.org/veilidgroup"
-            ]
-        )
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "CODE_REPOSITORY"
-                and "git" in e.tags
-                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
-                and str(e.module) == "gitlab"
-            ]
-        )

{bbot-2.7.1.7212rc0 → bbot-2.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.7.1.7212rc"
+version = "v2.7.2"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",
@@ -71,7 +71,7 @@ pytest-timeout = "^2.3.1"
 pytest-httpserver = "^1.0.11"
 pytest = "^8.3.1"
 pytest-asyncio = "1.2.0"
-uvicorn = ">=0.32,<0.38"
+uvicorn = ">=0.32,<0.39"
 fastapi = ">=0.115.5,<0.120.0"
 pytest-httpx = ">=0.35"
 pytest-benchmark = ">=4,<6"
@@ -121,7 +121,7 @@ lint.ignore = ["E402", "E711", "E713", "E721", "E741", "F403", "F405", "E501"]
 [tool.poetry-dynamic-versioning]
 enable = false
 metadata = false
-format-jinja = 'v2.7.1{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
+format-jinja = 'v2.7.2{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
 
 [tool.poetry-dynamic-versioning.substitution]
 files = ["*/__init__.py"]

bbot-2.7.1.7212rc0/bbot/modules/gitlab.py

@@ -1,141 +0,0 @@
-from bbot.modules.base import BaseModule
-
-
-class gitlab(BaseModule):
-    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
-    produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"]
-    flags = ["active", "safe", "code-enum"]
-    meta = {
-        "description": "Detect GitLab instances and query them for repositories",
-        "created_date": "2024-03-11",
-        "author": "@TheTechromancer",
-    }
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Gitlab access token"}
-
-    scope_distance_modifier = 2
-
-    async def setup(self):
-        await self.require_api_key()
-        return True
-
-    async def filter_event(self, event):
-        # only accept out-of-scope SOCIAL events
-        if event.type == "HTTP_RESPONSE":
-            if event.scope_distance > self.scan.scope_search_distance:
-                return False, "event is out of scope distance"
-        elif event.type == "TECHNOLOGY":
-            if not event.data["technology"].lower().startswith("gitlab"):
-                return False, "technology is not gitlab"
-            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
-                return False, "gitlab instance is not self-hosted"
-        elif event.type == "SOCIAL":
-            if event.data["platform"] != "gitlab":
-                return False, "platform is not gitlab"
-        return True
-
-    async def handle_event(self, event):
-        if event.type == "HTTP_RESPONSE":
-            await self.handle_http_response(event)
-        elif event.type == "TECHNOLOGY":
-            await self.handle_technology(event)
-        elif event.type == "SOCIAL":
-            await self.handle_social(event)
-
-    async def handle_http_response(self, event):
-        # identify gitlab instances from HTTP responses
-        # HTTP_RESPONSE --> TECHNOLOGY
-        # HTTP_RESPONSE --> FINDING
-        headers = event.data.get("header", {})
-        if "x_gitlab_meta" in headers:
-            url = event.parsed_url._replace(path="/").geturl()
-            await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url},
-                "TECHNOLOGY",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
-            )
-            description = f"GitLab server at {event.host}"
-            await self.emit_event(
-                {"host": str(event.host), "description": description},
-                "FINDING",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: {description}",
-            )
-
-    async def handle_technology(self, event):
-        # retrieve gitlab groups from gitlab instances
-        # TECHNOLOGY --> SOCIAL
-        # TECHNOLOGY --> URL
-        # TECHNOLOGY --> CODE_REPOSITORY
-        base_url = self.get_base_url(event)
-        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
-        await self.handle_projects_url(projects_url, event)
-        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
-        await self.handle_groups_url(groups_url, event)
-
-    async def handle_social(self, event):
-        # retrieve repositories from gitlab user
-        # SOCIAL --> CODE_REPOSITORY
-        # SOCIAL --> SOCIAL
-        username = event.data.get("profile_name", "")
-        if not username:
-            return
-        base_url = self.get_base_url(event)
-        urls = [
-            # group
-            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
-            # user
-            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
-        ]
-        for url in urls:
-            await self.handle_projects_url(url, event)
-
-    async def handle_projects_url(self, projects_url, event):
-        for project in await self.gitlab_json_request(projects_url):
-            project_url = project.get("web_url", "")
-            if project_url:
-                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
-                await self.emit_event(
-                    code_event, context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}"
-                )
-            namespace = project.get("namespace", {})
-            if namespace:
-                await self.handle_namespace(namespace, event)
-
-    async def handle_groups_url(self, groups_url, event):
-        for group in await self.gitlab_json_request(groups_url):
-            await self.handle_namespace(group, event)
-
-    async def gitlab_json_request(self, url):
-        response = await self.api_request(url)
-        if response is not None:
-            try:
-                json = response.json()
-            except Exception:
-                return []
-            if json and isinstance(json, list):
-                return json
-        return []
-
-    async def handle_namespace(self, namespace, event):
-        namespace_name = namespace.get("path", "")
-        namespace_url = namespace.get("web_url", "")
-        namespace_path = namespace.get("full_path", "")
-        if namespace_name and namespace_url and namespace_path:
-            namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
-            social_event = self.make_event(
-                {"platform": "gitlab", "profile_name": namespace_path, "url": namespace_url},
-                "SOCIAL",
-                parent=event,
-            )
-            await self.emit_event(
-                social_event,
-                context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
-            )
-
-    def get_base_url(self, event):
-        base_url = event.data.get("url", "")
-        if not base_url:
-            base_url = f"https://{event.host}"
-        return self.helpers.urlparse(base_url)._replace(path="/").geturl()