bbot 2.7.1.7212rc0__py3-none-any.whl → 2.7.2.7271rc0__py3-none-any.whl

bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.7.1.7212rc"
+__version__ = "v2.7.2.7271rc"
 
 from .scanner import Scanner, Preset
 

bbot/cli.py

@@ -7,7 +7,7 @@ import multiprocessing
 from bbot.errors import *
 from bbot import __version__
 from bbot.logger import log_to_stderr
-from bbot.core.helpers.misc import chain_lists
+from bbot.core.helpers.misc import chain_lists, rm_rf
 
 
 if multiprocessing.current_process().name == "MainProcess":
@@ -173,13 +173,27 @@ async def _main():
 
         # --install-all-deps
         if options.install_all_deps:
-            all_modules = list(preset.module_loader.preloaded())
-            scan.helpers.depsinstaller.force_deps = True
-            succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            if failed:
-                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+            preloaded_modules = preset.module_loader.preloaded()
+            scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"]
+            output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"]
+            log.verbose("Creating dummy scan with all modules + output modules for deps installation")
+            dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)
+            dummy_scan.helpers.depsinstaller.force_deps = True
+            log.info("Installing module dependencies")
+            await dummy_scan.load_modules()
+            log.verbose("Running module setups")
+            succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)
+            # remove any leftovers from the dummy scan
+            rm_rf(dummy_scan.home, ignore_errors=True)
+            rm_rf(dummy_scan.temp_dir, ignore_errors=True)
+            if succeeded:
+                log.success(
+                    f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}"
+                )
+            if soft_failed or hard_failed:
+                failed = soft_failed + hard_failed
+                log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}")
                 return False
-            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
             return True
 
         scan_name = str(scan.name)

bbot/core/flags.py

@@ -6,6 +6,7 @@ flag_descriptions = {
     "cloud-enum": "Enumerates cloud resources",
     "code-enum": "Find public code repositories and search them for secrets etc.",
     "deadly": "Highly aggressive",
+    "download": "Modules that download files, apps, or repositories",
     "email-enum": "Enumerates email addresses",
     "iis-shortnames": "Scans for IIS Shortname vulnerability",
     "passive": "Never connects to target systems",

bbot/core/helpers/web/web.py

@@ -267,7 +267,8 @@ class WebHelper(EngineClient):
         if not path:
             raise WordlistError(f"Invalid wordlist: {path}")
         if "cache_hrs" not in kwargs:
-            kwargs["cache_hrs"] = 720
+            # 4320 hrs = 180 days = 6 months
+            kwargs["cache_hrs"] = 4320
         if self.parent_helper.is_url(path):
             filename = await self.download(str(path), **kwargs)
             if filename is None:

bbot/core/modules.py

@@ -56,7 +56,6 @@ class ModuleLoader:
         self._shared_deps = dict(SHARED_DEPS)
 
         self.__preloaded = {}
-        self._modules = {}
         self._configs = {}
         self.flag_choices = set()
         self.all_module_choices = set()
@@ -463,7 +462,6 @@ class ModuleLoader:
         for module_name in module_names:
             module = self.load_module(module_name)
             modules[module_name] = module
-            self._modules[module_name] = module
         return modules
 
     def load_module(self, module_name):

bbot/modules/apkpure.py

@@ -6,7 +6,7 @@ from bbot.modules.base import BaseModule
 class apkpure(BaseModule):
     watched_events = ["MOBILE_APP"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download android applications from apkpure.com",
         "created_date": "2024-10-11",

bbot/modules/base.py

@@ -213,6 +213,14 @@ class BaseModule:
 
         return True
 
+    async def setup_deps(self):
+        """
+        Similar to setup(), but reserved for installing dependencies not covered by Ansible.
+
+        This should always be used to install static dependencies like AI models, wordlists, etc.
+        """
+        return True
+
     async def handle_event(self, event, **kwargs):
         """Asynchronously handles incoming events that the module is configured to watch.
 
@@ -620,39 +628,26 @@ class BaseModule:
             name=f"{self.scan.name}.{self.name}._event_handler_watchdog()",
         )
 
-    async def _setup(self):
-        """
-        Asynchronously sets up the module by invoking its 'setup()' method.
-
-        This method catches exceptions during setup, sets the module's error state if necessary, and determines the
-        status code based on the result of the setup process.
-
-        Args:
-            None
-
-        Returns:
-            tuple: A tuple containing the module's name, status (True for success, False for hard-fail, None for soft-fail),
-            and an optional status message.
-
-        Raises:
-            Exception: Captured exceptions from the 'setup()' method are logged, but not propagated.
-
-        Notes:
-            - The 'setup()' method can return either a simple boolean status or a tuple of status and message.
-            - A WordlistError exception triggers a soft-fail status.
-            - The debug log will contain setup status information for the module.
-        """
+    async def _setup(self, deps_only=False):
+        """ """
         status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}
 
         status = False
         self.debug(f"Setting up module {self.name}")
         try:
-            result = await self.setup()
-            if type(result) == tuple and len(result) == 2:
-                status, msg = result
-            else:
-                status = result
-                msg = status_codes[status]
+            funcs = [self.setup_deps]
+            if not deps_only:
+                funcs.append(self.setup)
+            for func in funcs:
+                self.debug(f"Running {self.name}.{func.__name__}()")
+                result = await func()
+                if type(result) == tuple and len(result) == 2:
+                    status, msg = result
+                else:
+                    status = result
+                    msg = status_codes[status]
+                if status is False:
+                    break
             self.debug(f"Finished setting up module {self.name}")
         except Exception as e:
             self.set_error_state(f"Unexpected error during module setup: {e}", critical=True)

bbot/modules/dnsbrute.py

@@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum):
     dedup_strategy = "lowest_parent"
     _qsize = 10000
 
+    async def setup_deps(self):
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        # tell the dnsbrute helper to fetch the resolver file
+        await self.helpers.dns.brute.resolver_file()
+        return True
+
     async def setup(self):
         self.max_depth = max(1, self.config.get("max_depth", 5))
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
         self.wordlist_size = len(self.subdomain_list)
         return await super().setup()

bbot/modules/docker_pull.py

@@ -8,7 +8,7 @@ from bbot.modules.base import BaseModule
 class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",

bbot/modules/ffuf.py

@@ -37,12 +37,15 @@ class ffuf(BaseModule):
 
     in_scope_only = True
 
+    async def setup_deps(self):
+        self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
+        return True
+
     async def setup(self):
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist_url = self.config.get("wordlist", "")
         self.debug(f"Using wordlist [{wordlist_url}]")
-        self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
         self.rate = self.config.get("rate", 0)

bbot/modules/ffuf_shortnames.py

@@ -87,14 +87,17 @@ class ffuf_shortnames(ffuf):
                     found_prefixes.add(prefix)
         return list(found_prefixes)
 
-    async def setup(self):
-        self.proxy = self.scan.web_config.get("http_proxy", "")
-        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
+    async def setup_deps(self):
         wordlist_extensions = self.config.get("wordlist_extensions", "")
         if not wordlist_extensions:
             wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
         self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
+        return True
+
+    async def setup(self):
+        self.proxy = self.scan.web_config.get("http_proxy", "")
+        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")

bbot/modules/filedownload.py

@@ -14,7 +14,7 @@ class filedownload(BaseModule):
 
     watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"]
     produced_events = ["FILESYSTEM"]
-    flags = ["active", "safe", "web-basic"]
+    flags = ["active", "safe", "web-basic", "download"]
     meta = {
         "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
         "created_date": "2023-10-11",
@@ -94,6 +94,12 @@ class filedownload(BaseModule):
 
     scope_distance_modifier = 3
 
+    async def setup_deps(self):
+        self.mime_db_file = await self.helpers.wordlist(
+            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
+        )
+        return True
+
     async def setup(self):
         self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
         self.max_filesize = self.config.get("max_filesize", "10MB")
@@ -105,9 +111,6 @@ class filedownload(BaseModule):
         else:
             self.download_dir = self.scan.temp_dir / "filedownload"
         self.helpers.mkdir(self.download_dir)
-        self.mime_db_file = await self.helpers.wordlist(
-            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
-        )
         self.mime_db = {}
         with open(self.mime_db_file) as f:
             mime_db = json.load(f)

bbot/modules/git_clone.py

@@ -6,7 +6,7 @@ from bbot.modules.templates.github import github
 class git_clone(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Clone code github repositories",
         "created_date": "2024-03-08",

bbot/modules/gitdumper.py

@@ -7,7 +7,7 @@ from bbot.modules.base import BaseModule
 class gitdumper(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download a leaked .git folder recursively or by fuzzing common names",
         "created_date": "2025-02-11",

bbot/modules/github_workflows.py

@@ -8,7 +8,7 @@ from bbot.modules.templates.github import github
 class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download a github repositories workflow logs and workflow artifacts",
         "created_date": "2024-04-29",

bbot/modules/gitlab_com.py

@@ -0,0 +1,31 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_com(GitLabBaseModule):
+    watched_events = ["SOCIAL"]
+    produced_events = [
+        "CODE_REPOSITORY",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Enumerate GitLab SaaS (gitlab.com/org) for projects and groups",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"}
+
+    # This is needed because we are consuming SOCIAL events, which aren't in scope
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        await self.handle_social(event)
+
+    async def filter_event(self, event):
+        if event.data["platform"] != "gitlab":
+            return False, "platform is not gitlab"
+        _, domain = self.helpers.split_domain(event.host)
+        if domain not in self.saas_domains:
+            return False, "gitlab instance is not gitlab.com/org"
+        return True

bbot/modules/gitlab_onprem.py

@@ -0,0 +1,84 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_onprem(GitLabBaseModule):
+    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
+    produced_events = [
+        "TECHNOLOGY",
+        "SOCIAL",
+        "CODE_REPOSITORY",
+        "FINDING",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Detect self-hosted GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    # Optional GitLab access token (only required for gitlab.com, but still
+    # supported for on-prem installations that expose private projects).
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"}
+
+    # Allow accepting events slightly beyond configured max distance so we can
+    # discover repos on neighbouring infrastructure.
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        if event.type == "HTTP_RESPONSE":
+            await self.handle_http_response(event)
+        elif event.type == "TECHNOLOGY":
+            await self.handle_technology(event)
+        elif event.type == "SOCIAL":
+            await self.handle_social(event)
+
+    async def filter_event(self, event):
+        # only accept out-of-scope SOCIAL events
+        if event.type == "HTTP_RESPONSE":
+            if event.scope_distance > self.scan.scope_search_distance:
+                return False, "event is out of scope distance"
+        elif event.type == "TECHNOLOGY":
+            if not event.data["technology"].lower().startswith("gitlab"):
+                return False, "technology is not gitlab"
+            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
+                return False, "gitlab instance is not self-hosted"
+        elif event.type == "SOCIAL":
+            if event.data["platform"] != "gitlab":
+                return False, "platform is not gitlab"
+            _, domain = self.helpers.split_domain(event.host)
+            if domain in self.saas_domains:
+                return False, "gitlab instance is not self-hosted"
+        return True
+
+    async def handle_http_response(self, event):
+        """Identify GitLab servers from HTTP responses."""
+        headers = event.data.get("header", {})
+        if "x_gitlab_meta" in headers:
+            url = event.parsed_url._replace(path="/").geturl()
+            await self.emit_event(
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
+            )
+            description = f"GitLab server at {event.host}"
+            await self.emit_event(
+                {"host": str(event.host), "description": description},
+                "FINDING",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
+            )
+
+    async def handle_technology(self, event):
+        """Enumerate projects & groups once we know a host is GitLab."""
+        base_url = self.get_base_url(event)
+
+        # Projects owned by the authenticated user (or public projects if no
+        # authentication).
+        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
+        await self.handle_projects_url(projects_url, event)
+
+        # Group enumeration.
+        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
+        await self.handle_groups_url(groups_url, event)

bbot/modules/medusa.py

@@ -1,6 +1,5 @@
 import re
 from bbot.modules.base import BaseModule
-from bbot.errors import WordlistError
 
 
 class medusa(BaseModule):
@@ -102,13 +101,11 @@ class medusa(BaseModule):
         },
     ]
 
-    async def setup(self):
-        # Try to cache wordlist
-        try:
-            self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
-        except WordlistError as e:
-            return False, f"Error retrieving wordlist: {e}"
+    async def setup_deps(self):
+        self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist"))
+        return True
 
+    async def setup(self):
         self.password_match_regex = re.compile(r"Password:\s*(\S+)")
         self.success_indicator_match_regex = re.compile(r"\[([^\]]+)\]\s*$")
 

bbot/modules/paramminer_headers.py

@@ -82,18 +82,21 @@ class paramminer_headers(BaseModule):
 
     header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$")
 
-    async def setup(self):
-        self.recycle_words = self.config.get("recycle_words", True)
-        self.event_dict = {}
-        self.already_checked = set()
+    async def setup_deps(self):
         wordlist = self.config.get("wordlist", "")
         if not wordlist:
             wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}"
+        self.wordlist_file = await self.helpers.wordlist(wordlist)
         self.debug(f"Using wordlist: [{wordlist}]")
+        return True
+
+    async def setup(self):
+        self.recycle_words = self.config.get("recycle_words", True)
+        self.event_dict = {}
+        self.already_checked = set()
+
         self.wl = {
-            h.strip().lower()
-            for h in self.helpers.read_file(await self.helpers.wordlist(wordlist))
-            if len(h) > 0 and "%" not in h
+            h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and "%" not in h
         }
 
         # check against the boring list (if the option is set)

bbot/modules/postman_download.py

@@ -7,7 +7,7 @@ from bbot.modules.templates.postman import postman
 class postman_download(postman):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "subdomain-enum", "safe", "code-enum"]
+    flags = ["passive", "subdomain-enum", "safe", "code-enum", "download"]
     meta = {
         "description": "Download workspaces, collections, requests from Postman",
         "created_date": "2024-09-07",

bbot/modules/templates/gitlab.py

@@ -0,0 +1,98 @@
+from bbot.modules.base import BaseModule
+
+
+class GitLabBaseModule(BaseModule):
+    """Common functionality for interacting with GitLab instances.
+
+    This template is intended to be inherited by two concrete modules:
+    1. ``gitlab_com``   – Handles public SaaS instances (gitlab.com / gitlab.org).
+    2. ``gitlab_onprem`` – Handles self-hosted, on-premises GitLab servers.
+
+    Both child modules share identical behaviour when talking to the GitLab
+    REST API; they only differ in which events they are willing to accept.
+    """
+
+    # domains owned by GitLab
+    saas_domains = ["gitlab.com", "gitlab.org"]
+
+    async def setup(self):
+        if self.options.get("api_key") is not None:
+            await self.require_api_key()
+        return True
+
+    async def handle_social(self, event):
+        """Enumerate projects belonging to a user or group profile."""
+        username = event.data.get("profile_name", "")
+        if not username:
+            return
+        base_url = self.get_base_url(event)
+        urls = [
+            # User-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
+            # Group-owned projects
+            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
+        ]
+        for url in urls:
+            await self.handle_projects_url(url, event)
+
+    async def handle_projects_url(self, projects_url, event):
+        for project in await self.gitlab_json_request(projects_url):
+            project_url = project.get("web_url", "")
+            if project_url:
+                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
+                await self.emit_event(
+                    code_event,
+                    context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}",
+                )
+            namespace = project.get("namespace", {})
+            if namespace:
+                await self.handle_namespace(namespace, event)
+
+    async def handle_groups_url(self, groups_url, event):
+        for group in await self.gitlab_json_request(groups_url):
+            await self.handle_namespace(group, event)
+
+    async def gitlab_json_request(self, url):
+        """Helper that performs an HTTP request and safely returns JSON list."""
+        response = await self.api_request(url)
+        if response is not None:
+            try:
+                json_data = response.json()
+            except Exception:
+                return []
+            if json_data and isinstance(json_data, list):
+                return json_data
+        return []
+
+    async def handle_namespace(self, namespace, event):
+        namespace_name = namespace.get("path", "")
+        namespace_url = namespace.get("web_url", "")
+        namespace_path = namespace.get("full_path", "")
+
+        if not (namespace_name and namespace_url and namespace_path):
+            return
+
+        namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
+
+        social_event = self.make_event(
+            {
+                "platform": "gitlab",
+                "profile_name": namespace_path,
+                "url": namespace_url,
+            },
+            "SOCIAL",
+            parent=event,
+        )
+        await self.emit_event(
+            social_event,
+            context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
+        )
+
+    # ------------------------------------------------------------------
+    # Utility helpers
+    # ------------------------------------------------------------------
+    def get_base_url(self, event):
+        base_url = event.data.get("url", "")
+        if not base_url:
+            base_url = f"https://{event.host}"
+        return self.helpers.urlparse(base_url)._replace(path="/").geturl()

bbot/modules/trufflehog.py

@@ -41,11 +41,14 @@ class trufflehog(BaseModule):
 
     scope_distance_modifier = 2
 
-    async def setup(self):
-        self.verified = self.config.get("only_verified", True)
+    async def setup_deps(self):
         self.config_file = self.config.get("config", "")
         if self.config_file:
             self.config_file = await self.helpers.wordlist(self.config_file)
+        return True
+
+    async def setup(self):
+        self.verified = self.config.get("only_verified", True)
         self.concurrency = int(self.config.get("concurrency", 8))
 
         self.deleted_forks = self.config.get("deleted_forks", False)

bbot/scanner/scanner.py

@@ -484,7 +484,7 @@ class Scanner:
         for module in self.modules.values():
             module.start()
 
-    async def setup_modules(self, remove_failed=True):
+    async def setup_modules(self, remove_failed=True, deps_only=False):
         """Asynchronously initializes all loaded modules by invoking their `setup()` methods.
 
         Args:
@@ -509,7 +509,7 @@ class Scanner:
         hard_failed = []
         soft_failed = []
 
-        async for task in self.helpers.as_completed([m._setup() for m in self.modules.values()]):
+        async for task in self.helpers.as_completed([m._setup(deps_only=deps_only) for m in self.modules.values()]):
             module, status, msg = await task
             if status is True:
                 self.debug(f"Setup succeeded for {module.name} ({msg})")

bbot/test/test_step_1/test_modules_basic.py

@@ -342,6 +342,31 @@ async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
     await per_domain_scan._cleanup()
 
 
+@pytest.mark.asyncio
+async def test_modules_basic_setup_deps(bbot_scanner):
+    from bbot.modules.base import BaseModule
+
+    class dummy(BaseModule):
+        _name = "dummy"
+        deps_ran = False
+        setup_ran = False
+
+        async def setup_deps(self):
+            self.deps_ran = True
+            return True
+
+        async def setup(self):
+            self.setup_ran = True
+            return True
+
+    scan = bbot_scanner()
+    scan.modules["dummy"] = dummy(scan)
+    await scan.setup_modules(deps_only=True)
+    assert scan.modules["dummy"].deps_ran
+    assert not scan.modules["dummy"].setup_ran
+    await scan._cleanup()
+
+
 @pytest.mark.asyncio
 async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, monkeypatch):
     from bbot.modules.base import BaseModule

bbot/test/test_step_2/module_tests/test_module_gitlab_com.py

@@ -0,0 +1,66 @@
+from .base import ModuleTestBase
+
+
+class TestGitlab_Com(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["gitlab_com", "httpx", "social", "excavate"]
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
+        module_test.httpx_mock.add_response(
+            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
+            json=[
+                {
+                    "id": 55490429,
+                    "description": None,
+                    "name": "Veilid",
+                    "name_with_namespace": "Veilid / Veilid",
+                    "path": "veilid",
+                    "path_with_namespace": "veilidgroup/veilid",
+                    "created_at": "2024-03-03T05:22:53.169Z",
+                    "default_branch": "master",
+                    "tag_list": [],
+                    "topics": [],
+                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
+                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
+                    "web_url": "https://gitlab.org/veilidgroup/veilid",
+                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
+                    "forks_count": 0,
+                    "avatar_url": None,
+                    "star_count": 0,
+                    "last_activity_at": "2024-03-03T05:22:53.097Z",
+                    "namespace": {
+                        "id": 66882294,
+                        "name": "veilidgroup",
+                        "path": "veilidgroup",
+                        "kind": "group",
+                        "full_path": "veilidgroup",
+                        "parent_id": None,
+                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
+                        "web_url": "https://gitlab.org/groups/veilidgroup",
+                    },
+                },
+            ],
+        )
+
+    def check(self, module_test, events):
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "gitlab"
+                and e.data["profile_name"] == "veilidgroup"
+                and e.data["url"] == "https://gitlab.org/veilidgroup"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
+                and str(e.module) == "gitlab_com"
+            ]
+        )

bbot/test/test_step_2/module_tests/{test_module_gitlab.py → test_module_gitlab_onprem.py}

@@ -1,10 +1,10 @@
 from .base import ModuleTestBase
 
 
-class TestGitlab(ModuleTestBase):
+class TestGitlab_OnPrem(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx"]
-    config_overrides = {"modules": {"gitlab": {"api_key": "asdf"}}}
+    modules_overrides = ["gitlab_onprem", "httpx"]
+    config_overrides = {"modules": {"gitlab_onprem": {"api_key": "asdf"}}}
 
     async def setup_before_prep(self, module_test):
         module_test.httpserver.expect_request("/").respond_with_data(headers={"X-Gitlab-Meta": "asdf"})
@@ -179,7 +179,7 @@ class TestGitlab(ModuleTestBase):
                 and e.data["platform"] == "gitlab"
                 and e.data["profile_name"] == "bbotgroup"
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup"
-                and str(e.module) == "gitlab"
+                and str(e.module) == "gitlab_onprem"
             ]
         )
         assert 1 == len(
@@ -209,68 +209,3 @@ class TestGitlab(ModuleTestBase):
                 and e.data["url"] == "http://127.0.0.1:8888/bbotgroup/bbot3"
             ]
         )
-
-
-class TestGitlabDotOrg(ModuleTestBase):
-    targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["gitlab", "httpx", "social", "excavate"]
-
-    async def setup_before_prep(self, module_test):
-        module_test.httpserver.expect_request("/").respond_with_data("<a href='https://gitlab.org/veilidgroup'/>")
-        module_test.httpx_mock.add_response(
-            url="https://gitlab.org/api/v4/groups/veilidgroup/projects?simple=true",
-            json=[
-                {
-                    "id": 55490429,
-                    "description": None,
-                    "name": "Veilid",
-                    "name_with_namespace": "Veilid / Veilid",
-                    "path": "veilid",
-                    "path_with_namespace": "veilidgroup/veilid",
-                    "created_at": "2024-03-03T05:22:53.169Z",
-                    "default_branch": "master",
-                    "tag_list": [],
-                    "topics": [],
-                    "ssh_url_to_repo": "git@gitlab.org:veilid/veilid.git",
-                    "http_url_to_repo": "https://gitlab.org/veilidgroup/veilid.git",
-                    "web_url": "https://gitlab.org/veilidgroup/veilid",
-                    "readme_url": "https://gitlab.org/veilidgroup/veilid/-/blob/master/README.md",
-                    "forks_count": 0,
-                    "avatar_url": None,
-                    "star_count": 0,
-                    "last_activity_at": "2024-03-03T05:22:53.097Z",
-                    "namespace": {
-                        "id": 66882294,
-                        "name": "veilidgroup",
-                        "path": "veilidgroup",
-                        "kind": "group",
-                        "full_path": "veilidgroup",
-                        "parent_id": None,
-                        "avatar_url": "/uploads/-/system/group/avatar/66882294/signal-2023-07-04-192426_003.jpeg",
-                        "web_url": "https://gitlab.org/groups/veilidgroup",
-                    },
-                },
-            ],
-        )
-
-    def check(self, module_test, events):
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "SOCIAL"
-                and e.data["platform"] == "gitlab"
-                and e.data["profile_name"] == "veilidgroup"
-                and e.data["url"] == "https://gitlab.org/veilidgroup"
-            ]
-        )
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "CODE_REPOSITORY"
-                and "git" in e.tags
-                and e.data["url"] == "https://gitlab.org/veilidgroup/veilid"
-                and str(e.module) == "gitlab"
-            ]
-        )

{bbot-2.7.1.7212rc0.dist-info → bbot-2.7.2.7271rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bbot
-Version: 2.7.1.7212rc0
+Version: 2.7.2.7271rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 License-File: LICENSE

{bbot-2.7.1.7212rc0.dist-info → bbot-2.7.2.7271rc0.dist-info}/RECORD

@@ -1,5 +1,5 @@
-bbot/__init__.py,sha256=VSc_ku85NcucmahEOo7gG1JKRDfMFITM_ihdRv4OvPs,163
-bbot/cli.py,sha256=1QJbANVw9Q3GFM92H2QRV2ds5756ulm08CDZwzwPpeI,11888
+bbot/__init__.py,sha256=mzX1fioFjIfbuMCUdqxHKWZtvwkr1KdOqpKsmuve0MA,163
+bbot/cli.py,sha256=lcBQ7TxDxJEQVu89UJSkti_W-rc40Nfw4AlgRUvrT6E,12783
 bbot/core/__init__.py,sha256=l255GJE_DvUnWvrRb0J5lG-iMztJ8zVvoweDOfegGtI,46
 bbot/core/config/__init__.py,sha256=zYNw2Me6tsEr8hOOkLb4BQ97GB7Kis2k--G81S8vofU,342
 bbot/core/config/files.py,sha256=zANvrTRLJQIOWSNkxd9MpWmf9cQFr0gRZLUxeIbTwQc,1412
@@ -9,7 +9,7 @@ bbot/core/engine.py,sha256=DxlbxUWU1x20DTIsVsYXWuR5Z8eYJRmP-SOLyvO4Eek,29362
 bbot/core/event/__init__.py,sha256=pRi5lC9YBKGxx6ZgrnE4shqKYYdqKR1Ps6tDw2WKGOw,113
 bbot/core/event/base.py,sha256=1jUgd3I3TDITKoobh92ir_tIm38EN1ZbhoaX1W9fKts,67125
 bbot/core/event/helpers.py,sha256=MohOCVBjkn_K1p4Ipgx-MKliZtV6l4NJPq3YgagkvSM,6507
-bbot/core/flags.py,sha256=Ltvm8Bc4D65I55HuU5bzyjO1R3yMDNpVmreGU83ZBXE,1266
+bbot/core/flags.py,sha256=3bT3BYActmxomZLusjPROzrLhVlRJKLOCluZBPVauEg,1336
 bbot/core/helpers/__init__.py,sha256=cpOGLKIgA3vdHYqsOtx63BFO_qbtwCmez2amFPu6YTs,111
 bbot/core/helpers/async_helpers.py,sha256=bVHEUIOZo8iCmuovLYb3oNLPdLFUoEyc6wZIIvtELVs,4399
 bbot/core/helpers/bloom.py,sha256=fTMdgnrJPC9fzNXwBg1GYbz1P05YPhjxy9aUg6-z3a4,2605
@@ -44,10 +44,10 @@ bbot/core/helpers/web/client.py,sha256=RPm4kYdHzqPom0EdVFvAiUPuDpztW8cuqGaLfl2Tx
 bbot/core/helpers/web/engine.py,sha256=XmyDMXsJWasOklfPWOcJ6SzuLaUXP3HGJUBO5Gj2xFk,9221
 bbot/core/helpers/web/envelopes.py,sha256=mMmr4QGi28KJSwCkaogMGYhiqeqm3_kO9KziLtEKefc,10074
 bbot/core/helpers/web/ssl_context.py,sha256=aWVgl-d0HoE8B4EBKNxaa5UAzQmx79DjDByfBw9tezo,356
-bbot/core/helpers/web/web.py,sha256=q9X6JNufbZzRijzsc6bXkxH0ntly7rDr-uptscSKxRo,24042
+bbot/core/helpers/web/web.py,sha256=Imi29a1lxfcFeFpAwGui3Ckzxki3oTBGJ5np-JdGOhc,24088
 bbot/core/helpers/wordcloud.py,sha256=QM8Z1N01_hXrRFKQjvRL-IzOOC7ZMKjuSBID3u77Sxg,19809
 bbot/core/helpers/yara_helper.py,sha256=ypwC_H_ovJp9BpOwqgPkIdZEwqWfvqrRdKlwzBLfm8Q,1592
-bbot/core/modules.py,sha256=_h-TFaMw7NmpEO1orJ9kVgeVbATbMxo7zlGoa8p9Cs8,29354
+bbot/core/modules.py,sha256=aQtltmNVDBZuvaSzCpI7wxvzXy0thL6z9oN_HIkTYCo,29279
 bbot/core/multiprocess.py,sha256=ocQHanskJ09gHwe7RZmwNdZyCOQyeyUoIHCtLbtvXUk,1771
 bbot/core/shared_deps.py,sha256=NeJmyakKxQQjN-H3rYGwGuHeVxDiVM_KnDfeVEeIbf4,9498
 bbot/db/sql/models.py,sha256=SrUdDOBCICzXJBY29p0VvILhMQ1JCuh725bqvIYogX0,4884
@@ -57,7 +57,7 @@ bbot/logger.py,sha256=wE-532v5FyKuSSoTdyW1xSfaOnLZB1axAJnB-uW2xrI,2745
 bbot/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bbot/modules/ajaxpro.py,sha256=daE1yQoCsSI5c4dh3YKwRSggTISNjWgrK7qTPidk7cU,3764
 bbot/modules/anubisdb.py,sha256=JCy2YCfa0e_VawpzNmcPXAosKUthmYGutireJ0gMDws,1916
-bbot/modules/apkpure.py,sha256=h26zh-nv1G9IvvTwywbFcARiZPfHUhjIUGVl0Achbek,2555
+bbot/modules/apkpure.py,sha256=q15cOGsj7AfRnOZQAahJefnD1YGzsYWqkFz0RXs3RPk,2567
 bbot/modules/aspnet_bin_exposure.py,sha256=X16mK8tff7-LOcbS4fUschkDRdVVz9r2N8Ta2EZzw2Y,3737
 bbot/modules/azure_realm.py,sha256=pP2PUlLy0K9KKaE8aNcznWjDW3PKHvnMejdOSc-o4ms,1612
 bbot/modules/azure_tenant.py,sha256=qBn7CUA_hth2PqW55XZVjYxIw20xLYrMntXc6mYpmKU,5366
@@ -65,7 +65,7 @@ bbot/modules/baddns.py,sha256=vSWWBiPfVowAg1yrBAx0rm8ViSh1O3VX7nI9Pn2Z5mo,6694
 bbot/modules/baddns_direct.py,sha256=G5WTKQ-jKJnSLijsOjyQKoR2Sx49fvACBvAC-b_yPck,3820
 bbot/modules/baddns_zone.py,sha256=8s2vIbU2MsLW6w12bDyw8FWBfdDRn2A2gWoMFEX4gPw,1036
 bbot/modules/badsecrets.py,sha256=2M515_nmDg24WTDqM01w-2DNN2H8BewvqnG_8hG6n3o,5110
-bbot/modules/base.py,sha256=uatsSs15fcaawbu7JTfQiY8WhkKQ-6WGGG41hIT2YeQ,78811
+bbot/modules/base.py,sha256=mRGn1Td9V58P9saNQ8KOJkEGA3fmfwEf6FP_ZuWSIug,78508
 bbot/modules/bevigil.py,sha256=0VLIxmeXRUI2-EoR6IzuHJMcX8KCHNNta-WYa3gVlDg,2862
 bbot/modules/bucket_amazon.py,sha256=mwjYeEAcdfOpjbOa1sD8U9KBMMVY_c8FoHjSGR9GQbg,730
 bbot/modules/bucket_azure.py,sha256=Jaa9XEL7w7VM0a-WAp05MOGdP5nt7hMpLzBsPq74_IM,1284
@@ -86,31 +86,32 @@ bbot/modules/crt_db.py,sha256=xaIm2457_xGJjnKss73l1HpPn7pLPHksVzejsimTfZA,2198
 bbot/modules/dehashed.py,sha256=0lzcqMEgwRmprwurZ2-8Y8aOO4KTueJgpY_vh0DWQwA,5155
 bbot/modules/digitorus.py,sha256=XQY0eAQrA7yo8S57tGncP1ARud-yG4LiWxx5VBYID34,1027
 bbot/modules/dnsbimi.py,sha256=XqqZj_7dipYn-ByXC6IBTJ2bUSY3QOAG4e8F40kHuAI,6931
-bbot/modules/dnsbrute.py,sha256=Y2bSbG2IcwIJID1FSQ6Qe9fdpWwG7GIO-wVQw7MdQFM,2439
+bbot/modules/dnsbrute.py,sha256=w396XpfNmU3QDnByrgYL4FNXuD9sV9TA_jnYGw41nT8,2607
 bbot/modules/dnsbrute_mutations.py,sha256=EbAZ-ZOqk98OAMacc8PuX_zx6eXyn6gJxgFuZ8A71YA,7242
 bbot/modules/dnscaa.py,sha256=pyaLqHrdsVhqtd1JBZVjKKcuYT_ywUbFYkrnfXcGD5s,5014
 bbot/modules/dnscommonsrv.py,sha256=wrCRTlqVuxFIScWH0Cb0UQAVk0TWxgVc5fo5awl3R24,1568
 bbot/modules/dnsdumpster.py,sha256=x4_1ZcPRAKDiCWMt7x4bbfcar2-VN6fLjWx0ijPUEmY,2775
 bbot/modules/dnstlsrpt.py,sha256=v8V72RBsawmDPrMrTcKXEyoFt9bgbfm-cpoPYgKEKLQ,6238
-bbot/modules/docker_pull.py,sha256=zNQcQdS-JWM2-TbQ_iyjeGA9CKcpuXdeO5ucoJgzZNY,9189
+bbot/modules/docker_pull.py,sha256=G1B4mjZBFIfxlbmi_Q02-Y9N53moRKpjQ7Li_aU_MYM,9201
 bbot/modules/dockerhub.py,sha256=JQkujjqvQRzQuvHjQ7JbFs_VlJj8dLRPRObAkBgUQhc,3493
 bbot/modules/dotnetnuke.py,sha256=zipcHyNYr2FEecStb1Yrm938ps01RvHV8NnyqAvnGGc,10537
 bbot/modules/emailformat.py,sha256=Koi2aSng-FSRJVhpbFaclrqZxo4lQoPMcUMn_qXTfVE,1518
 bbot/modules/extractous.py,sha256=VSGKmHPAA_4r62jaN8Yqi3JcjehjxpI2lhe8i2j786s,4648
-bbot/modules/ffuf.py,sha256=94TJ5xvqKwH0JaWmC_t1dLTpRsO8HEy4lnbsu8LF_HY,14965
-bbot/modules/ffuf_shortnames.py,sha256=y5vnypLPN-KrjpmoG5zlqcX8VwfcLBpNg1yQI7bP9Hg,18737
-bbot/modules/filedownload.py,sha256=5MctNWSYyjoXPshRXbltsn92KDAr9fsLqbPGP4eK7Es,8903
+bbot/modules/ffuf.py,sha256=M08iRvt7HjpmwuvmS8OgKhCuNg2PU1LbZt_DZmZauEs,15033
+bbot/modules/ffuf_shortnames.py,sha256=saAW_csgMyGd1OVAPQa5hfdHkxf98VWRPiUIZWm29LY,18790
+bbot/modules/filedownload.py,sha256=IOmoK7_Y784hkUP27ImMFWcHqOYg441KJ0XkoLV6KFg,8968
 bbot/modules/fingerprintx.py,sha256=rdlR9d64AntAhbS_eJzh8bZCeLPTJPSKdkdKdhH_qAo,3269
 bbot/modules/fullhunt.py,sha256=2ntu1yBh51N4e_l-kpXc1UBoVVcxEE2JPkyaMYCuUb4,1336
 bbot/modules/generic_ssrf.py,sha256=KFdcHpUV9-Z7oN7emzbirimsNc2xZ_1IFqnsfIkEbcM,9196
 bbot/modules/git.py,sha256=zmHeI0bn181T1P8C55HSebkdVGLTpzGxPc-LRqiHrbc,1723
-bbot/modules/git_clone.py,sha256=SwtCnOpVqEgSMfqaN54NUpS2jYZWt4Fk8Y_TqUIO724,3764
-bbot/modules/gitdumper.py,sha256=mzlEJuWLlZIWXj-0V5kC8qTVLEvVtbrPColCXQGFEoQ,11588
+bbot/modules/git_clone.py,sha256=DzYQ5p_ZmezxSXKamjcxTVB0RNIywehs_UvtnFFVW2g,3776
+bbot/modules/gitdumper.py,sha256=F1qpErD3dv4qvWODgLi8MWMENfJxrl12OAVSbusiF34,11600
 bbot/modules/github_codesearch.py,sha256=a-r2vE9N9WyBpFUiKCsg0TK4Qn7DaEGyVRTUKzkDLWA,3641
 bbot/modules/github_org.py,sha256=WM18vJCHuOHJJ5rPzQzQ3Pmp7XPPuaMeVgNfW-FlO0k,8938
 bbot/modules/github_usersearch.py,sha256=G8knkQBJsn7EKcMhcEaFPiB_Y5S96e2VaseBubsqOyk,3407
-bbot/modules/github_workflows.py,sha256=xKntAFDeGuE4MqbEmhJyYXKbzoSh9tWYlHNlnF37PYA,10040
-bbot/modules/gitlab.py,sha256=9oWWpBijeHCjuFBfWW4HvNqt7bvJvrBgBjaaz_UPPnE,5964
+bbot/modules/github_workflows.py,sha256=cj9YLoW01v5Iey6s89zShlrAF80TEBhFcEWyrvdNe38,10052
+bbot/modules/gitlab_com.py,sha256=WBNGw4ec-xd_Iz8yxJcxEgTOpsBPxfn5pDU1DtONFgs,1051
+bbot/modules/gitlab_onprem.py,sha256=OwbYeldAUCQvFiYAIikX1-waHii1F0cMPLAtqc4pyHs,3622
 bbot/modules/google_playstore.py,sha256=N4QjzQag_bgDXfX17rytBiiWA-SQtYI2N0J_ZNEOdv0,3701
 bbot/modules/gowitness.py,sha256=hMhCz4O1sDJCzCzRIcmu0uNDgDDf9JzkFBwL1WuUum0,13144
 bbot/modules/graphql_introspection.py,sha256=Y-MqXrN6qmXTv2T6t7hJ-SU3R2guZQRWkrrCLC56bAc,4239
@@ -143,7 +144,7 @@ bbot/modules/lightfuzz/submodules/serial.py,sha256=Vry3J0Bs3QJqgVPzxhDFmEZQt4FYz
 bbot/modules/lightfuzz/submodules/sqli.py,sha256=HX0wP-aVn02zzBDujpLgzXPos7w_eiSiALTNCN2O_Bo,8597
 bbot/modules/lightfuzz/submodules/ssti.py,sha256=Pib49rXFuf567msnlec-A1Tnvolw4aILjqn7INLWQTY,1413
 bbot/modules/lightfuzz/submodules/xss.py,sha256=BZz1_nqzV8dqJptpoqZEMdVBdtZHmRae3HWo3S9yzIc,9507
-bbot/modules/medusa.py,sha256=44psluRg9VFo6WNLKlnTtH66afqhrOF0STBbLhFkHCE,8859
+bbot/modules/medusa.py,sha256=K_U_s2cZJi8UvzMB8qN61qj_x0uD16xdZUpC5ejUaGA,8729
 bbot/modules/myssl.py,sha256=DoMF7o6MxIrcglCrC-W3nM-GPcyJRM4PlGdKfnOlIvs,942
 bbot/modules/newsletters.py,sha256=1Q4JjShPsxHJ-by2CbGfCvEt80blUGPX0hxQIzB_a9M,2630
 bbot/modules/ntlm.py,sha256=EGmb4k3YC_ZuHIU3mGUZ4yaMjE35wVQQSv8HwTsQJzY,4391
@@ -175,13 +176,13 @@ bbot/modules/output/web_report.py,sha256=lZ0FqRZ7Jz1lljI9JMhH9gjtWLaTCSpSnAKQGAc
 bbot/modules/output/websocket.py,sha256=oxMcYu3hIrA3BE3c3aJXe63JmGv_HWjADB2uO470-BE,2721
 bbot/modules/paramminer_cookies.py,sha256=q1PzftHQpCHLz81_VgLZsO6moia7ZtnU32igfcySi2w,1816
 bbot/modules/paramminer_getparams.py,sha256=_j6rgaqV5wGJoa8p5-KKbe2YsVGUtmWIanCVtFiF97Y,1893
-bbot/modules/paramminer_headers.py,sha256=GMErLmTO0w7JRIpJE2VFvRTrjmoux_-jTx3EfaWLdpM,10518
+bbot/modules/paramminer_headers.py,sha256=W9fpSwFmrhiWt6wIrhzeksSbzDFzjhg6ncyEzucCm04,10596
 bbot/modules/passivetotal.py,sha256=dHYk9QWIKdO6Z8Bip4IdcButiqy_fr4FrpRUQaiH1a0,1678
 bbot/modules/pgp.py,sha256=Xu2M9WEIlwTm5-Lv29g7BblI05tD9Dl0XsYSeY6UURs,2065
 bbot/modules/portfilter.py,sha256=3iu4xqCsHafhVMbA32Mw6K_7Yn576Rz6GxXMevZQEpM,1752
 bbot/modules/portscan.py,sha256=emhNhnFYBVMnVm7IZKmzHJRCKRwVpF3S9dtOV6H9iYA,13091
 bbot/modules/postman.py,sha256=vo761Nzu3kPBzfCY3KJcvsGEsjImaa7iA2z-LyASBDc,4589
-bbot/modules/postman_download.py,sha256=OXC4hHInwD2Ps-BURc2uM_Z7PCl95KPdcxSI34Vzdcc,3604
+bbot/modules/postman_download.py,sha256=nR7Zpy8DsF1xwPskWysr4i9Bhn0JQq9E__A-UcjjDM8,3616
 bbot/modules/rapiddns.py,sha256=uONESr0B5pv9cSAr7lF4WWV31APUhXyHexvI04rUcyk,787
 bbot/modules/reflected_parameters.py,sha256=RjS-4C-XC9U-jC9J7AYNqwn6I-O2y3LvTRhB68dpgKI,3281
 bbot/modules/report/affiliates.py,sha256=vvus8LylqOfP-lfGid0z4FS6MwOpNuRTcSJ9aSnybp4,1713
@@ -203,13 +204,14 @@ bbot/modules/subdomainradar.py,sha256=YlRNMtNGLpa13KZ7aksAMVZdSjxe1tkywU5RXlwXpP
 bbot/modules/telerik.py,sha256=kWi498zihl02gHaS7AvyAxlEAZvmfKgKMSTAG8CS62A,19108
 bbot/modules/templates/bucket.py,sha256=muLPpfAGtcNhL0tLU-qHTlTNIz4yncRcVjdZMqVRtUI,7153
 bbot/modules/templates/github.py,sha256=lrV1EYPqjtPkJsS0fQfqmLvGchNo_fO3A75W9-03gxY,2531
+bbot/modules/templates/gitlab.py,sha256=XOwCaYO77ISbVPnjzws2M1klueTnJbXRef-ZsHUtwvA,3895
 bbot/modules/templates/postman.py,sha256=MIpz2q_r6LP0kIEgByo7oX5qHhMZLOhr7oKzJI9Beec,6959
 bbot/modules/templates/shodan.py,sha256=MXBvlmfw3jZFqT47v10UkqMSyQR-zBIxMJmK7PWw6uw,1174
 bbot/modules/templates/sql.py,sha256=o-CdyyoJvHJdJBKkj3CIGXYxUta4w2AB_2Vr-k7cDDU,3553
 bbot/modules/templates/subdomain_enum.py,sha256=epyKSly08jqaINV_AMMWbNafIeQjJqvd3aj63KD0Mck,8402
 bbot/modules/templates/webhook.py,sha256=uGFmcJ81GzGN1UI2k2O7nQF_fyh4ehLDEg2NSXaPnhk,3373
 bbot/modules/trickest.py,sha256=MRgLW0YiDWzlWdAjyqfPPLFb-a51r-Ffn_dphiJI_gA,1550
-bbot/modules/trufflehog.py,sha256=uW8smAJUwU5PM6jnIE_wlsz-EwgaJD-iaHmFciX59H8,8717
+bbot/modules/trufflehog.py,sha256=aMHaruUX6ZdPZ-SI8iIHBAsfEqIXz3GqZtynaFeEWnc,8770
 bbot/modules/url_manipulation.py,sha256=4J3oFkqTSJPPmbKEKAHJg2Q2w4QNKtQhiN03ZJq5VtI,4326
 bbot/modules/urlscan.py,sha256=-w_3Bm6smyG2GLQyIbnMUkKmeQVauo-V6F4_kJDYG7s,3740
 bbot/modules/vhost.py,sha256=cirOe0HR4M0TEBN8JdXo2l0s2flc8ZSdxggGm79blT8,5459
@@ -255,7 +257,7 @@ bbot/scanner/preset/conditions.py,sha256=hFL9cSIWGEsv2TfM5UGurf0c91cyaM8egb5IngB
 bbot/scanner/preset/environ.py,sha256=9KbEOLWkUdoAf5Ez_2A1NNm6QduQElbnNnrPi6VDhZs,4731
 bbot/scanner/preset/path.py,sha256=X32-ZUmL7taIv37VKF1KfmeiK9fjuQOE7pWUTEbPK8c,2483
 bbot/scanner/preset/preset.py,sha256=G_aMMI33d2OlzNUwjfi5ddJdxa8nK0oF5HrYAsuregU,40708
-bbot/scanner/scanner.py,sha256=Zz_syqeBpzc0Df1RH5HizylM2IPbFe7ZKd6e9quHWY4,55578
+bbot/scanner/scanner.py,sha256=RoVq12XXGNL4I7zu9uqBk0UG70xUpjuaYsrbALEHlho,55614
 bbot/scanner/stats.py,sha256=re93sArKXZSiD0Owgqk2J3Kdvfm3RL4Y9Qy_VOcaVk8,3623
 bbot/scanner/target.py,sha256=lI0Tn5prQiPiJE3WW-ZLx_l6EFqzAVabtyL-nfXJ8cE,10636
 bbot/scripts/benchmark_report.py,sha256=kBMaKFrmURk8tFYbGKvkGQpfNxrX46CBWG9DdaXhHgs,16230
@@ -293,7 +295,7 @@ bbot/test/test_step_1/test_files.py,sha256=5Q_3jPpMXULxDHsanSDUaj8zF8bXzKdiJZHOm
 bbot/test/test_step_1/test_helpers.py,sha256=7GP6-95yWRBGhx0-p6N7zZVEDcF9EO9wc0rkhs1JDsg,40281
 bbot/test/test_step_1/test_manager_deduplication.py,sha256=hZQpDXzg6zvzxFolVOcJuY-ME8NXjZUsqS70BRNXp8A,15594
 bbot/test/test_step_1/test_manager_scope_accuracy.py,sha256=JV1bQHt9EIM0GmGS4T4Brz_L2lfcwTxtNC06cfv7r64,79763
-bbot/test/test_step_1/test_modules_basic.py,sha256=ELpGlsthSq8HaxB5My8-ESVHqMxqdL5Of0STMIyaWzA,20001
+bbot/test/test_step_1/test_modules_basic.py,sha256=1A5saKsZ254cgdv5o9pk7iU5g6kxmcCeLfuLiYIsihs,20640
 bbot/test/test_step_1/test_presets.py,sha256=HnJhKwDnVh9Y6adgxqe85677rWpnFil_WS5GjX21ZvM,40959
 bbot/test/test_step_1/test_python_api.py,sha256=Fk5bxEsPSjsMZ_CcRMTJft8I48EizwHJivG9Fy4jIu0,5502
 bbot/test/test_step_1/test_regexes.py,sha256=GEJE4NY6ge0WnG3BcFgRiT78ksy2xpFk6UdS9vGQMPs,15254
@@ -369,7 +371,8 @@ bbot/test/test_step_2/module_tests/test_module_github_codesearch.py,sha256=M50xB
 bbot/test/test_step_2/module_tests/test_module_github_org.py,sha256=5tKO6NH4TPBeIdeTf7Bz9PUZ1pcvKsjrG0nFhc3YgT0,25458
 bbot/test/test_step_2/module_tests/test_module_github_usersearch.py,sha256=IIQ0tYZjQN8_L8u_N4m8Nz3kbB4IyBp95tYCPcQeScg,5264
 bbot/test/test_step_2/module_tests/test_module_github_workflows.py,sha256=o_teEaskm3H22QEKod5KJayFvvcgOQoG4eItGWv8C8E,38006
-bbot/test/test_step_2/module_tests/test_module_gitlab.py,sha256=fnwE7BWTU6EQquKdGLCiaX_LwVwvzOLev3Y9GheTLSY,11859
+bbot/test/test_step_2/module_tests/test_module_gitlab_com.py,sha256=fGnjYyMvMZE2hu0Fms9H8rMnPPN6_uynDDDEmcVE9-8,2753
+bbot/test/test_step_2/module_tests/test_module_gitlab_onprem.py,sha256=Soo72Ppt5hYWVUIxMYGnBGPL47EnVDPbTsEHUziKimg,9173
 bbot/test/test_step_2/module_tests/test_module_google_playstore.py,sha256=uTRqpAGI9HI-rOk_6jdV44OoSqi0QQQ3aTVzvuV0dtc,3034
 bbot/test/test_step_2/module_tests/test_module_gowitness.py,sha256=8kSeBowX4eejMW791mIaFqP9SDn1l2EDRJatvmZVWug,6500
 bbot/test/test_step_2/module_tests/test_module_graphql_introspection.py,sha256=qac8DJ_exe6Ra4UgRvVMSdgBhLIZP9lmXyKhi9RPOK8,1241
@@ -460,8 +463,8 @@ bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt,sha256=ZSIVebs7ptMvHx
 bbot/wordlists/top_open_ports_nmap.txt,sha256=LmdFYkfapSxn1pVuQC2LkOIY2hMLgG-Xts7DVtYzweM,42727
 bbot/wordlists/valid_url_schemes.txt,sha256=0B_VAr9Dv7aYhwi6JSBDU-3M76vNtzN0qEC_RNLo7HE,3310
 bbot/wordlists/wordninja_dns.txt.gz,sha256=DYHvvfW0TvzrVwyprqODAk4tGOxv5ezNmCPSdPuDUnQ,570241
-bbot-2.7.1.7212rc0.dist-info/METADATA,sha256=7rDrwHdnOGfp7sZS3vDHbnlFYUnX_d7gwDpya7NkQY4,18420
-bbot-2.7.1.7212rc0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-bbot-2.7.1.7212rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
-bbot-2.7.1.7212rc0.dist-info/licenses/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
-bbot-2.7.1.7212rc0.dist-info/RECORD,,
+bbot-2.7.2.7271rc0.dist-info/METADATA,sha256=_oIi9ERuKbl0buhGO6vJ8BJku9jGX5p0XII95-JjNOg,18420
+bbot-2.7.2.7271rc0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+bbot-2.7.2.7271rc0.dist-info/entry_points.txt,sha256=cWjvcU_lLrzzJgjcjF7yeGuRA_eDS8pQ-kmPUAyOBfo,38
+bbot-2.7.2.7271rc0.dist-info/licenses/LICENSE,sha256=GzeCzK17hhQQDNow0_r0L8OfLpeTKQjFQwBQU7ZUymg,32473
+bbot-2.7.2.7271rc0.dist-info/RECORD,,

bbot/modules/gitlab.py

@@ -1,141 +0,0 @@
-from bbot.modules.base import BaseModule
-
-
-class gitlab(BaseModule):
-    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
-    produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"]
-    flags = ["active", "safe", "code-enum"]
-    meta = {
-        "description": "Detect GitLab instances and query them for repositories",
-        "created_date": "2024-03-11",
-        "author": "@TheTechromancer",
-    }
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Gitlab access token"}
-
-    scope_distance_modifier = 2
-
-    async def setup(self):
-        await self.require_api_key()
-        return True
-
-    async def filter_event(self, event):
-        # only accept out-of-scope SOCIAL events
-        if event.type == "HTTP_RESPONSE":
-            if event.scope_distance > self.scan.scope_search_distance:
-                return False, "event is out of scope distance"
-        elif event.type == "TECHNOLOGY":
-            if not event.data["technology"].lower().startswith("gitlab"):
-                return False, "technology is not gitlab"
-            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
-                return False, "gitlab instance is not self-hosted"
-        elif event.type == "SOCIAL":
-            if event.data["platform"] != "gitlab":
-                return False, "platform is not gitlab"
-        return True
-
-    async def handle_event(self, event):
-        if event.type == "HTTP_RESPONSE":
-            await self.handle_http_response(event)
-        elif event.type == "TECHNOLOGY":
-            await self.handle_technology(event)
-        elif event.type == "SOCIAL":
-            await self.handle_social(event)
-
-    async def handle_http_response(self, event):
-        # identify gitlab instances from HTTP responses
-        # HTTP_RESPONSE --> TECHNOLOGY
-        # HTTP_RESPONSE --> FINDING
-        headers = event.data.get("header", {})
-        if "x_gitlab_meta" in headers:
-            url = event.parsed_url._replace(path="/").geturl()
-            await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url},
-                "TECHNOLOGY",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
-            )
-            description = f"GitLab server at {event.host}"
-            await self.emit_event(
-                {"host": str(event.host), "description": description},
-                "FINDING",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: {description}",
-            )
-
-    async def handle_technology(self, event):
-        # retrieve gitlab groups from gitlab instances
-        # TECHNOLOGY --> SOCIAL
-        # TECHNOLOGY --> URL
-        # TECHNOLOGY --> CODE_REPOSITORY
-        base_url = self.get_base_url(event)
-        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
-        await self.handle_projects_url(projects_url, event)
-        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
-        await self.handle_groups_url(groups_url, event)
-
-    async def handle_social(self, event):
-        # retrieve repositories from gitlab user
-        # SOCIAL --> CODE_REPOSITORY
-        # SOCIAL --> SOCIAL
-        username = event.data.get("profile_name", "")
-        if not username:
-            return
-        base_url = self.get_base_url(event)
-        urls = [
-            # group
-            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
-            # user
-            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
-        ]
-        for url in urls:
-            await self.handle_projects_url(url, event)
-
-    async def handle_projects_url(self, projects_url, event):
-        for project in await self.gitlab_json_request(projects_url):
-            project_url = project.get("web_url", "")
-            if project_url:
-                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
-                await self.emit_event(
-                    code_event, context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}"
-                )
-            namespace = project.get("namespace", {})
-            if namespace:
-                await self.handle_namespace(namespace, event)
-
-    async def handle_groups_url(self, groups_url, event):
-        for group in await self.gitlab_json_request(groups_url):
-            await self.handle_namespace(group, event)
-
-    async def gitlab_json_request(self, url):
-        response = await self.api_request(url)
-        if response is not None:
-            try:
-                json = response.json()
-            except Exception:
-                return []
-            if json and isinstance(json, list):
-                return json
-        return []
-
-    async def handle_namespace(self, namespace, event):
-        namespace_name = namespace.get("path", "")
-        namespace_url = namespace.get("web_url", "")
-        namespace_path = namespace.get("full_path", "")
-        if namespace_name and namespace_url and namespace_path:
-            namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
-            social_event = self.make_event(
-                {"platform": "gitlab", "profile_name": namespace_path, "url": namespace_url},
-                "SOCIAL",
-                parent=event,
-            )
-            await self.emit_event(
-                social_event,
-                context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
-            )
-
-    def get_base_url(self, event):
-        base_url = event.data.get("url", "")
-        if not base_url:
-            base_url = f"https://{event.host}"
-        return self.helpers.urlparse(base_url)._replace(path="/").geturl()