bbot 2.6.0.6840rc0__py3-none-any.whl → 2.7.2.7424rc0__py3-none-any.whl

bbot/modules/dnsbrute.py

@@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum):
     dedup_strategy = "lowest_parent"
     _qsize = 10000
 
+    async def setup_deps(self):
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        # tell the dnsbrute helper to fetch the resolver file
+        await self.helpers.dns.brute.resolver_file()
+        return True
+
     async def setup(self):
         self.max_depth = max(1, self.config.get("max_depth", 5))
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
         self.wordlist_size = len(self.subdomain_list)
         return await super().setup()

bbot/modules/dnsdumpster.py

@@ -1,4 +1,4 @@
-import re
+import json
 
 from bbot.modules.templates.subdomain_enum import subdomain_enum
 
@@ -15,78 +15,61 @@ class dnsdumpster(subdomain_enum):
 
     base_url = "https://dnsdumpster.com"
 
+    async def setup(self):
+        self.apikey_regex = self.helpers.re.compile(r'<form[^>]*data-form-id="mainform"[^>]*hx-headers=\'([^\']*)\'')
+        return True
+
     async def query(self, domain):
         ret = []
-        # first, get the CSRF tokens
+        # first, get the JWT token from the main page
         res1 = await self.api_request(self.base_url)
         status_code = getattr(res1, "status_code", 0)
-        if status_code in [429]:
-            self.verbose(f'Too many requests "{status_code}"')
-            return ret
-        elif status_code not in [200]:
+        if status_code not in [200]:
             self.verbose(f'Bad response code "{status_code}" from DNSDumpster')
             return ret
-        else:
-            self.debug(f'Valid response code "{status_code}" from DNSDumpster')
-
-        html = self.helpers.beautifulsoup(res1.content, "html.parser")
-        if html is False:
-            self.verbose("BeautifulSoup returned False")
-            return ret
 
-        csrftoken = None
-        csrfmiddlewaretoken = None
+        # Extract JWT token from the form's hx-headers attribute using regex
+        jwt_token = None
         try:
-            for cookie in res1.headers.get("set-cookie", "").split(";"):
-                try:
-                    k, v = cookie.split("=", 1)
-                except ValueError:
-                    self.verbose("Error retrieving cookie")
-                    return ret
-                if k == "csrftoken":
-                    csrftoken = str(v)
-            csrfmiddlewaretoken = html.find("input", {"name": "csrfmiddlewaretoken"}).attrs.get("value", None)
-        except AttributeError:
-            pass
+            # Look for the form with data-form-id="mainform" and extract hx-headers
+            form_match = await self.helpers.re.search(self.apikey_regex, res1.text)
+            if form_match:
+                headers_json = form_match.group(1)
+                headers_data = json.loads(headers_json)
+                jwt_token = headers_data.get("Authorization")
+        except (AttributeError, json.JSONDecodeError, KeyError):
+            self.log.warning("Error obtaining JWT token")
+            return ret
 
-        # Abort if we didn't get the tokens
-        if not csrftoken or not csrfmiddlewaretoken:
-            self.verbose("Error obtaining CSRF tokens")
+        # Abort if we didn't get the JWT token
+        if not jwt_token:
+            self.verbose("Error obtaining JWT token")
             self.errorState = True
             return ret
         else:
-            self.debug("Successfully obtained CSRF tokens")
+            self.debug("Successfully obtained JWT token")
 
         if self.scan.stopping:
-            return
+            return ret
 
-        # Otherwise, do the needful
-        subdomains = set()
+        # Query the API with the JWT token
         res2 = await self.api_request(
-            f"{self.base_url}/",
+            "https://api.dnsdumpster.com/htmld/",
             method="POST",
-            cookies={"csrftoken": csrftoken},
-            data={
-                "csrfmiddlewaretoken": csrfmiddlewaretoken,
-                "targetip": str(domain).lower(),
-                "user": "free",
-            },
+            data={"target": str(domain).lower()},
             headers={
-                "origin": "https://dnsdumpster.com",
-                "referer": "https://dnsdumpster.com/",
+                "Authorization": jwt_token,
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Origin": "https://dnsdumpster.com",
+                "Referer": "https://dnsdumpster.com/",
+                "HX-Request": "true",
+                "HX-Target": "results",
+                "HX-Current-URL": "https://dnsdumpster.com/",
             },
         )
         status_code = getattr(res2, "status_code", 0)
         if status_code not in [200]:
-            self.verbose(f'Bad response code "{status_code}" from DNSDumpster')
-            return ret
-        html = self.helpers.beautifulsoup(res2.content, "html.parser")
-        if html is False:
-            self.verbose("BeautifulSoup returned False")
+            self.verbose(f'Bad response code "{status_code}" from DNSDumpster API')
             return ret
-        escaped_domain = re.escape(domain)
-        match_pattern = re.compile(r"^[\w\.-]+\." + escaped_domain + r"$")
-        for subdomain in html.findAll(text=match_pattern):
-            subdomains.add(str(subdomain).strip().lower())
 
-        return list(subdomains)
+        return await self.scan.extract_in_scope_hostnames(res2.text)

bbot/modules/dnstlsrpt.py

@@ -44,20 +44,17 @@ class dnstlsrpt(BaseModule):
         "emit_emails": True,
         "emit_raw_dns_records": False,
         "emit_urls": True,
-        "emit_vulnerabilities": True,
     }
     options_desc = {
         "emit_emails": "Emit EMAIL_ADDRESS events",
         "emit_raw_dns_records": "Emit RAW_DNS_RECORD events",
         "emit_urls": "Emit URL_UNVERIFIED events",
-        "emit_vulnerabilities": "Emit VULNERABILITY events",
     }
 
     async def setup(self):
         self.emit_emails = self.config.get("emit_emails", True)
         self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False)
         self.emit_urls = self.config.get("emit_urls", True)
-        self.emit_vulnerabilities = self.config.get("emit_vulnerabilities", True)
         return await super().setup()
 
     def _incoming_dedup_hash(self, event):
@@ -139,6 +136,3 @@ class dnstlsrpt(BaseModule):
                                                     tags=tags.append(f"tlsrpt-record-{key}"),
                                                     parent=event,
                                                 )
-
-
-# EOF

bbot/modules/docker_pull.py

@@ -8,7 +8,7 @@ from bbot.modules.base import BaseModule
 class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",

bbot/modules/emailformat.py

@@ -15,13 +15,29 @@ class emailformat(BaseModule):
 
     base_url = "https://www.email-format.com"
 
+    async def setup(self):
+        self.cfemail_regex = self.helpers.re.compile(r'data-cfemail="([0-9a-z]+)"')
+        return True
+
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
         url = f"{self.base_url}/d/{self.helpers.quote(query)}/"
         r = await self.api_request(url)
         if not r:
             return
-        for email in await self.helpers.re.extract_emails(r.text):
+
+        encrypted_emails = await self.helpers.re.findall(self.cfemail_regex, r.text)
+
+        for enc in encrypted_emails:
+            enc_len = len(enc)
+
+            if enc_len < 2 or enc_len % 2 != 0:
+                continue
+
+            key = int(enc[:2], 16)
+
+            email = "".join([chr(int(enc[i : i + 2], 16) ^ key) for i in range(2, enc_len, 2)]).lower()
+
             if email.endswith(query):
                 await self.emit_event(
                     email,

bbot/modules/ffuf.py

@@ -37,12 +37,15 @@ class ffuf(BaseModule):
 
     in_scope_only = True
 
+    async def setup_deps(self):
+        self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
+        return True
+
     async def setup(self):
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         wordlist_url = self.config.get("wordlist", "")
         self.debug(f"Using wordlist [{wordlist_url}]")
-        self.wordlist = await self.helpers.wordlist(wordlist_url)
         self.wordlist_lines = self.generate_wordlist(self.wordlist)
         self.tempfile, tempfile_len = self.generate_templist()
         self.rate = self.config.get("rate", 0)

bbot/modules/ffuf_shortnames.py

@@ -87,14 +87,17 @@ class ffuf_shortnames(ffuf):
                     found_prefixes.add(prefix)
         return list(found_prefixes)
 
-    async def setup(self):
-        self.proxy = self.scan.web_config.get("http_proxy", "")
-        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
+    async def setup_deps(self):
         wordlist_extensions = self.config.get("wordlist_extensions", "")
         if not wordlist_extensions:
             wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
         self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
+        return True
+
+    async def setup(self):
+        self.proxy = self.scan.web_config.get("http_proxy", "")
+        self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
         self.ignore_redirects = self.config.get("ignore_redirects")
         self.max_predictions = self.config.get("max_predictions")
         self.find_subwords = self.config.get("find_subwords")

bbot/modules/filedownload.py

@@ -14,7 +14,7 @@ class filedownload(BaseModule):
 
     watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"]
     produced_events = ["FILESYSTEM"]
-    flags = ["active", "safe", "web-basic"]
+    flags = ["active", "safe", "web-basic", "download"]
     meta = {
         "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
         "created_date": "2023-10-11",
@@ -94,6 +94,12 @@ class filedownload(BaseModule):
 
     scope_distance_modifier = 3
 
+    async def setup_deps(self):
+        self.mime_db_file = await self.helpers.wordlist(
+            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
+        )
+        return True
+
     async def setup(self):
         self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
         self.max_filesize = self.config.get("max_filesize", "10MB")
@@ -105,9 +111,6 @@ class filedownload(BaseModule):
         else:
             self.download_dir = self.scan.temp_dir / "filedownload"
         self.helpers.mkdir(self.download_dir)
-        self.mime_db_file = await self.helpers.wordlist(
-            "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json"
-        )
         self.mime_db = {}
         with open(self.mime_db_file) as f:
             mime_db = json.load(f)

bbot/modules/git_clone.py

@@ -6,7 +6,7 @@ from bbot.modules.templates.github import github
 class git_clone(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Clone code github repositories",
         "created_date": "2024-03-08",
@@ -24,44 +24,69 @@ class git_clone(github):
 
     async def setup(self):
         output_folder = self.config.get("output_folder")
-        if output_folder:
-            self.output_dir = Path(output_folder) / "git_repos"
-        else:
-            self.output_dir = self.scan.temp_dir / "git_repos"
+        self.output_dir = Path(output_folder) / "git_repos" if output_folder else self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
 
     async def filter_event(self, event):
-        if event.type == "CODE_REPOSITORY":
-            if "git" not in event.tags:
-                return False, "event is not a git repository"
+        if event.type == "CODE_REPOSITORY" and "git" not in event.tags:
+            return False, "event is not a git repository"
         return True
 
     async def handle_event(self, event):
-        repo_url = event.data.get("url")
-        repo_path = await self.clone_git_repository(repo_url)
-        if repo_path:
-            self.verbose(f"Cloned {repo_url} to {repo_path}")
-            codebase_event = self.make_event({"path": str(repo_path)}, "FILESYSTEM", tags=["git"], parent=event)
+        repository_url = event.data.get("url")
+        repository_path = await self.clone_git_repository(repository_url)
+        if repository_path:
+            self.verbose(f"Cloned {repository_url} to {repository_path}")
+            codebase_event = self.make_event({"path": str(repository_path)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
                 codebase_event,
-                context=f"{{module}} downloaded git repo at {repo_url} to {{event.type}}: {repo_path}",
+                context=f"{{module}} cloned git repository at {repository_url} to {{event.type}}: {repository_path}",
             )
 
     async def clone_git_repository(self, repository_url):
         owner = repository_url.split("/")[-2]
         folder = self.output_dir / owner
         self.helpers.mkdir(folder)
-        if self.api_key:
-            url = repository_url.replace("https://github.com", f"https://user:{self.api_key}@github.com")
-        else:
-            url = repository_url
-        command = ["git", "-C", folder, "clone", url]
+
+        command = ["git", "-C", folder, "clone", repository_url]
+        env = {"GIT_TERMINAL_PROMPT": "0"}
+
         try:
-            output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, check=True)
+            hostname = self.helpers.urlparse(repository_url).hostname
+            if hostname and self.api_key:
+                _, domain = self.helpers.split_domain(hostname)
+                # only use the api key if the domain is github.com
+                if domain == "github.com":
+                    env["GIT_HELPER"] = (
+                        f'!f() {{ case "$1" in get) '
+                        f"echo username=x-access-token; "
+                        f"echo password={self.api_key};; "
+                        f'esac; }}; f "$@"'
+                    )
+                    command = (
+                        command[:1]
+                        + [
+                            "-c",
+                            "credential.helper=",
+                            "-c",
+                            "credential.useHttpPath=true",
+                            "--config-env=credential.helper=GIT_HELPER",
+                        ]
+                        + command[1:]
+                    )
+
+            output = await self.run_process(command, env=env, check=True)
         except CalledProcessError as e:
-            self.debug(f"Error cloning {url}. STDERR: {repr(e.stderr)}")
+            self.debug(f"Error cloning {repository_url}. STDERR: {repr(e.stderr)}")
             return
 
         folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
-        return folder / folder_name
+        repo_folder = folder / folder_name
+
+        # sanitize the repo
+        # this moves the git config, index file, and hooks folder out of the .git folder to prevent nasty things
+        # Note: the index file can be regenerated by running "git checkout HEAD -- ."
+        self.helpers.sanitize_git_repo(repo_folder)
+
+        return repo_folder

bbot/modules/gitdumper.py

@@ -1,5 +1,4 @@
 import asyncio
-import regex as re
 from pathlib import Path
 from subprocess import CalledProcessError
 from bbot.modules.base import BaseModule
@@ -8,7 +7,7 @@ from bbot.modules.base import BaseModule
 class gitdumper(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "slow", "code-enum"]
+    flags = ["passive", "safe", "slow", "code-enum", "download"]
     meta = {
         "description": "Download a leaked .git folder recursively or by fuzzing common names",
         "created_date": "2025-02-11",
@@ -35,7 +34,6 @@ class gitdumper(BaseModule):
         else:
             self.output_dir = self.scan.temp_dir / "git_repos"
         self.helpers.mkdir(self.output_dir)
-        self.unsafe_regex = self.helpers.re.compile(r"^\s*fsmonitor|sshcommand|askpass|editor|pager", re.IGNORECASE)
         self.ref_regex = self.helpers.re.compile(r"ref: refs/heads/([a-zA-Z\d_-]+)")
         self.obj_regex = self.helpers.re.compile(r"[a-f0-9]{40}")
         self.pack_regex = self.helpers.re.compile(r"pack-([a-f0-9]{40})\.pack")
@@ -131,7 +129,6 @@ class gitdumper(BaseModule):
         else:
             result = await self.git_fuzz(repo_url, repo_folder)
         if result:
-            await self.sanitize_config(repo_folder)
             await self.git_checkout(repo_folder)
             codebase_event = self.make_event({"path": str(repo_folder)}, "FILESYSTEM", tags=["git"], parent=event)
             await self.emit_event(
@@ -251,15 +248,6 @@ class gitdumper(BaseModule):
             self.debug(f"Unable to download git files to {folder}")
             return False
 
-    async def sanitize_config(self, folder):
-        config_file = folder / ".git/config"
-        if config_file.exists():
-            with config_file.open("r", encoding="utf-8", errors="ignore") as file:
-                content = file.read()
-                sanitized = await self.helpers.re.sub(self.unsafe_regex, r"# \g<0>", content)
-            with config_file.open("w", encoding="utf-8") as file:
-                file.write(sanitized)
-
     async def git_catfile(self, hash, option="-t", folder=Path()):
         command = ["git", "cat-file", option, hash]
         try:
@@ -270,8 +258,10 @@ class gitdumper(BaseModule):
         return output.stdout
 
     async def git_checkout(self, folder):
+        self.helpers.sanitize_git_repo(folder)
         self.verbose(f"Running git checkout to reconstruct the git repository at {folder}")
-        command = ["git", "checkout", "."]
+        # we do "checkout head -- ." because the sanitization deletes the index file, and it needs to be reconstructed
+        command = ["git", "checkout", "HEAD", "--", "."]
         try:
             await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, cwd=folder, check=True)
         except CalledProcessError as e:

bbot/modules/github_workflows.py

@@ -8,11 +8,12 @@ from bbot.modules.templates.github import github
 class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
-    flags = ["passive", "safe", "code-enum"]
+    flags = ["passive", "safe", "code-enum", "download"]
     meta = {
         "description": "Download a github repositories workflow logs and workflow artifacts",
         "created_date": "2024-04-29",
         "author": "@domwhewell-sage",
+        "auth_required": True,
     }
     options = {"api_key": "", "num_logs": 1, "output_folder": ""}
     options_desc = {
@@ -152,7 +153,7 @@ class github_workflows(github):
         filename = f"run_{run_id}.zip"
         file_destination = folder / filename
         try:
-            await self.helpers.download(
+            await self.api_download(
                 f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs",
                 filename=file_destination,
                 headers=self.headers,
@@ -166,7 +167,7 @@ class github_workflows(github):
             status_code = getattr(response, "status_code", 0)
             if status_code == 403:
                 self.warning(
-                    f"The current access key does not have access to workflow {owner}/{repo}/{run_id} (status: {status_code})"
+                    f"The current access key does not have access to workflow {owner}/{repo}/{run_id}, The API key must have the 'repo' scope or read 'Actions' repository permissions (status: {status_code})"
                 )
             else:
                 self.info(
@@ -212,7 +213,7 @@ class github_workflows(github):
         self.helpers.mkdir(folder)
         file_destination = folder / artifact_name
         try:
-            await self.helpers.download(
+            await self.api_download(
                 f"{self.base_url}/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip",
                 filename=file_destination,
                 headers=self.headers,
@@ -228,6 +229,6 @@ class github_workflows(github):
             status_code = getattr(response, "status_code", 0)
             if status_code == 403:
                 self.warning(
-                    f"The current access key does not have access to workflow artifacts {owner}/{repo}/{artifact_id} (status: {status_code})"
+                    f"The current access key does not have access to workflow artifacts {owner}/{repo}/{artifact_id}, The API key must have the 'repo' scope or read 'Actions' repository permissions (status: {status_code})"
                 )
         return file_destination

bbot/modules/gitlab_com.py

@@ -0,0 +1,31 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_com(GitLabBaseModule):
+    watched_events = ["SOCIAL"]
+    produced_events = [
+        "CODE_REPOSITORY",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Enumerate GitLab SaaS (gitlab.com/org) for projects and groups",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"}
+
+    # This is needed because we are consuming SOCIAL events, which aren't in scope
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        await self.handle_social(event)
+
+    async def filter_event(self, event):
+        if event.data["platform"] != "gitlab":
+            return False, "platform is not gitlab"
+        _, domain = self.helpers.split_domain(event.host)
+        if domain not in self.saas_domains:
+            return False, "gitlab instance is not gitlab.com/org"
+        return True

bbot/modules/gitlab_onprem.py

@@ -0,0 +1,84 @@
+from bbot.modules.templates.gitlab import GitLabBaseModule
+
+
+class gitlab_onprem(GitLabBaseModule):
+    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
+    produced_events = [
+        "TECHNOLOGY",
+        "SOCIAL",
+        "CODE_REPOSITORY",
+        "FINDING",
+    ]
+    flags = ["active", "safe", "code-enum"]
+    meta = {
+        "description": "Detect self-hosted GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer",
+    }
+
+    # Optional GitLab access token (only required for gitlab.com, but still
+    # supported for on-prem installations that expose private projects).
+    options = {"api_key": ""}
+    options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"}
+
+    # Allow accepting events slightly beyond configured max distance so we can
+    # discover repos on neighbouring infrastructure.
+    scope_distance_modifier = 2
+
+    async def handle_event(self, event):
+        if event.type == "HTTP_RESPONSE":
+            await self.handle_http_response(event)
+        elif event.type == "TECHNOLOGY":
+            await self.handle_technology(event)
+        elif event.type == "SOCIAL":
+            await self.handle_social(event)
+
+    async def filter_event(self, event):
+        # only accept out-of-scope SOCIAL events
+        if event.type == "HTTP_RESPONSE":
+            if event.scope_distance > self.scan.scope_search_distance:
+                return False, "event is out of scope distance"
+        elif event.type == "TECHNOLOGY":
+            if not event.data["technology"].lower().startswith("gitlab"):
+                return False, "technology is not gitlab"
+            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
+                return False, "gitlab instance is not self-hosted"
+        elif event.type == "SOCIAL":
+            if event.data["platform"] != "gitlab":
+                return False, "platform is not gitlab"
+            _, domain = self.helpers.split_domain(event.host)
+            if domain in self.saas_domains:
+                return False, "gitlab instance is not self-hosted"
+        return True
+
+    async def handle_http_response(self, event):
+        """Identify GitLab servers from HTTP responses."""
+        headers = event.data.get("header", {})
+        if "x_gitlab_meta" in headers:
+            url = event.parsed_url._replace(path="/").geturl()
+            await self.emit_event(
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
+            )
+            description = f"GitLab server at {event.host}"
+            await self.emit_event(
+                {"host": str(event.host), "description": description},
+                "FINDING",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
+            )
+
+    async def handle_technology(self, event):
+        """Enumerate projects & groups once we know a host is GitLab."""
+        base_url = self.get_base_url(event)
+
+        # Projects owned by the authenticated user (or public projects if no
+        # authentication).
+        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
+        await self.handle_projects_url(projects_url, event)
+
+        # Group enumeration.
+        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
+        await self.handle_groups_url(groups_url, event)

bbot/modules/gowitness.py

@@ -161,7 +161,6 @@ class gowitness(BaseModule):
                 key = e.data["url"]
             event_dict[key] = e
         stdin = "\n".join(list(event_dict))
-        self.hugeinfo(f"Gowitness input: {stdin}")
 
         try:
             async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
@@ -182,7 +181,6 @@ class gowitness(BaseModule):
             # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
             filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
-            self.hugewarning(event_dict)
             parent_event = event_dict[url]
             await self.emit_event(
                 webscreenshot_data,
@@ -259,9 +257,7 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 con.text_factory = self.helpers.smart_decode
                 async with con.execute("SELECT * FROM results") as cur:
-                    self.critical(f"CUR: {cur}")
                     async for row in cur:
-                        self.critical(f"SCREENSHOT: {row}")
                         row = dict(row)
                         _id = row["id"]
                         if _id not in self.screenshots_taken:
@@ -276,7 +272,6 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM network_logs") as cur:
                     async for row in cur:
-                        self.critical(f"NETWORK LOG: {row}")
                         row = dict(row)
                         url = row["url"]
                         if url not in self.connections_logged:
@@ -291,7 +286,6 @@ class gowitness(BaseModule):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM technologies") as cur:
                     async for row in cur:
-                        self.critical(f"TECHNOLOGY: {row}")
                         _id = row["id"]
                         if _id not in self.technologies_found:
                             self.technologies_found.add(_id)