bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/modules/dnstlsrpt.py

@@ -0,0 +1,144 @@
+# dnstlsrpt.py
+#
+# Checks for and parses common TLS-RPT TXT records, e.g. _smtp._tls.target.domain
+#
+# TLS-RPT policies may contain email addresses or URL's for reporting destinations, typically the email addresses are software processed inboxes, but they may also be to individual humans or team inboxes.
+#
+# The domain portion of any email address or URL is also passively checked and added as appropriate, for additional inspection by other modules.
+#
+# Example records,
+# _smtp._tls.example.com TXT "v=TLSRPTv1;rua=https://tlsrpt.azurewebsites.net/report"
+# _smtp._tls.example.net TXT "v=TLSRPTv1; rua=mailto:sts-reports@example.net;"
+#
+# TODO: extract %{UNIQUE_ID}% from hosted services as ORG_STUB ?
+#   e.g. %{UNIQUE_ID}%@tlsrpt.hosted.service.provider is usually a tenant specific ID.
+#   e.g. tlsrpt@%{UNIQUE_ID}%.hosted.service.provider is usually a tenant specific ID.
+
+from bbot.modules.base import BaseModule
+from bbot.core.helpers.dns.helpers import service_record
+
+import re
+
+from bbot.core.helpers.regexes import email_regex, url_regexes
+
+_tlsrpt_regex = r"^v=(?P<v>TLSRPTv[0-9]+); *(?P<kvps>.*)$"
+tlsrpt_regex = re.compile(_tlsrpt_regex, re.I)
+
+_tlsrpt_kvp_regex = r"(?P<k>\w+)=(?P<v>[^;]+);*"
+tlsrpt_kvp_regex = re.compile(_tlsrpt_kvp_regex)
+
+_csul = r"(?P<uri>[^, ]+)"
+csul = re.compile(_csul)
+
+
+class dnstlsrpt(BaseModule):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["EMAIL_ADDRESS", "URL_UNVERIFIED", "RAW_DNS_RECORD"]
+    flags = ["subdomain-enum", "cloud-enum", "email-enum", "passive", "safe"]
+    meta = {
+        "description": "Check for TLS-RPT records",
+        "author": "@colin-stubbs",
+        "created_date": "2024-07-26",
+    }
+    options = {
+        "emit_emails": True,
+        "emit_raw_dns_records": False,
+        "emit_urls": True,
+        "emit_vulnerabilities": True,
+    }
+    options_desc = {
+        "emit_emails": "Emit EMAIL_ADDRESS events",
+        "emit_raw_dns_records": "Emit RAW_DNS_RECORD events",
+        "emit_urls": "Emit URL_UNVERIFIED events",
+        "emit_vulnerabilities": "Emit VULNERABILITY events",
+    }
+
+    async def setup(self):
+        self.emit_emails = self.config.get("emit_emails", True)
+        self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False)
+        self.emit_urls = self.config.get("emit_urls", True)
+        self.emit_vulnerabilities = self.config.get("emit_vulnerabilities", True)
+        return await super().setup()
+
+    def _incoming_dedup_hash(self, event):
+        # dedupe by parent
+        parent_domain = self.helpers.parent_domain(event.data)
+        return hash(parent_domain), "already processed parent domain"
+
+    async def filter_event(self, event):
+        if "_wildcard" in str(event.host).split("."):
+            return False, "event is wildcard"
+
+        # there's no value in inspecting service records
+        if service_record(event.host) is True:
+            return False, "service record detected"
+
+        return True
+
+    async def handle_event(self, event):
+        rdtype = "TXT"
+        tags = ["tlsrpt-record"]
+        hostname = f"_smtp._tls.{event.host}"
+
+        r = await self.helpers.resolve_raw(hostname, type=rdtype)
+
+        if r:
+            raw_results, errors = r
+            for answer in raw_results:
+                if self.emit_raw_dns_records:
+                    await self.emit_event(
+                        {"host": hostname, "type": rdtype, "answer": answer.to_text()},
+                        "RAW_DNS_RECORD",
+                        parent=event,
+                        tags=tags.append(f"{rdtype.lower()}-record"),
+                        context=f"{rdtype} lookup on {hostname} produced {{event.type}}",
+                    )
+
+                # we need to fix TXT data that may have been split across two different rdata's
+                # e.g. we will get a single string, but within that string we may have two parts such as:
+                # answer = '"part 1 that was really long" "part 2 that did not fit in part 1"'
+                # NOTE: the leading and trailing double quotes are essential as part of a raw DNS TXT record, or another record type that contains a free form text string as a component.
+                s = answer.to_text().strip('"').replace('" "', "")
+
+                # validate TLSRPT record, tag appropriately
+                tlsrpt_match = tlsrpt_regex.search(s)
+
+                if (
+                    tlsrpt_match
+                    and tlsrpt_match.group("v")
+                    and tlsrpt_match.group("kvps")
+                    and tlsrpt_match.group("kvps") != ""
+                ):
+                    for kvp_match in tlsrpt_kvp_regex.finditer(tlsrpt_match.group("kvps")):
+                        key = kvp_match.group("k").lower()
+
+                        if key == "rua":
+                            for csul_match in csul.finditer(kvp_match.group("v")):
+                                if csul_match.group("uri"):
+                                    for match in email_regex.finditer(csul_match.group("uri")):
+                                        start, end = match.span()
+                                        email = csul_match.group("uri")[start:end]
+
+                                        if self.emit_emails:
+                                            await self.emit_event(
+                                                email,
+                                                "EMAIL_ADDRESS",
+                                                tags=tags.append(f"tlsrpt-record-{key}"),
+                                                parent=event,
+                                            )
+
+                                    for url_regex in url_regexes:
+                                        for match in url_regex.finditer(csul_match.group("uri")):
+                                            start, end = match.span()
+                                            url = csul_match.group("uri")[start:end]
+
+                                            if self.emit_urls:
+                                                await self.emit_event(
+                                                    url,
+                                                    "URL_UNVERIFIED",
+                                                    tags=tags.append(f"tlsrpt-record-{key}"),
+                                                    parent=event,
+                                                )
+
+
+# EOF

bbot/modules/docker_pull.py

@@ -86,12 +86,12 @@ class docker_pull(BaseModule):
                 service = www_authenticate_headers.split('service="')[1].split('"')[0]
                 scope = www_authenticate_headers.split('scope="')[1].split('"')[0]
             except (KeyError, IndexError):
-                self.log.error(f"Could not obtain realm, service or scope from {url}")
+                self.log.warning(f"Could not obtain realm, service or scope from {url}")
                 break
             auth_url = f"{realm}?service={service}&scope={scope}"
             auth_response = await self.helpers.request(auth_url)
             if not auth_response:
-                self.log.error(f"Could not obtain token from {auth_url}")
+                self.log.warning(f"Could not obtain token from {auth_url}")
                 break
             auth_json = auth_response.json()
             token = auth_json["token"]
@@ -102,7 +102,8 @@ class docker_pull(BaseModule):
         url = f"{registry}/v2/{repository}/tags/list"
         r = await self.docker_api_request(url)
         if r is None or r.status_code != 200:
-            self.log.warning(f"Could not retrieve all tags for {repository} asuming tag:latest only.")
+            self.log.warning(f"Could not retrieve all tags for {repository} assuming tag:latest only.")
+            self.log.debug(f"Response: {r}")
             return ["latest"]
         try:
             tags = r.json().get("tags", ["latest"])
@@ -115,14 +116,15 @@ class docker_pull(BaseModule):
                 else:
                     return [tags[-1]]
         except (KeyError, IndexError):
-            self.log.error(f"Could not retrieve tags for {repository}.")
+            self.log.warning(f"Could not retrieve tags for {repository}.")
             return ["latest"]
 
     async def get_manifest(self, registry, repository, tag):
         url = f"{registry}/v2/{repository}/manifests/{tag}"
         r = await self.docker_api_request(url)
         if r is None or r.status_code != 200:
-            self.log.error(f"Could not retrieve manifest for {repository}:{tag}.")
+            self.log.warning(f"Could not retrieve manifest for {repository}:{tag}.")
+            self.log.debug(f"Response: {r}")
             return {}
         response_json = r.json()
         if response_json.get("manifests", []):

bbot/modules/dockerhub.py

@@ -31,7 +31,7 @@ class dockerhub(BaseModule):
     async def handle_org_stub(self, event):
         profile_name = event.data
         # docker usernames are case sensitive, so if there are capitalizations we also try a lowercase variation
-        profiles_to_check = set([profile_name, profile_name.lower()])
+        profiles_to_check = {profile_name, profile_name.lower()}
         for p in profiles_to_check:
             api_url = f"{self.api_url}/users/{p}"
             api_result = await self.helpers.request(api_url, follow_redirects=True)
@@ -64,7 +64,7 @@ class dockerhub(BaseModule):
     async def get_repos(self, username):
         repos = []
         url = f"{self.api_url}/repositories/{username}?page_size=25&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, json=False)
+        agen = self.api_page_iter(url, json=False)
         try:
             async for r in agen:
                 if r is None:

bbot/modules/dotnetnuke.py

@@ -31,8 +31,7 @@ class dotnetnuke(BaseModule):
         self.interactsh_subdomain_tags = {}
         self.interactsh_instance = None
 
-        if self.scan.config.get("interactsh_disable", False) == False:
-
+        if self.scan.config.get("interactsh_disable", False) is False:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
@@ -94,7 +93,7 @@ class dotnetnuke(BaseModule):
                     detected = True
                     break
 
-        if detected == True:
+        if detected is True:
             # DNNPersonalization Deserialization Detection
             for probe_url in [f'{event.data["url"]}/__', f'{event.data["url"]}/', f'{event.data["url"]}']:
                 result = await self.helpers.request(probe_url, cookies=self.exploit_probe)
@@ -114,7 +113,6 @@ class dotnetnuke(BaseModule):
                         )
 
             if "endpoint" not in event.tags:
-
                 # NewsArticlesSlider ImageHandler.ashx File Read
                 result = await self.helpers.request(
                     f'{event.data["url"]}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx?img=~/web.config'
@@ -155,24 +153,25 @@ class dotnetnuke(BaseModule):
 
                 # InstallWizard SuperUser Privilege Escalation
                 result = await self.helpers.request(f'{event.data["url"]}/Install/InstallWizard.aspx')
-                if result.status_code == 200:
-                    result_confirm = await self.helpers.request(
-                        f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
-                    )
-                    if result_confirm.status_code == 500:
-                        description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
-                        await self.emit_event(
-                            {
-                                "severity": "CRITICAL",
-                                "description": description,
-                                "host": str(event.host),
-                                "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
-                            },
-                            "VULNERABILITY",
-                            event,
-                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                if result:
+                    if result.status_code == 200:
+                        result_confirm = await self.helpers.request(
+                            f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
                         )
-                        return
+                        if result_confirm.status_code == 500:
+                            description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
+                            await self.emit_event(
+                                {
+                                    "severity": "CRITICAL",
+                                    "description": description,
+                                    "host": str(event.host),
+                                    "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
+                                },
+                                "VULNERABILITY",
+                                event,
+                                context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
+                            )
+                            return
 
                 # DNNImageHandler.ashx Blind SSRF
                 self.event_dict[event.data["url"]] = event

bbot/modules/emailformat.py

@@ -18,7 +18,7 @@ class emailformat(BaseModule):
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
         url = f"{self.base_url}/d/{self.helpers.quote(query)}/"
-        r = await self.request_with_fail_count(url)
+        r = await self.api_request(url)
         if not r:
             return
         for email in await self.helpers.re.extract_emails(r.text):

bbot/modules/extractous.py

@@ -0,0 +1,122 @@
+from extractous import Extractor
+
+from bbot.modules.base import BaseModule
+
+
+class extractous(BaseModule):
+    watched_events = ["FILESYSTEM"]
+    produced_events = ["RAW_TEXT"]
+    flags = ["passive", "safe"]
+    meta = {
+        "description": "Module to extract data from files",
+        "created_date": "2024-06-03",
+        "author": "@domwhewell-sage",
+    }
+    options = {
+        "extensions": [
+            "bak",  #  Backup File
+            "bash",  #  Bash Script or Configuration
+            "bashrc",  #  Bash Script or Configuration
+            "conf",  #  Configuration File
+            "cfg",  #  Configuration File
+            "crt",  #  Certificate File
+            "csv",  #  Comma Separated Values File
+            "db",  #  SQLite Database File
+            "sqlite",  #  SQLite Database File
+            "doc",  #  Microsoft Word Document (Old Format)
+            "docx",  #  Microsoft Word Document
+            "ica",  #  Citrix Independent Computing Architecture File
+            "indd",  #  Adobe InDesign Document
+            "ini",  #  Initialization File
+            "key",  #  Private Key File
+            "pub",  #  Public Key File
+            "log",  #  Log File
+            "markdown",  #  Markdown File
+            "md",  #  Markdown File
+            "odg",  #  OpenDocument Graphics (LibreOffice, OpenOffice)
+            "odp",  #  OpenDocument Presentation (LibreOffice, OpenOffice)
+            "ods",  #  OpenDocument Spreadsheet (LibreOffice, OpenOffice)
+            "odt",  #  OpenDocument Text (LibreOffice, OpenOffice)
+            "pdf",  #  Adobe Portable Document Format
+            "pem",  #  Privacy Enhanced Mail (SSL certificate)
+            "pps",  #  Microsoft PowerPoint Slideshow (Old Format)
+            "ppsx",  #  Microsoft PowerPoint Slideshow
+            "ppt",  #  Microsoft PowerPoint Presentation (Old Format)
+            "pptx",  #  Microsoft PowerPoint Presentation
+            "ps1",  #  PowerShell Script
+            "rdp",  #  Remote Desktop Protocol File
+            "sh",  #  Shell Script
+            "sql",  #  SQL Database Dump
+            "swp",  #  Swap File (temporary file, often Vim)
+            "sxw",  #  OpenOffice.org Writer document
+            "txt",  #  Plain Text Document
+            "vbs",  #  Visual Basic Script
+            "wpd",  #  WordPerfect Document
+            "xls",  #  Microsoft Excel Spreadsheet (Old Format)
+            "xlsx",  #  Microsoft Excel Spreadsheet
+            "xml",  #  eXtensible Markup Language File
+            "yml",  #  YAML Ain't Markup Language
+            "yaml",  #  YAML Ain't Markup Language
+        ],
+    }
+    options_desc = {
+        "extensions": "File extensions to parse",
+    }
+
+    deps_pip = ["extractous"]
+    scope_distance_modifier = 1
+
+    async def setup(self):
+        self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
+        return True
+
+    async def filter_event(self, event):
+        if "file" in event.tags:
+            if not any(event.data["path"].endswith(f".{ext}") for ext in self.extensions):
+                return False, "File extension not in the allowed list"
+        else:
+            return False, "Event is not a file"
+        return True
+
+    async def handle_event(self, event):
+        file_path = event.data["path"]
+        content = await self.scan.helpers.run_in_executor_mp(extract_text, file_path)
+        if isinstance(content, tuple):
+            error, traceback = content
+            self.error(f"Error extracting text from {file_path}: {error}")
+            self.trace(traceback)
+            return
+
+        if content:
+            raw_text_event = self.make_event(
+                content,
+                "RAW_TEXT",
+                context=f"Extracted text from {file_path}",
+                parent=event,
+            )
+            await self.emit_event(raw_text_event)
+
+
+def extract_text(file_path):
+    """
+    extract_text Extracts plaintext from a document path using extractous.
+
+    :param file_path: The path of the file to extract text from.
+    :return: ASCII-encoded plaintext extracted from the document.
+    """
+
+    try:
+        extractor = Extractor()
+        reader, metadata = extractor.extract_file(str(file_path))
+
+        result = ""
+        buffer = reader.read(4096)
+        while len(buffer) > 0:
+            result += buffer.decode("utf-8")
+            buffer = reader.read(4096)
+
+        return result.strip()
+    except Exception as e:
+        import traceback
+
+        return (str(e), traceback.format_exc())

bbot/modules/filedownload.py

@@ -25,12 +25,12 @@ class filedownload(BaseModule):
             "bak",  #  Backup File
             "bash",  #  Bash Script or Configuration
             "bashrc",  #  Bash Script or Configuration
-            "conf",  #  Configuration File
             "cfg",  #  Configuration File
+            "conf",  #  Configuration File
             "crt",  #  Certificate File
             "csv",  #  Comma Separated Values File
             "db",  #  SQLite Database File
-            "sqlite",  #  SQLite Database File
+            "dll",  #  Windows Dynamic Link Library
             "doc",  #  Microsoft Word Document (Old Format)
             "docx",  #  Microsoft Word Document
             "exe",  #  Windows PE executable
@@ -39,7 +39,6 @@ class filedownload(BaseModule):
             "ini",  #  Initialization File
             "jar",  #  Java Archive
             "key",  #  Private Key File
-            "pub",  #  Public Key File
             "log",  #  Log File
             "markdown",  #  Markdown File
             "md",  #  Markdown File
@@ -55,23 +54,26 @@ class filedownload(BaseModule):
             "ppt",  #  Microsoft PowerPoint Presentation (Old Format)
             "pptx",  #  Microsoft PowerPoint Presentation
             "ps1",  #  PowerShell Script
+            "pub",  #  Public Key File
             "raw",  #  Raw Image File Format
             "rdp",  #  Remote Desktop Protocol File
             "sh",  #  Shell Script
             "sql",  #  SQL Database Dump
+            "sqlite",  #  SQLite Database File
             "swp",  #  Swap File (temporary file, often Vim)
             "sxw",  #  OpenOffice.org Writer document
-            "tar",  #  Tar Archive
             "tar.gz",  # Gzip-Compressed Tar Archive
-            "zip",  #  Zip Archive
+            "tar",  #  Tar Archive
             "txt",  #  Plain Text Document
             "vbs",  #  Visual Basic Script
+            "war",  #  Java Web Archive
             "wpd",  #  WordPerfect Document
             "xls",  #  Microsoft Excel Spreadsheet (Old Format)
             "xlsx",  #  Microsoft Excel Spreadsheet
             "xml",  #  eXtensible Markup Language File
-            "yml",  #  YAML Ain't Markup Language
             "yaml",  #  YAML Ain't Markup Language
+            "yml",  #  YAML Ain't Markup Language
+            "zip",  #  Zip Archive
         ],
         "max_filesize": "10MB",
         "base_64_encoded_file": "false",
@@ -85,7 +87,7 @@ class filedownload(BaseModule):
     scope_distance_modifier = 3
 
     async def setup(self):
-        self.extensions = list(set([e.lower().strip(".") for e in self.config.get("extensions", [])]))
+        self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])})
         self.max_filesize = self.config.get("max_filesize", "10MB")
         self.download_dir = self.scan.home / "filedownload"
         self.helpers.mkdir(self.download_dir)

bbot/modules/fullhunt.py

@@ -18,19 +18,22 @@ class fullhunt(subdomain_enum_apikey):
 
     async def setup(self):
         self.api_key = self.config.get("api_key", "")
-        self.headers = {"x-api-key": self.api_key}
         return await super().setup()
 
     async def ping(self):
         url = f"{self.base_url}/auth/status"
-        j = (await self.request_with_fail_count(url, headers=self.headers)).json()
+        j = (await self.api_request(url)).json()
         remaining = j["user_credits"]["remaining_credits"]
         assert remaining > 0, "No credits remaining"
 
+    def prepare_api_request(self, url, kwargs):
+        kwargs["headers"]["x-api-key"] = self.api_key
+        return url, kwargs
+
     async def request_url(self, query):
         url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains"
-        response = await self.request_with_fail_count(url, headers=self.headers)
+        response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         return r.json().get("hosts", [])

bbot/modules/generic_ssrf.py

@@ -137,10 +137,10 @@ class Generic_XXE(BaseSubmodule):
         post_body = f"""<?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE foo [
 <!ELEMENT foo ANY >
-<!ENTITY % {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
+<!ENTITY {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
 ]>
 <foo>&{rand_entity};</foo>"""
-        test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
+        test_url = event.parsed_url.geturl()
         r = await self.parent_module.helpers.curl(
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )
@@ -163,7 +163,7 @@ class generic_ssrf(BaseModule):
         self.severity = None
         self.generic_only = self.config.get("generic_only", False)
 
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
@@ -216,7 +216,7 @@ class generic_ssrf(BaseModule):
                 self.debug("skipping result because subdomain tag was missing")
 
     async def cleanup(self):
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             try:
                 await self.interactsh_instance.deregister()
                 self.debug(
@@ -226,7 +226,7 @@ class generic_ssrf(BaseModule):
                 self.warning(f"Interactsh failure: {e}")
 
     async def finish(self):
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             await self.helpers.sleep(5)
             try:
                 for r in await self.interactsh_instance.poll():

bbot/modules/github_codesearch.py

@@ -42,7 +42,7 @@ class github_codesearch(github, subdomain_enum):
     async def query(self, query):
         repos = {}
         url = f"{self.base_url}/search/code?per_page=100&type=Code&q={self.helpers.quote(query)}&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, headers=self.headers, json=False)
         num_results = 0
         try:
             async for r in agen:
@@ -50,9 +50,10 @@ class github_codesearch(github, subdomain_enum):
                     break
                 status_code = getattr(r, "status_code", 0)
                 if status_code == 429:
-                    "Github is rate-limiting us (HTTP status: 429)"
+                    self.info("Github is rate-limiting us (HTTP status: 429)")
                     break
                 if status_code != 200:
+                    self.info(f"Unexpected response (HTTP status: {status_code})")
                     break
                 try:
                     j = r.json()

bbot/modules/github_org.py

@@ -104,7 +104,7 @@ class github_org(github):
     async def query_org_repos(self, query):
         repos = []
         url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -132,7 +132,7 @@ class github_org(github):
     async def query_org_members(self, query):
         members = []
         url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/members?per_page=100&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -160,7 +160,7 @@ class github_org(github):
     async def query_user_repos(self, query):
         repos = []
         url = f"{self.base_url}/users/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -189,7 +189,7 @@ class github_org(github):
         is_org = False
         in_scope = False
         url = f"{self.base_url}/orgs/{org}"
-        r = await self.helpers.request(url, headers=self.headers)
+        r = await self.api_request(url)
         if r is None:
             return is_org, in_scope
         status_code = getattr(r, "status_code", 0)

bbot/modules/github_workflows.py

@@ -88,7 +88,7 @@ class github_workflows(github):
     async def get_workflows(self, owner, repo):
         workflows = []
         url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows?per_page=100&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        agen = self.api_page_iter(url, json=False)
         try:
             async for r in agen:
                 if r is None:
@@ -115,7 +115,7 @@ class github_workflows(github):
     async def get_workflow_runs(self, owner, repo, workflow_id):
         runs = []
         url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?status=success&per_page={self.num_logs}"
-        r = await self.helpers.request(url, headers=self.headers)
+        r = await self.api_request(url)
         if r is None:
             return runs
         status_code = getattr(r, "status_code", 0)
@@ -166,7 +166,7 @@ class github_workflows(github):
             main_logs = []
             with zipfile.ZipFile(file_destination, "r") as logzip:
                 for name in logzip.namelist():
-                    if fnmatch.fnmatch(name, "*.txt") and not "/" in name:
+                    if fnmatch.fnmatch(name, "*.txt") and "/" not in name:
                         logzip.extract(name, folder)
                         main_logs.append(folder / name)
             return main_logs
@@ -176,7 +176,7 @@ class github_workflows(github):
     async def get_run_artifacts(self, owner, repo, run_id):
         artifacts = []
         url = f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts"
-        r = await self.helpers.request(url, headers=self.headers)
+        r = await self.api_request(url)
         if r is None:
             return artifacts
         status_code = getattr(r, "status_code", 0)

bbot/modules/gitlab.py

@@ -16,10 +16,7 @@ class gitlab(BaseModule):
     scope_distance_modifier = 2
 
     async def setup(self):
-        self.headers = {}
-        self.api_key = self.config.get("api_key", "")
-        if self.api_key:
-            self.headers.update({"Authorization": f"Bearer {self.api_key}"})
+        await self.require_api_key()
         return True
 
     async def filter_event(self, event):
@@ -111,7 +108,7 @@ class gitlab(BaseModule):
             await self.handle_namespace(group, event)
 
     async def gitlab_json_request(self, url):
-        response = await self.helpers.request(url, headers=self.headers)
+        response = await self.api_request(url)
         if response is not None:
             try:
                 json = response.json()