bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/modules/google_playstore.py

@@ -0,0 +1,93 @@
+from bbot.modules.base import BaseModule
+
+
+class google_playstore(BaseModule):
+    watched_events = ["ORG_STUB", "CODE_REPOSITORY"]
+    produced_events = ["MOBILE_APP"]
+    flags = ["passive", "safe", "code-enum"]
+    meta = {
+        "description": "Search for android applications on play.google.com",
+        "created_date": "2024-10-08",
+        "author": "@domwhewell-sage",
+    }
+
+    base_url = "https://play.google.com"
+
+    async def setup(self):
+        self.app_link_regex = self.helpers.re.compile(r"/store/apps/details\?id=([a-zA-Z0-9._-]+)")
+        return True
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "android" not in event.tags:
+                return False, "event is not an android repository"
+        return True
+
+    async def handle_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            await self.handle_url(event)
+        elif event.type == "ORG_STUB":
+            await self.handle_org_stub(event)
+
+    async def handle_url(self, event):
+        repo_url = event.data.get("url")
+        app_id = repo_url.split("id=")[1].split("&")[0]
+        await self.emit_event(
+            {"id": app_id, "url": repo_url},
+            "MOBILE_APP",
+            tags="android",
+            parent=event,
+            context=f'{{module}} extracted the mobile app name "{app_id}"  from: {repo_url}',
+        )
+
+    async def handle_org_stub(self, event):
+        org_name = event.data
+        self.verbose(f"Searching for any android applications for {org_name}")
+        for apk_name in await self.query(org_name):
+            valid_apk = await self.validate_apk(apk_name)
+            if valid_apk:
+                self.verbose(f"Got {apk_name} from playstore")
+                await self.emit_event(
+                    {"id": apk_name, "url": f"{self.base_url}/store/apps/details?id={apk_name}"},
+                    "MOBILE_APP",
+                    tags="android",
+                    parent=event,
+                    context=f'{{module}} searched play.google.com for apps belonging to "{org_name}" and found "{apk_name}" to be in scope',
+                )
+            else:
+                self.debug(f"Got {apk_name} from playstore app details does not contain any in-scope URLs or Emails")
+
+    async def query(self, query):
+        app_links = []
+        url = f"{self.base_url}/store/search?q={self.helpers.quote(query)}&c=apps"
+        r = await self.helpers.request(url)
+        if r is None:
+            return app_links
+        status_code = getattr(r, "status_code", 0)
+        try:
+            html_content = r.content.decode("utf-8")
+            # Use regex to find all app links
+            app_links = await self.helpers.re.findall(self.app_link_regex, html_content)
+        except Exception as e:
+            self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}")
+            return app_links
+        return app_links
+
+    async def validate_apk(self, apk_name):
+        """
+        Check the app details page the "App support" section will include URLs or Emails to the app developer
+        """
+        in_scope = False
+        url = f"{self.base_url}/store/apps/details?id={apk_name}"
+        r = await self.helpers.request(url)
+        if r is None:
+            return in_scope
+        status_code = getattr(r, "status_code", 0)
+        if status_code == 200:
+            html = r.text
+            in_scope_hosts = await self.scan.extract_in_scope_hostnames(html)
+            if in_scope_hosts:
+                in_scope = True
+        else:
+            self.warning(f"Failed to fetch {url} (HTTP status: {status_code})")
+        return in_scope

bbot/modules/gowitness.py

@@ -1,5 +1,5 @@
 import asyncio
-import sqlite3
+import aiosqlite
 import multiprocessing
 from pathlib import Path
 from contextlib import suppress
@@ -34,6 +34,7 @@ class gowitness(BaseModule):
         "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds",
     }
     deps_common = ["chromium"]
+    deps_pip = ["aiosqlite"]
     deps_ansible = [
         {
             "name": "Download gowitness",
@@ -72,7 +73,7 @@ class gowitness(BaseModule):
 
         # make sure we have a working chrome install
         chrome_test_pass = False
-        for binary in ("chrome", "chromium", custom_chrome_path):
+        for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path):
             binary_path = self.helpers.which(binary)
             if binary_path and Path(binary_path).is_file():
                 chrome_test_proc = await self.run_process([binary_path, "--version"])
@@ -87,7 +88,7 @@ class gowitness(BaseModule):
         self.screenshot_path = self.base_path / "screenshots"
         self.command = self.construct_command()
         self.prepped = False
-        self.screenshots_taken = dict()
+        self.screenshots_taken = {}
         self.connections_logged = set()
         self.technologies_found = set()
         return True
@@ -136,7 +137,8 @@ class gowitness(BaseModule):
             return
 
         # emit web screenshots
-        for filename, screenshot in self.new_screenshots.items():
+        new_screenshots = await self.get_new_screenshots()
+        for filename, screenshot in new_screenshots.items():
             url = screenshot["url"]
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
@@ -150,7 +152,8 @@ class gowitness(BaseModule):
             )
 
         # emit URLs
-        for url, row in self.new_network_logs.items():
+        new_network_logs = await self.get_new_network_logs()
+        for url, row in new_network_logs.items():
             ip = row["ip"]
             status_code = row["status_code"]
             tags = [f"status-{status_code}", f"ip-{ip}", "spider-danger"]
@@ -168,7 +171,8 @@ class gowitness(BaseModule):
                 )
 
         # emit technologies
-        for _, row in self.new_technologies.items():
+        new_technologies = await self.get_new_technologies()
+        for row in new_technologies.values():
             parent_id = row["url_id"]
             parent_url = self.screenshots_taken[parent_id]
             parent_event = event_dict[parent_url]
@@ -207,67 +211,61 @@ class gowitness(BaseModule):
         command += ["--timeout", str(self.timeout)]
         return command
 
-    @property
-    def new_screenshots(self):
+    async def get_new_screenshots(self):
         screenshots = {}
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
                 con.text_factory = self.helpers.smart_decode
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM urls")
-                for row in res:
-                    row = dict(row)
-                    _id = row["id"]
-                    if _id not in self.screenshots_taken:
-                        self.screenshots_taken[_id] = row["url"]
-                        screenshots[_id] = row
+                async with con.execute("SELECT * FROM urls") as cur:
+                    async for row in cur:
+                        row = dict(row)
+                        _id = row["id"]
+                        if _id not in self.screenshots_taken:
+                            self.screenshots_taken[_id] = row["url"]
+                            screenshots[_id] = row
         return screenshots
 
-    @property
-    def new_network_logs(self):
-        network_logs = dict()
+    async def get_new_network_logs(self):
+        network_logs = {}
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM network_logs")
-                for row in res:
-                    row = dict(row)
-                    url = row["final_url"]
-                    if url not in self.connections_logged:
-                        self.connections_logged.add(url)
-                        network_logs[url] = row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
+                async with con.execute("SELECT * FROM network_logs") as cur:
+                    async for row in cur:
+                        row = dict(row)
+                        url = row["final_url"]
+                        if url not in self.connections_logged:
+                            self.connections_logged.add(url)
+                            network_logs[url] = row
         return network_logs
 
-    @property
-    def new_technologies(self):
-        technologies = dict()
+    async def get_new_technologies(self):
+        technologies = {}
         if self.db_path.is_file():
-            with sqlite3.connect(str(self.db_path)) as con:
-                con.row_factory = sqlite3.Row
-                cur = con.cursor()
-                res = self.cur_execute(cur, "SELECT * FROM technologies")
-                for row in res:
-                    _id = row["id"]
-                    if _id not in self.technologies_found:
-                        self.technologies_found.add(_id)
-                        row = dict(row)
-                        technologies[_id] = row
+            async with aiosqlite.connect(str(self.db_path)) as con:
+                con.row_factory = aiosqlite.Row
+                async with con.execute("SELECT * FROM technologies") as cur:
+                    async for row in cur:
+                        _id = row["id"]
+                        if _id not in self.technologies_found:
+                            self.technologies_found.add(_id)
+                            row = dict(row)
+                            technologies[_id] = row
         return technologies
 
-    def cur_execute(self, cur, query):
+    async def cur_execute(self, cur, query):
         try:
-            return cur.execute(query)
-        except sqlite3.OperationalError as e:
+            return await cur.execute(query)
+        except aiosqlite.OperationalError as e:
             self.warning(f"Error executing query: {query}: {e}")
             return []
 
     async def report(self):
         if self.screenshots_taken:
             self.success(f"{len(self.screenshots_taken):,} web screenshots captured. To view:")
-            self.success(f"    - Start gowitness")
+            self.success("    - Start gowitness")
             self.success(f"        - cd {self.base_path} && ./gowitness server")
-            self.success(f"    - Browse to http://localhost:7171")
+            self.success("    - Browse to http://localhost:7171")
         else:
-            self.info(f"No web screenshots captured")
+            self.info("No web screenshots captured")

bbot/modules/hackertarget.py

@@ -15,15 +15,17 @@ class hackertarget(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}"
-        response = await self.request_with_fail_count(url)
+        response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
+        results = set()
         for line in r.text.splitlines():
             host = line.split(",")[0]
             try:
                 self.helpers.validators.validate_host(host)
-                yield host
+                results.add(host)
             except ValueError:
                 self.debug(f"Error validating API result: {line}")
                 continue
+        return results

bbot/modules/host_header.py

@@ -19,7 +19,7 @@ class host_header(BaseModule):
 
     async def setup(self):
         self.subdomain_tags = {}
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             try:
                 self.interactsh_instance = self.helpers.interactsh()
                 self.domain = await self.interactsh_instance.register(callback=self.interactsh_callback)
@@ -60,7 +60,7 @@ class host_header(BaseModule):
                 self.debug("skipping results because subdomain tag was missing")
 
     async def finish(self):
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             await self.helpers.sleep(5)
             try:
                 for r in await self.interactsh_instance.poll():
@@ -69,7 +69,7 @@ class host_header(BaseModule):
                 self.debug(f"Error in interact.sh: {e}")
 
     async def cleanup(self):
-        if self.scan.config.get("interactsh_disable", False) == False:
+        if self.scan.config.get("interactsh_disable", False) is False:
             try:
                 await self.interactsh_instance.deregister()
                 self.debug(
@@ -84,7 +84,7 @@ class host_header(BaseModule):
 
         added_cookies = {}
 
-        for header, header_values in event.data["header-dict"].items():
+        for header_values in event.data["header-dict"].values():
             for header_value in header_values:
                 if header_value.lower() == "set-cookie":
                     header_split = header_value.split("=")
@@ -136,7 +136,7 @@ class host_header(BaseModule):
 
         split_output = output.split("\n")
         if " 4" in split_output:
-            description = f"Duplicate Host Header Tolerated"
+            description = "Duplicate Host Header Tolerated"
             await self.emit_event(
                 {
                     "host": str(event.host),

bbot/modules/httpx.py

@@ -90,7 +90,7 @@ class httpx(BaseModule):
         else:
             url = str(event.data)
             url_hash = hash((event.host, event.port, has_spider_max))
-        if url_hash == None:
+        if url_hash is None:
             url_hash = hash((url, has_spider_max))
         return url, url_hash
 
@@ -172,9 +172,6 @@ class httpx(BaseModule):
             httpx_ip = j.get("host", "")
             if httpx_ip:
                 tags.append(f"ip-{httpx_ip}")
-            # detect login pages
-            if self.helpers.web.is_login_page(j.get("body", "")):
-                tags.append("login-page")
             # grab title
             title = self.helpers.tagify(j.get("title", ""), maxlen=30)
             if title:

bbot/modules/hunterio.py

@@ -15,14 +15,9 @@ class hunterio(subdomain_enum_apikey):
     options_desc = {"api_key": "Hunter.IO API key"}
 
     base_url = "https://api.hunter.io/v2"
+    ping_url = f"{base_url}/account?api_key={{api_key}}"
     limit = 100
 
-    async def ping(self):
-        url = f"{self.base_url}/account?api_key={self.api_key}"
-        r = await self.helpers.request(url)
-        resp_content = getattr(r, "text", "")
-        assert getattr(r, "status_code", 0) == 200, resp_content
-
     async def handle_event(self, event):
         query = self.make_query(event)
         for entry in await self.query(query):
@@ -56,10 +51,9 @@ class hunterio(subdomain_enum_apikey):
     async def query(self, query):
         emails = []
         url = (
-            f"{self.base_url}/domain-search?domain={query}&api_key={self.api_key}"
-            + "&limit={page_size}&offset={offset}"
+            f"{self.base_url}/domain-search?domain={query}&api_key={{api_key}}" + "&limit={page_size}&offset={offset}"
         )
-        agen = self.helpers.api_page_iter(url, page_size=self.limit)
+        agen = self.api_page_iter(url, page_size=self.limit)
         try:
             async for j in agen:
                 new_emails = j.get("data", {}).get("emails", [])

bbot/modules/iis_shortnames.py

@@ -20,7 +20,7 @@ class iis_shortnames(BaseModule):
     meta = {
         "description": "Check for IIS shortname vulnerability",
         "created_date": "2022-04-15",
-        "author": "@pmueller",
+        "author": "@liquidsec",
     }
     options = {"detect_only": True, "max_node_count": 50}
     options_desc = {
@@ -38,37 +38,26 @@ class iis_shortnames(BaseModule):
         control_url = f"{target}{random_string}*~1*/a.aspx"
         test_url = f"{target}*~1*/a.aspx"
 
-        urls_and_kwargs = []
         for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]:
-            kwargs = dict(method=method, allow_redirects=False, timeout=10)
-            urls_and_kwargs.append((control_url, kwargs, method))
-            urls_and_kwargs.append((test_url, kwargs, method))
-
-        results = {}
-        async for url, kwargs, method, response in self.helpers.request_custom_batch(urls_and_kwargs):
-            try:
-                results[method][url] = response
-            except KeyError:
-                results[method] = {url: response}
-        for method, result in results.items():
+            kwargs = {"method": method, "allow_redirects": False, "timeout": 10}
             confirmations = 0
-            iterations = 4  # one failed detection is tolerated, as long as its not the first run
+            iterations = 5  # one failed detection is tolerated, as long as its not the first run
             while iterations > 0:
-                control = results[method].get(control_url, None)
-                test = results[method].get(test_url, None)
-                if control and test:
-                    if control.status_code != test.status_code:
+                control_result = await self.helpers.request(control_url, **kwargs)
+                test_result = await self.helpers.request(test_url, **kwargs)
+                if control_result and test_result:
+                    if control_result.status_code != test_result.status_code:
                         confirmations += 1
-                        self.debug(f"New detection, number of confirmations: [{str(confirmations)}]")
-                        if confirmations > 2:
-                            technique = f"{str(control.status_code)}/{str(test.status_code)} HTTP Code"
-                            detections.append((method, test.status_code, technique))
+                        self.debug(f"New detection on {target}, number of confirmations: [{str(confirmations)}]")
+                        if confirmations > 3:
+                            technique = f"{str(control_result.status_code)}/{str(test_result.status_code)} HTTP Code"
+                            detections.append((method, test_result.status_code, technique))
                             break
-                    elif ("Error Code</th><td>0x80070002" in control.text) and (
-                        "Error Code</th><td>0x00000000" in test.text
+                    elif ("Error Code</th><td>0x80070002" in control_result.text) and (
+                        "Error Code</th><td>0x00000000" in test_result.text
                     ):
                         confirmations += 1
-                        if confirmations > 2:
+                        if confirmations > 3:
                             detections.append((method, 0, technique))
                             technique = "HTTP Body Error Message"
                             break
@@ -139,7 +128,7 @@ class iis_shortnames(BaseModule):
         suffix = "/a.aspx"
 
         urls_and_kwargs = []
-        kwargs = dict(method=method, allow_redirects=False, retries=2, timeout=10)
+        kwargs = {"method": method, "allow_redirects": False, "retries": 2, "timeout": 10}
         for c in valid_chars:
             for file_part in ("stem", "ext"):
                 payload = encode_all(f"*{c}*~1*")
@@ -171,7 +160,7 @@ class iis_shortnames(BaseModule):
         url_hint_list = []
         found_results = False
 
-        cl = ext_char_list if extension_mode == True else char_list
+        cl = ext_char_list if extension_mode is True else char_list
 
         urls_and_kwargs = []
 
@@ -180,7 +169,7 @@ class iis_shortnames(BaseModule):
             wildcard = "*" if extension_mode else "*~1*"
             payload = encode_all(f"{prefix}{c}{wildcard}")
             url = f"{target}{payload}{suffix}"
-            kwargs = dict(method=method)
+            kwargs = {"method": method}
             urls_and_kwargs.append((url, kwargs, c))
 
         async for url, kwargs, c, response in self.helpers.request_custom_batch(urls_and_kwargs):
@@ -220,7 +209,7 @@ class iis_shortnames(BaseModule):
                         extension_mode,
                         node_count=node_count,
                     )
-        if len(prefix) > 0 and found_results == False:
+        if len(prefix) > 0 and found_results is False:
             url_hint_list.append(f"{prefix}")
             self.verbose(f"Found new (possibly partial) URL_HINT: {prefix} from node {target}")
         return url_hint_list
@@ -245,7 +234,7 @@ class iis_shortnames(BaseModule):
                 {"severity": "LOW", "host": str(event.host), "url": normalized_url, "description": description},
                 "VULNERABILITY",
                 event,
-                context=f"{{module}} detected low {{event.type}}: IIS shortname enumeration",
+                context="{module} detected low {event.type}: IIS shortname enumeration",
             )
             if not self.config.get("detect_only"):
                 for detection in detections:

bbot/modules/internal/cloudcheck.py

@@ -1,7 +1,9 @@
-from bbot.modules.base import InterceptModule
+from contextlib import suppress
 
+from bbot.modules.base import BaseInterceptModule
 
-class CloudCheck(InterceptModule):
+
+class CloudCheck(BaseInterceptModule):
     watched_events = ["*"]
     meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"}
     scope_distance_modifier = 1
@@ -13,7 +15,7 @@ class CloudCheck(InterceptModule):
 
     def make_dummy_modules(self):
         self.dummy_modules = {}
-        for provider_name, provider in self.helpers.cloud.providers.items():
+        for provider_name in self.helpers.cloud.providers.keys():
             module = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
             module.default_discovery_context = "{module} derived {event.type}: {event.host}"
             self.dummy_modules[provider_name] = module
@@ -28,22 +30,35 @@ class CloudCheck(InterceptModule):
         if self.dummy_modules is None:
             self.make_dummy_modules()
         # cloud tagging by hosts
-        hosts_to_check = set(str(s) for s in event.resolved_hosts)
-        # we use the original host, since storage buckets hostnames might be collapsed to _wildcard
-        hosts_to_check.add(str(event.host_original))
-        for host in hosts_to_check:
+        hosts_to_check = set(event.resolved_hosts)
+        with suppress(KeyError):
+            hosts_to_check.remove(event.host_original)
+        hosts_to_check = [event.host_original] + list(hosts_to_check)
+
+        for i, host in enumerate(hosts_to_check):
+            host_is_ip = self.helpers.is_ip(host)
             for provider, provider_type, subnet in self.helpers.cloudcheck(host):
                 if provider:
                     event.add_tag(f"{provider_type}-{provider}")
+                    if host_is_ip:
+                        event.add_tag(f"{provider_type}-ip")
+                    else:
+                        # if the original hostname is a cloud domain, tag it as such
+                        if i == 0:
+                            event.add_tag(f"{provider_type}-domain")
+                        # any children are tagged as CNAMEs
+                        else:
+                            event.add_tag(f"{provider_type}-cname")
 
         found = set()
+        str_hosts_to_check = [str(host) for host in hosts_to_check]
         # look for cloud assets in hosts, http responses
         # loop through each provider
         for provider in self.helpers.cloud.providers.values():
             provider_name = provider.name.lower()
-            base_kwargs = dict(
-                parent=event, tags=[f"{provider.provider_type}-{provider_name}"], _provider=provider_name
-            )
+            base_kwargs = {
+                "parent": event, "tags": [f"{provider.provider_type}-{provider_name}"], "_provider": provider_name
+            }
             # loop through the provider's regex signatures, if any
             for event_type, sigs in provider.signatures.items():
                 if event_type != "STORAGE_BUCKET":
@@ -54,12 +69,12 @@ class CloudCheck(InterceptModule):
                     if event.type == "HTTP_RESPONSE":
                         matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
                     elif event.type.startswith("DNS_NAME"):
-                        for host in hosts_to_check:
+                        for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:
                                 matches.append(match.groups())
                     for match in matches:
-                        if not match in found:
+                        if match not in found:
                             found.add(match)
 
                             _kwargs = dict(base_kwargs)