bbot 2.4.2.6611rc0__tar.gz → 2.4.2.6621rc0__tar.gz

{bbot-2.4.2.6611rc0 → bbot-2.4.2.6621rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bbot
-Version: 2.4.2.6611rc0
+Version: 2.4.2.6621rc0
 Summary: OSINT automation for hackers.
 License: GPL-3.0
 Keywords: python,cli,automation,osint,threat-intel,intelligence,neo4j,scanner,python-library,hacking,recursion,pentesting,recon,command-line-tool,bugbounty,subdomains,security-tools,subdomain-scanner,osint-framework,attack-surface,subdomain-enumeration,osint-tool

{bbot-2.4.2.6611rc0 → bbot-2.4.2.6621rc0}/bbot/__init__.py

@@ -1,5 +1,5 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
-__version__ = "v2.4.2.6611rc"
+__version__ = "v2.4.2.6621rc"
 
 from .scanner import Scanner, Preset
 

{bbot-2.4.2.6611rc0 → bbot-2.4.2.6621rc0}/bbot/modules/dehashed.py

@@ -13,23 +13,23 @@ class dehashed(subdomain_enum):
         "author": "@SpamFaux",
         "auth_required": True,
     }
-    options = {"username": "", "api_key": ""}
-    options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"}
+    options = {"api_key": ""}
+    options_desc = {"api_key": "DeHashed API Key"}
     target_only = True
 
-    base_url = "https://api.dehashed.com/search"
+    base_url = "https://api.dehashed.com/v2/search"
 
     async def setup(self):
-        self.username = self.config.get("username", "")
         self.api_key = self.config.get("api_key", "")
-        self.auth = (self.username, self.api_key)
         self.headers = {
             "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Dehashed-Api-Key": self.api_key,
         }
 
         # soft-fail if we don't have the necessary information to make queries
-        if not (self.username and self.api_key):
-            return None, "No username / API key set"
+        if not self.api_key:
+            return None, "No API key set"
 
         return await super().setup()
 
@@ -38,29 +38,31 @@ class dehashed(subdomain_enum):
         async for entries in self.query(query):
             for entry in entries:
                 # we have to clean up the email field because dehashed does a poor job of it
-                email_str = entry.get("email", "").replace("\\", "")
-                found_emails = list(await self.helpers.re.extract_emails(email_str))
-                if not found_emails:
-                    self.debug(f"Invalid email from dehashed.com: {email_str}")
-                    continue
-                email = found_emails[0]
+                emails = []
+                for email in entry.get("email", []):
+                    email_str = email.replace("\\", "")
+                    found_emails = list(await self.helpers.re.extract_emails(email_str))
+                    if not found_emails:
+                        self.debug(f"Invalid email from dehashed.com: {email_str}")
+                        continue
+                    emails += found_emails
 
-                user = entry.get("username", "")
-                pw = entry.get("password", "")
-                h_pw = entry.get("hashed_password", "")
+                users = entry.get("username", [])
+                pws = entry.get("password", [])
+                h_pws = entry.get("hashed_password", [])
                 db_name = entry.get("database_name", "")
 
                 tags = []
                 if db_name:
                     tags = [f"db-{db_name}"]
-                if email:
+                for email in emails:
                     email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
                     if email_event is not None:
                         await self.emit_event(
                             email_event,
                             context=f'{{module}} searched API for "{query}" and found {{event.type}}: {{event.data}}',
                         )
-                        if user:
+                        for user in users:
                             await self.emit_event(
                                 f"{email}:{user}",
                                 "USERNAME",
@@ -68,7 +70,7 @@ class dehashed(subdomain_enum):
                                 tags=tags,
                                 context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
                             )
-                        if pw:
+                        for pw in pws:
                             await self.emit_event(
                                 f"{email}:{pw}",
                                 "PASSWORD",
@@ -76,7 +78,7 @@ class dehashed(subdomain_enum):
                                 tags=tags,
                                 context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
                             )
-                        if h_pw:
+                        for h_pw in h_pws:
                             await self.emit_event(
                                 f"{email}:{h_pw}",
                                 "HASHED_PASSWORD",
@@ -86,30 +88,33 @@ class dehashed(subdomain_enum):
                             )
 
     async def query(self, domain):
-        query = f"domain:{domain}"
-        url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
-        page = 0
-        num_entries = 0
-        agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, _json=False)
+        url = self.base_url
+        json = {
+            "query": "",
+            "page": 1,
+            "size": 10000,  # The maximum permitted size and pagination.
+        }
+        json["query"] = f"domain:{domain}"
+        json["page"] = 1
+        max_pages = 1
+        agen = self.api_page_iter(url=url, headers=self.headers, _json=False, method="POST", json=json)
         async for result in agen:
             result_json = {}
             with suppress(Exception):
                 result_json = result.json()
             total = result_json.get("total", 0)
             entries = result_json.get("entries", [])
-            if entries is None:
-                entries = []
-            num_entries += len(entries)
-            page += 1
-            if (page >= 3) or (not entries):
-                if result is not None and result.status_code != 200:
-                    self.warning(
-                        f"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}"
-                    )
-                elif (page >= 3) and (total > num_entries):
-                    self.info(
-                        f"{domain} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results."
-                    )
+            json["page"] += 1
+            if result is not None and result.status_code != 200:
+                self.warning(
+                    f"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}"
+                )
+            elif (json["page"] > max_pages) and (total > (json["size"] * max_pages)):
+                self.info(
+                    f"{domain} has {total:,} results in Dehashed. The API can only process the first 10,000 results. Please check dehashed.com to get the remaining results."
+                )
+            if entries:
+                yield entries
+            if not entries or json["page"] > max_pages:
                 await agen.aclose()
                 break
-            yield entries

bbot-2.4.2.6621rc0/bbot/modules/github_usersearch.py

@@ -0,0 +1,75 @@
+from bbot.modules.templates.github import github
+from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+
+class github_usersearch(github, subdomain_enum):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["SOCIAL", "EMAIL_ADDRESS"]
+    flags = ["passive", "safe", "code-enum"]
+    meta = {
+        "description": "Query Github's API for users with emails matching in scope domains that may not be discoverable by listing members of the organization.",
+        "created_date": "2025-05-10",
+        "author": "@domwhewell-sage",
+        "auth_required": True,
+    }
+    options = {"api_key": ""}
+    options_desc = {"api_key": "Github token"}
+
+    async def handle_event(self, event):
+        self.verbose("Searching for users with emails matching in scope domains")
+        query = self.make_query(event)
+        users = await self.query_users(query)
+        for user, email in users:
+            user_url = f"https://github.com/{user}"
+            event_data = {"platform": "github", "profile_name": user, "url": user_url}
+            user_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", parent=event)
+            if user_event:
+                await self.emit_event(
+                    user_event,
+                    context=f"{{module}} searched for users with {{DNS_NAME}} in the profile and discovered {{event.type}}: {user_url}",
+                )
+            if email:
+                await self.emit_event(
+                    email,
+                    "EMAIL_ADDRESS",
+                    parent=event,
+                    context=f"{{module}} found an {{event.type}} on the github profile {user_url}: {{event.data}}",
+                )
+
+    async def query_users(self, query):
+        users = []
+        graphql_query = f"""query search_users {{
+            search(query: "{query}", type: USER, first: 100, after: "{{NEXT_KEY}}") {{
+                userCount
+                pageInfo {{
+                    hasNextPage
+                    endCursor
+                }}
+                edges {{
+                    node {{
+                        ... on User {{
+                          login
+                          # bio Commented out as user can add arbritrary domains to their bio
+                          email # Email is verified by github
+                          websiteUrl # Website is not verified by github
+                        }}
+                    }}
+                }}
+            }}
+        }}"""
+        async for data in self.github_graphql_request(graphql_query, "search"):
+            if data:
+                user_count = data.get("userCount", 0)
+                self.verbose(f"Found {user_count} users with the query {query}, verifying if they are in-scope...")
+                edges = data.get("edges", [])
+                for node in edges:
+                    user = node.get("node", {})
+                    in_scope_hosts = await self.scan.extract_in_scope_hostnames(str(user))
+                    if in_scope_hosts:
+                        login = user.get("login", "")
+                        email = user.get("email", None)
+                        self.verbose(
+                            f'Found in-scope hostname(s): "{in_scope_hosts}" in the profile https://github.com/{login}, the profile appears to be in-scope'
+                        )
+                        users.append((login, email))
+        return users

{bbot-2.4.2.6611rc0 → bbot-2.4.2.6621rc0}/bbot/modules/templates/github.py

@@ -44,3 +44,31 @@ class github(BaseModule):
             self.trace(traceback.format_exc())
             return None, f"Error with API ({str(e).strip()})"
         return True
+
+    async def github_graphql_request(self, graphql_query, resp_key):
+        url = f"{self.base_url}/graphql"
+        next_key = ""
+        has_next_page = True
+
+        while has_next_page:
+            query = graphql_query.replace("{NEXT_KEY}", next_key)
+            r = await self.api_request(url, method="POST", json={"query": query})
+            if r is None:
+                break
+            status_code = getattr(r, "status_code", 0)
+            if status_code == 403:
+                self.warning("Github is rate-limiting us (HTTP status: 403)")
+                break
+            try:
+                json = r.json()
+            except Exception as e:
+                self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                break
+
+            data = json.get("data", {}).get(resp_key, {})
+            yield data
+
+            # Update pagination variables
+            page_info = data.get("pageInfo", {})
+            has_next_page = page_info.get("hasNextPage", False)
+            next_key = page_info.get("endCursor", "")

bbot-2.4.2.6621rc0/bbot/test/test_step_2/module_tests/test_module_dehashed.py

@@ -0,0 +1,159 @@
+from .base import ModuleTestBase
+
+
+class TestDehashed(ModuleTestBase):
+    modules_overrides = ["dehashed", "speculate"]
+    config_overrides = {
+        "scope": {"report_distance": 2},
+        "modules": {"dehashed": {"api_key": "deadbeef"}},
+    }
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://api.dehashed.com/v2/search",
+            method="POST",
+            json={
+                "balance": 10000,
+                "entries": [
+                    {
+                        "id": "4363462346",
+                        "email": ["bob@blacklanternsecurity.com"],
+                        "ip_address": ["127.0.0.9"],
+                        "username": ["bob@bob.com"],
+                        "hashed_password": ["$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve"],
+                        "name": ["Bob Smith"],
+                        "phone": ["+91283423839"],
+                        "database_name": "eatstreet",
+                        "raw_record": {"le_only": True, "unstructured": True},
+                    },
+                    {
+                        "id": "234623453454",
+                        "email": ["tim@blacklanternsecurity.com"],
+                        "username": ["timmy"],
+                        "password": ["TimTamSlam69"],
+                        "name": "Tim Tam",
+                        "phone": ["+123455667"],
+                        "database_name": "eatstreet",
+                    },
+                ],
+                "took": "61ms",
+                "total": 2,
+            },
+        )
+        await module_test.mock_dns(
+            {
+                "bob.com": {"A": ["127.0.0.1"]},
+                "blacklanternsecurity.com": {"A": ["127.0.0.1"]},
+            }
+        )
+
+    def check(self, module_test, events):
+        assert len(events) == 12
+        assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
+        assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "EMAIL_ADDRESS"
+                and e.data == "bob@bob.com"
+                and e.scope_distance == 1
+                and "affiliate" in e.tags
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME" and e.data == "bob.com" and e.scope_distance == 1 and "affiliate" in e.tags
+            ]
+        )
+        assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "USERNAME"
+                and e.data == "bob@blacklanternsecurity.com:bob@bob.com"
+                and e.parent.data == "bob@blacklanternsecurity.com"
+            ]
+        )
+        assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "HASHED_PASSWORD"
+                and e.data
+                == "bob@blacklanternsecurity.com:$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve"
+            ]
+        )
+        assert 1 == len(
+            [e for e in events if e.type == "PASSWORD" and e.data == "tim@blacklanternsecurity.com:TimTamSlam69"]
+        )
+        assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim@blacklanternsecurity.com:timmy"])
+
+
+class TestDehashedBadEmail(TestDehashed):
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://api.dehashed.com/v2/search",
+            method="POST",
+            json={
+                "balance": 10000,
+                "entries": [
+                    {
+                        "id": "EZxg4Lz-INLUt6uRXZaV",
+                        "email": ["foo.example.com"],
+                        "database_name": "Collections",
+                    },
+                ],
+                "took": "41ms",
+                "total": 1,
+            },
+        )
+
+    def check(self, module_test, events):
+        debug_log_content = open(module_test.scan.home / "debug.log").read()
+        assert "Invalid email from dehashed.com: foo.example.com" in debug_log_content
+
+
+class TestDehashedHTTPError(TestDehashed):
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://api.dehashed.com/v2/search",
+            method="POST",
+            json={"error": "issue with request body"},
+            status_code=400,
+        )
+
+    def check(self, module_test, events):
+        scan_log_content = open(module_test.scan.home / "scan.log").read()
+        assert (
+            'Error retrieving results from dehashed.com (status code 400): {"error":"issue with request body"}'
+            in scan_log_content
+        )
+
+
+class TestDehashedTooManyResults(TestDehashed):
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://api.dehashed.com/v2/search",
+            method="POST",
+            json={
+                "balance": 10000,
+                "entries": [
+                    {
+                        "id": "VXhNxj46SGsW4Lworh-G",
+                        "email": ["bob@bob.com"],
+                        "database_name": "Collections",
+                    },
+                ],
+                "took": "40ms",
+                "total": 10001,
+            },
+        )
+
+    def check(self, module_test, events):
+        scan_log_content = open(module_test.scan.home / "scan.log").read()
+        assert "has 10,001 results in Dehashed. The API can only process the first 10,000 results." in scan_log_content

bbot-2.4.2.6621rc0/bbot/test/test_step_2/module_tests/test_module_github_usersearch.py

@@ -0,0 +1,138 @@
+from .base import ModuleTestBase
+
+
+class TestGithub_Usersearch(ModuleTestBase):
+    config_overrides = {"modules": {"github_usersearch": {"api_key": "asdf"}}}
+    query_1 = """query search_users {
+            search(query: "blacklanternsecurity.com", type: USER, first: 100, after: "") {
+                userCount
+                pageInfo {
+                    hasNextPage
+                    endCursor
+                }
+                edges {
+                    node {
+                        ... on User {
+                          login
+                          # bio Commented out as user can add arbritrary domains to their bio
+                          email # Email is verified by github
+                          websiteUrl # Website is not verified by github
+                        }
+                    }
+                }
+            }
+        }"""
+    query_2 = """query search_users {
+            search(query: "blacklanternsecurity.com", type: USER, first: 100, after: "Y3Vyc29yOjUz") {
+                userCount
+                pageInfo {
+                    hasNextPage
+                    endCursor
+                }
+                edges {
+                    node {
+                        ... on User {
+                          login
+                          # bio Commented out as user can add arbritrary domains to their bio
+                          email # Email is verified by github
+                          websiteUrl # Website is not verified by github
+                        }
+                    }
+                }
+            }
+        }"""
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(url="https://api.github.com/zen")
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/graphql",
+            match_headers={"Authorization": "token asdf"},
+            match_json={"query": self.query_1},
+            json={
+                "data": {
+                    "search": {
+                        "userCount": 2,
+                        "pageInfo": {"hasNextPage": True, "endCursor": "Y3Vyc29yOjUz"},
+                        "edges": [
+                            {
+                                "node": {
+                                    "login": "user_one",
+                                    "email": "test@blacklanternsecurity.com",
+                                    "websiteUrl": None,
+                                }
+                            },
+                            {"node": {"login": "user_two", "email": None, "websiteUrl": None}},
+                        ],
+                    }
+                }
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/graphql",
+            match_headers={"Authorization": "token asdf"},
+            match_json={"query": self.query_2},
+            json={
+                "data": {
+                    "search": {
+                        "userCount": 1,
+                        "pageInfo": {"hasNextPage": False, "endCursor": "Y3Vyc29yOjU"},
+                        "edges": [
+                            {
+                                "node": {
+                                    "login": "user_three",
+                                    "email": None,
+                                    "websiteUrl": "https://blog.blacklanternsecurity.com",
+                                }
+                            }
+                        ],
+                    }
+                }
+            },
+        )
+
+    def check(self, module_test, events):
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "user_one"
+                and str(e.module) == "github_usersearch"
+                and "github-org-member" in e.tags
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find user_one github"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "EMAIL_ADDRESS"
+                and e.data == "test@blacklanternsecurity.com"
+                and str(e.module) == "github_usersearch"
+            ]
+        ), "Failed to find email address for user_one"
+        assert 0 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "user_two"
+                and str(e.module) == "github_usersearch"
+                and "github-org-member" in e.tags
+                and e.scope_distance == 1
+            ]
+        ), "user_two should not be in scope due to no email or website"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "user_three"
+                and str(e.module) == "github_usersearch"
+                and "github-org-member" in e.tags
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find user_three github"

{bbot-2.4.2.6611rc0 → bbot-2.4.2.6621rc0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bbot"
-version = "v2.4.2.6611rc"
+version = "v2.4.2.6621rc"
 description = "OSINT automation for hackers."
 authors = [
     "TheTechromancer",