bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/test/test_step_2/module_tests/test_module_trufflehog.py

@@ -9,6 +9,7 @@ from .base import ModuleTestBase
 
 
 class TestTrufflehog(ModuleTestBase):
+    config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}}
     modules_overrides = [
         "github_org",
         "speculate",
@@ -16,6 +17,8 @@ class TestTrufflehog(ModuleTestBase):
         "github_workflows",
         "dockerhub",
         "docker_pull",
+        "postman",
+        "postman_download",
         "trufflehog",
     ]
 
@@ -23,6 +26,37 @@ class TestTrufflehog(ModuleTestBase):
 
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(url="https://api.github.com/zen")
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/me",
+            json={
+                "user": {
+                    "id": 000000,
+                    "username": "test_key",
+                    "email": "blacklanternsecurity@test.com",
+                    "fullName": "Test Key",
+                    "avatar": "",
+                    "isPublic": True,
+                    "teamId": 0,
+                    "teamDomain": "",
+                    "roles": ["user"],
+                },
+                "operations": [
+                    {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0},
+                    {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0},
+                    {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0},
+                    {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0},
+                    {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0},
+                    {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0},
+                    {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0},
+                    {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0},
+                    {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0},
+                    {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0},
+                ],
+            },
+        )
         module_test.httpx_mock.add_response(
             url="https://api.github.com/orgs/blacklanternsecurity",
             json={
@@ -812,6 +846,248 @@ class TestTrufflehog(ModuleTestBase):
         )
 
     async def setup_after_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/ws/proxy",
+            match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}',
+            json={
+                "data": [
+                    {
+                        "score": 611.41156,
+                        "normalizedScore": 23,
+                        "document": {
+                            "watcherCount": 6,
+                            "apiCount": 0,
+                            "forkCount": 0,
+                            "isblacklisted": "false",
+                            "createdAt": "2021-06-15T14:03:51",
+                            "publishertype": "team",
+                            "publisherHandle": "blacklanternsecurity",
+                            "id": "11498add-357d-4bc5-a008-0a2d44fb8829",
+                            "slug": "bbot-public",
+                            "updatedAt": "2024-07-30T11:00:35",
+                            "entityType": "workspace",
+                            "visibilityStatus": "public",
+                            "forkcount": "0",
+                            "tags": [],
+                            "createdat": "2021-06-15T14:03:51",
+                            "forkLabel": "",
+                            "publisherName": "blacklanternsecurity",
+                            "name": "BlackLanternSecurity BBOT [Public]",
+                            "dependencyCount": 7,
+                            "collectionCount": 6,
+                            "warehouse__updated_at": "2024-07-30 11:00:00",
+                            "privateNetworkFolders": [],
+                            "isPublisherVerified": False,
+                            "publisherType": "team",
+                            "curatedInList": [],
+                            "creatorId": "6900157",
+                            "description": "",
+                            "forklabel": "",
+                            "publisherId": "299401",
+                            "publisherLogo": "",
+                            "popularity": 5,
+                            "isPublic": True,
+                            "categories": [],
+                            "universaltags": "",
+                            "views": 5788,
+                            "summary": "BLS public workspaces.",
+                            "memberCount": 2,
+                            "isBlacklisted": False,
+                            "publisherid": "299401",
+                            "isPrivateNetworkEntity": False,
+                            "isDomainNonTrivial": True,
+                            "privateNetworkMeta": "",
+                            "updatedat": "2021-10-20T16:19:29",
+                            "documentType": "workspace",
+                        },
+                        "highlight": {"summary": "<b>BLS</b> BBOT api test."},
+                    },
+                ],
+                "meta": {
+                    "queryText": "blacklanternsecurity",
+                    "total": {
+                        "collection": 0,
+                        "request": 0,
+                        "workspace": 1,
+                        "api": 0,
+                        "team": 0,
+                        "user": 0,
+                        "flow": 0,
+                        "apiDefinition": 0,
+                        "privateNetworkFolder": 0,
+                    },
+                    "state": "AQ4",
+                    "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None},
+                    "featureFlags": {
+                        "enabledPublicResultCuration": True,
+                        "boostByPopularity": True,
+                        "reRankPostNormalization": True,
+                        "enableUrlBarHostNameSearch": True,
+                    },
+                },
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/ws/proxy",
+            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}',
+            json={
+                "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
+                "data": [
+                    {
+                        "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                        "name": "BlackLanternSecurity BBOT [Public]",
+                        "description": None,
+                        "summary": "BLS public workspaces.",
+                        "createdBy": "299401",
+                        "updatedBy": "299401",
+                        "team": None,
+                        "createdAt": "2021-10-20T16:19:29",
+                        "updatedAt": "2021-10-20T16:19:29",
+                        "visibilityStatus": "public",
+                        "profileInfo": {
+                            "slug": "bbot-public",
+                            "profileType": "team",
+                            "profileId": "000000",
+                            "publicHandle": "https://www.postman.com/blacklanternsecurity",
+                            "publicImageURL": "",
+                            "publicName": "BlackLanternSecurity",
+                            "isVerified": False,
+                        },
+                    }
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+            json={
+                "workspace": {
+                    "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                    "name": "BlackLanternSecurity BBOT [Public]",
+                    "type": "personal",
+                    "description": None,
+                    "visibility": "public",
+                    "createdBy": "00000000",
+                    "updatedBy": "00000000",
+                    "createdAt": "2021-11-17T06:09:01.000Z",
+                    "updatedAt": "2021-11-17T08:57:16.000Z",
+                    "collections": [
+                        {
+                            "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+                            "name": "BBOT Public",
+                            "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+                        },
+                    ],
+                    "environments": [
+                        {
+                            "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                            "name": "BBOT Test",
+                            "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                        }
+                    ],
+                    "apis": [],
+                }
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals",
+            json={
+                "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e",
+                "meta": {"model": "globals", "action": "find"},
+                "data": {
+                    "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                    "lastUpdatedBy": "00000000",
+                    "lastRevision": 1637239113000,
+                    "id": "8be7574b-219f-49e0-8d25-da447a882e4e",
+                    "values": [
+                        {
+                            "key": "endpoint_url",
+                            "value": "https://api.blacklanternsecurity.com/",
+                            "enabled": True,
+                        },
+                    ],
+                    "createdAt": "2021-11-17T06:09:01.000Z",
+                    "updatedAt": "2021-11-18T12:38:33.000Z",
+                },
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089",
+            json={
+                "environment": {
+                    "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                    "name": "BBOT Test",
+                    "owner": "00000000",
+                    "createdAt": "2021-11-17T06:29:54.000Z",
+                    "updatedAt": "2021-11-23T07:06:53.000Z",
+                    "values": [
+                        {
+                            "key": "temp_session_endpoint",
+                            "value": "https://api.blacklanternsecurity.com/",
+                            "enabled": True,
+                        },
+                    ],
+                    "isPublic": True,
+                }
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+            json={
+                "collection": {
+                    "info": {
+                        "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0",
+                        "name": "BBOT Public",
+                        "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+                        "updatedAt": "2021-11-17T07:13:16.000Z",
+                        "createdAt": "2021-11-17T07:13:15.000Z",
+                        "lastUpdatedBy": "00000000",
+                        "uid": "172983-62b91565-d2e2-4bcd-8248-4dba2e3452f0",
+                    },
+                    "item": [
+                        {
+                            "name": "Generate API Session",
+                            "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                            "protocolProfileBehavior": {"disableBodyPruning": True},
+                            "request": {
+                                "method": "POST",
+                                "header": [{"key": "Content-Type", "value": "application/json"}],
+                                "body": {
+                                    "mode": "raw",
+                                    "raw": '{"username": "test", "password": "Test"}',
+                                },
+                                "url": {
+                                    "raw": "https://admin:admin@the-internet.herokuapp.com/basic_auth",
+                                    "host": ["https://admin:admin@the-internet.herokuapp.com/basic_auth"],
+                                },
+                                "description": "",
+                            },
+                            "response": [],
+                            "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                        },
+                        {
+                            "name": "Generate API Session",
+                            "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                            "protocolProfileBehavior": {"disableBodyPruning": True},
+                            "request": {
+                                "method": "POST",
+                                "header": [{"key": "Content-Type", "value": "application/json"}],
+                                "body": {
+                                    "mode": "raw",
+                                    "raw": '{"username": "test", "password": "Test"}',
+                                },
+                                "url": {
+                                    "raw": "https://admin:admin@internal.host.com",
+                                    "host": ["https://admin:admin@internal.host.com"],
+                                },
+                                "description": "",
+                            },
+                            "response": [],
+                            "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                        },
+                    ],
+                }
+            },
+        )
         temp_path = Path("/tmp/.bbot_test")
         temp_repo_path = temp_path / "test_keys"
         shutil.rmtree(temp_repo_path, ignore_errors=True)
@@ -849,21 +1125,26 @@ class TestTrufflehog(ModuleTestBase):
             e
             for e in events
             if e.type == "VULNERABILITY"
-            and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com")
+            and (
+                e.data["host"] == "hub.docker.com"
+                or e.data["host"] == "github.com"
+                or e.data["host"] == "www.postman.com"
+            )
             and "Verified Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"]
             and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"]
         ]
-        assert 3 == len(vuln_events), "Failed to find secret in events"
+        # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.
+        assert 4 == len(vuln_events), "Failed to find secret in events"
         github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.parent for e in vuln_events if "bbot" in e.data["description"]]
-        assert len(filesystem_events) == 3
-        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        filesystem_events = [e.parent for e in vuln_events]
+        assert len(filesystem_events) == 4
+        assert all(e.type == "FILESYSTEM" for e in filesystem_events)
         assert 1 == len(
             [
                 e
@@ -887,30 +1168,45 @@ class TestTrufflehog(ModuleTestBase):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Docker image file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith(
+                    "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip"
+                )
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Failed to find blacklanternsecurity postman workspace"
 
 
 class TestTrufflehog_NonVerified(TestTrufflehog):
-    config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
+    config_overrides = {"modules": {"trufflehog": {"only_verified": False}, "postman_download": {"api_key": "asdf"}}}
 
     def check(self, module_test, events):
         finding_events = [
             e
             for e in events
             if e.type == e.type == "FINDING"
-            and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com")
+            and (
+                e.data["host"] == "hub.docker.com"
+                or e.data["host"] == "github.com"
+                or e.data["host"] == "www.postman.com"
+            )
             and "Potential Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
-        assert 3 == len(finding_events), "Failed to find secret in events"
+        # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.
+        assert 4 == len(finding_events), "Failed to find secret in events"
         github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.parent for e in finding_events if "bbot" in e.data["description"]]
-        assert len(filesystem_events) == 3
-        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        filesystem_events = [e.parent for e in finding_events]
+        assert len(filesystem_events) == 4
+        assert all(e.type == "FILESYSTEM" for e in filesystem_events)
         assert 1 == len(
             [
                 e
@@ -934,3 +1230,13 @@ class TestTrufflehog_NonVerified(TestTrufflehog):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Docker image file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith(
+                    "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip"
+                )
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Failed to find blacklanternsecurity postman workspace"

bbot/test/test_step_2/module_tests/test_module_wayback.py

@@ -4,7 +4,7 @@ from .base import ModuleTestBase
 class TestWayback(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         module_test.httpx_mock.add_response(
-            url=f"http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original",
+            url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original",
             json=[["original"], ["http://asdf.blacklanternsecurity.com"]],
         )
 

bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py

@@ -84,3 +84,138 @@ class TestSubdomainEnumLowestParent(TestSubdomainEnumHighestParent):
             "asdf.www.blacklanternsecurity.com",
             "www.blacklanternsecurity.com",
         }
+
+
+class TestSubdomainEnumWildcardBaseline(ModuleTestBase):
+    # oh walmart.cn why are you like this
+    targets = ["www.walmart.cn"]
+    whitelist = ["walmart.cn"]
+    modules_overrides = []
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}, "omit_event_types": []}
+    dedup_strategy = "highest_parent"
+
+    dns_mock_data = {
+        "walmart.cn": {"A": ["127.0.0.1"]},
+        "www.walmart.cn": {"A": ["127.0.0.1"]},
+        "test.walmart.cn": {"A": ["127.0.0.1"]},
+    }
+
+    async def setup_before_prep(self, module_test):
+        await module_test.mock_dns(self.dns_mock_data)
+        self.queries = []
+
+        async def mock_query(query):
+            self.queries.append(query)
+            return ["walmart.cn", "www.walmart.cn", "test.walmart.cn", "asdf.walmart.cn"]
+
+        # load subdomain enum template as module
+        from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+        subdomain_enum_module = subdomain_enum(module_test.scan)
+
+        subdomain_enum_module.query = mock_query
+        subdomain_enum_module._name = "subdomain_enum"
+        subdomain_enum_module.dedup_strategy = self.dedup_strategy
+        module_test.scan.modules["subdomain_enum"] = subdomain_enum_module
+
+    def check(self, module_test, events):
+        assert self.queries == ["walmart.cn"]
+        assert len(events) == 7
+        assert 2 == len(
+            [
+                e
+                for e in events
+                if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A" and e.scope_distance == 1
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "www.walmart.cn"
+                and str(e.module) == "TARGET"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "test.walmart.cn"
+                and str(e.module) == "subdomain_enum"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME_UNRESOLVED"
+                and e.data == "asdf.walmart.cn"
+                and str(e.module) == "subdomain_enum"
+                and e.scope_distance == 0
+            ]
+        )
+
+
+class TestSubdomainEnumWildcardDefense(TestSubdomainEnumWildcardBaseline):
+    # oh walmart.cn why are you like this
+    targets = ["walmart.cn"]
+    modules_overrides = []
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}}
+    dedup_strategy = "highest_parent"
+
+    dns_mock_data = {
+        "walmart.cn": {"A": ["127.0.0.2"], "TXT": ["asdf.walmart.cn"]},
+    }
+
+    async def setup_after_prep(self, module_test):
+        # simulate wildcard
+        custom_lookup = """
+def custom_lookup(query, rdtype):
+    import random
+    if rdtype == "A" and query.endswith(".walmart.cn"):
+        ip = ".".join([str(random.randint(0,256)) for _ in range(4)])
+        return {ip}
+"""
+        await module_test.mock_dns(self.dns_mock_data, custom_lookup_fn=custom_lookup)
+
+    def check(self, module_test, events):
+        # no subdomain enum should happen on this domain!
+        assert self.queries == []
+        assert len(events) == 7
+        assert 2 == len(
+            [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "walmart.cn"
+                and str(e.module) == "TARGET"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "asdf.walmart.cn"
+                and str(e.module) == "TXT"
+                and e.scope_distance == 0
+                and "wildcard-possible" in e.tags
+                and "a-wildcard-possible" in e.tags
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "RAW_DNS_RECORD"
+                and e.data == {"host": "walmart.cn", "type": "TXT", "answer": '"asdf.walmart.cn"'}
+            ]
+        )