bbot 2.0.1.4720rc0__py3-none-any.whl → 2.3.0.5401rc0__py3-none-any.whl

bbot/test/test_step_2/module_tests/test_module_trufflehog.py

@@ -9,14 +9,16 @@ from .base import ModuleTestBase
 
 
 class TestTrufflehog(ModuleTestBase):
+    config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}}
     modules_overrides = [
         "github_org",
         "speculate",
         "git_clone",
-        "unstructured",
         "github_workflows",
         "dockerhub",
         "docker_pull",
+        "postman",
+        "postman_download",
         "trufflehog",
     ]
 
@@ -24,6 +26,37 @@ class TestTrufflehog(ModuleTestBase):
 
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(url="https://api.github.com/zen")
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/me",
+            json={
+                "user": {
+                    "id": 000000,
+                    "username": "test_key",
+                    "email": "blacklanternsecurity@test.com",
+                    "fullName": "Test Key",
+                    "avatar": "",
+                    "isPublic": True,
+                    "teamId": 0,
+                    "teamDomain": "",
+                    "roles": ["user"],
+                },
+                "operations": [
+                    {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0},
+                    {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0},
+                    {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0},
+                    {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0},
+                    {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0},
+                    {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0},
+                    {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0},
+                    {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0},
+                    {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0},
+                    {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0},
+                    {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0},
+                ],
+            },
+        )
         module_test.httpx_mock.add_response(
             url="https://api.github.com/orgs/blacklanternsecurity",
             json={
@@ -813,6 +846,248 @@ class TestTrufflehog(ModuleTestBase):
         )
 
     async def setup_after_prep(self, module_test):
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/ws/proxy",
+            match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}',
+            json={
+                "data": [
+                    {
+                        "score": 611.41156,
+                        "normalizedScore": 23,
+                        "document": {
+                            "watcherCount": 6,
+                            "apiCount": 0,
+                            "forkCount": 0,
+                            "isblacklisted": "false",
+                            "createdAt": "2021-06-15T14:03:51",
+                            "publishertype": "team",
+                            "publisherHandle": "blacklanternsecurity",
+                            "id": "11498add-357d-4bc5-a008-0a2d44fb8829",
+                            "slug": "bbot-public",
+                            "updatedAt": "2024-07-30T11:00:35",
+                            "entityType": "workspace",
+                            "visibilityStatus": "public",
+                            "forkcount": "0",
+                            "tags": [],
+                            "createdat": "2021-06-15T14:03:51",
+                            "forkLabel": "",
+                            "publisherName": "blacklanternsecurity",
+                            "name": "BlackLanternSecurity BBOT [Public]",
+                            "dependencyCount": 7,
+                            "collectionCount": 6,
+                            "warehouse__updated_at": "2024-07-30 11:00:00",
+                            "privateNetworkFolders": [],
+                            "isPublisherVerified": False,
+                            "publisherType": "team",
+                            "curatedInList": [],
+                            "creatorId": "6900157",
+                            "description": "",
+                            "forklabel": "",
+                            "publisherId": "299401",
+                            "publisherLogo": "",
+                            "popularity": 5,
+                            "isPublic": True,
+                            "categories": [],
+                            "universaltags": "",
+                            "views": 5788,
+                            "summary": "BLS public workspaces.",
+                            "memberCount": 2,
+                            "isBlacklisted": False,
+                            "publisherid": "299401",
+                            "isPrivateNetworkEntity": False,
+                            "isDomainNonTrivial": True,
+                            "privateNetworkMeta": "",
+                            "updatedat": "2021-10-20T16:19:29",
+                            "documentType": "workspace",
+                        },
+                        "highlight": {"summary": "<b>BLS</b> BBOT api test."},
+                    },
+                ],
+                "meta": {
+                    "queryText": "blacklanternsecurity",
+                    "total": {
+                        "collection": 0,
+                        "request": 0,
+                        "workspace": 1,
+                        "api": 0,
+                        "team": 0,
+                        "user": 0,
+                        "flow": 0,
+                        "apiDefinition": 0,
+                        "privateNetworkFolder": 0,
+                    },
+                    "state": "AQ4",
+                    "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None},
+                    "featureFlags": {
+                        "enabledPublicResultCuration": True,
+                        "boostByPopularity": True,
+                        "reRankPostNormalization": True,
+                        "enableUrlBarHostNameSearch": True,
+                    },
+                },
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/ws/proxy",
+            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}',
+            json={
+                "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
+                "data": [
+                    {
+                        "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                        "name": "BlackLanternSecurity BBOT [Public]",
+                        "description": None,
+                        "summary": "BLS public workspaces.",
+                        "createdBy": "299401",
+                        "updatedBy": "299401",
+                        "team": None,
+                        "createdAt": "2021-10-20T16:19:29",
+                        "updatedAt": "2021-10-20T16:19:29",
+                        "visibilityStatus": "public",
+                        "profileInfo": {
+                            "slug": "bbot-public",
+                            "profileType": "team",
+                            "profileId": "000000",
+                            "publicHandle": "https://www.postman.com/blacklanternsecurity",
+                            "publicImageURL": "",
+                            "publicName": "BlackLanternSecurity",
+                            "isVerified": False,
+                        },
+                    }
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+            json={
+                "workspace": {
+                    "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                    "name": "BlackLanternSecurity BBOT [Public]",
+                    "type": "personal",
+                    "description": None,
+                    "visibility": "public",
+                    "createdBy": "00000000",
+                    "updatedBy": "00000000",
+                    "createdAt": "2021-11-17T06:09:01.000Z",
+                    "updatedAt": "2021-11-17T08:57:16.000Z",
+                    "collections": [
+                        {
+                            "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+                            "name": "BBOT Public",
+                            "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+                        },
+                    ],
+                    "environments": [
+                        {
+                            "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                            "name": "BBOT Test",
+                            "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                        }
+                    ],
+                    "apis": [],
+                }
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals",
+            json={
+                "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e",
+                "meta": {"model": "globals", "action": "find"},
+                "data": {
+                    "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b",
+                    "lastUpdatedBy": "00000000",
+                    "lastRevision": 1637239113000,
+                    "id": "8be7574b-219f-49e0-8d25-da447a882e4e",
+                    "values": [
+                        {
+                            "key": "endpoint_url",
+                            "value": "https://api.blacklanternsecurity.com/",
+                            "enabled": True,
+                        },
+                    ],
+                    "createdAt": "2021-11-17T06:09:01.000Z",
+                    "updatedAt": "2021-11-18T12:38:33.000Z",
+                },
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089",
+            json={
+                "environment": {
+                    "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089",
+                    "name": "BBOT Test",
+                    "owner": "00000000",
+                    "createdAt": "2021-11-17T06:29:54.000Z",
+                    "updatedAt": "2021-11-23T07:06:53.000Z",
+                    "values": [
+                        {
+                            "key": "temp_session_endpoint",
+                            "value": "https://api.blacklanternsecurity.com/",
+                            "enabled": True,
+                        },
+                    ],
+                    "isPublic": True,
+                }
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f",
+            json={
+                "collection": {
+                    "info": {
+                        "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0",
+                        "name": "BBOT Public",
+                        "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+                        "updatedAt": "2021-11-17T07:13:16.000Z",
+                        "createdAt": "2021-11-17T07:13:15.000Z",
+                        "lastUpdatedBy": "00000000",
+                        "uid": "172983-62b91565-d2e2-4bcd-8248-4dba2e3452f0",
+                    },
+                    "item": [
+                        {
+                            "name": "Generate API Session",
+                            "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                            "protocolProfileBehavior": {"disableBodyPruning": True},
+                            "request": {
+                                "method": "POST",
+                                "header": [{"key": "Content-Type", "value": "application/json"}],
+                                "body": {
+                                    "mode": "raw",
+                                    "raw": '{"username": "test", "password": "Test"}',
+                                },
+                                "url": {
+                                    "raw": "https://admin:admin@the-internet.herokuapp.com/basic_auth",
+                                    "host": ["https://admin:admin@the-internet.herokuapp.com/basic_auth"],
+                                },
+                                "description": "",
+                            },
+                            "response": [],
+                            "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                        },
+                        {
+                            "name": "Generate API Session",
+                            "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                            "protocolProfileBehavior": {"disableBodyPruning": True},
+                            "request": {
+                                "method": "POST",
+                                "header": [{"key": "Content-Type", "value": "application/json"}],
+                                "body": {
+                                    "mode": "raw",
+                                    "raw": '{"username": "test", "password": "Test"}',
+                                },
+                                "url": {
+                                    "raw": "https://admin:admin@internal.host.com",
+                                    "host": ["https://admin:admin@internal.host.com"],
+                                },
+                                "description": "",
+                            },
+                            "response": [],
+                            "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1",
+                        },
+                    ],
+                }
+            },
+        )
         temp_path = Path("/tmp/.bbot_test")
         temp_repo_path = temp_path / "test_keys"
         shutil.rmtree(temp_repo_path, ignore_errors=True)
@@ -850,22 +1125,26 @@ class TestTrufflehog(ModuleTestBase):
             e
             for e in events
             if e.type == "VULNERABILITY"
-            and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com")
+            and (
+                e.data["host"] == "hub.docker.com"
+                or e.data["host"] == "github.com"
+                or e.data["host"] == "www.postman.com"
+            )
             and "Verified Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"]
             and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"]
         ]
-        # Trufflehog should find 3 verifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder
-        assert 3 == len(vuln_events), "Failed to find secret in events"
+        # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.
+        assert 4 == len(vuln_events), "Failed to find secret in events"
         github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.parent for e in vuln_events if "bbot" in e.data["description"]]
-        assert len(filesystem_events) == 3
-        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        filesystem_events = [e.parent for e in vuln_events]
+        assert len(filesystem_events) == 4
+        assert all(e.type == "FILESYSTEM" for e in filesystem_events)
         assert 1 == len(
             [
                 e
@@ -889,31 +1168,45 @@ class TestTrufflehog(ModuleTestBase):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Docker image file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith(
+                    "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip"
+                )
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Failed to find blacklanternsecurity postman workspace"
 
 
 class TestTrufflehog_NonVerified(TestTrufflehog):
-    config_overrides = {"modules": {"trufflehog": {"only_verified": False}}}
+    config_overrides = {"modules": {"trufflehog": {"only_verified": False}, "postman_download": {"api_key": "asdf"}}}
 
     def check(self, module_test, events):
         finding_events = [
             e
             for e in events
             if e.type == e.type == "FINDING"
-            and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com")
+            and (
+                e.data["host"] == "hub.docker.com"
+                or e.data["host"] == "github.com"
+                or e.data["host"] == "www.postman.com"
+            )
             and "Potential Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
-        # Trufflehog should find 3 unverifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder
-        assert 3 == len(finding_events), "Failed to find secret in events"
+        # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.
+        assert 4 == len(finding_events), "Failed to find secret in events"
         github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.parent for e in finding_events if "bbot" in e.data["description"]]
-        assert len(filesystem_events) == 3
-        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        filesystem_events = [e.parent for e in finding_events]
+        assert len(filesystem_events) == 4
+        assert all(e.type == "FILESYSTEM" for e in filesystem_events)
         assert 1 == len(
             [
                 e
@@ -937,3 +1230,13 @@ class TestTrufflehog_NonVerified(TestTrufflehog):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Docker image file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith(
+                    "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip"
+                )
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Failed to find blacklanternsecurity postman workspace"

bbot/test/test_step_2/module_tests/test_module_viewdns.py

@@ -66,7 +66,7 @@ web_body = """<html>
          <tr>
             <td>
                <font size="2">
-                  <a href="/" style="color: #00721e;">ViewDNS.info</a> > <a href="/" style="color: #00721e;">Tools</a> > 
+                  <a href="/" style="color: #00721e;">ViewDNS.info</a> > <a href="/" style="color: #00721e;">Tools</a> >
                   <H1 style="font-size: 16; display: inline;">Reverse Whois Lookup</H1>
                   <br><br>This free tool will allow you to find domain names owned by an individual person or company.  Simply enter the email address or name of the person or company to find other domains registered using those same details. <a href="#" onclick="javascript:document.getElementById('faq').style.visibility = 'visible'; document.getElementById('faq').style.display = 'block';"  style="color: #00721e;">FAQ</a>.<br><br>
                   <div id="faq" style="visibility: hidden; display: none;"><u><b>Frequently Asked Questions</b></u><br>Q. Will this tool return results for all domains including ccTLD's?<br>A. Unfortunately no.  Whilst we do our best to ensure our data is as complete as possible, we are not able to return results for all ccTLD's.  Due to a number of technical limitations with whois data, the results from any Reverse Whois tool should not be considered as exhaustive.<br><br>Q. Is your data live?<br>A. Our data is not live.  We do our best to update the data as often as possible with daily updates for selected TLD's and quarterly updates for others.<br><br>Q. How do I see all records for a specific person/company rather than the limited number you show on your site?<br>A. Please <a href="mailto:feedback@viewdns.info?subject=Reverse Whois" style="color: #00721e;">email us</a> with your request and we'll see what we can do for you.<br><br></div>

bbot/test/test_step_2/module_tests/test_module_wayback.py

@@ -4,7 +4,7 @@ from .base import ModuleTestBase
 class TestWayback(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         module_test.httpx_mock.add_response(
-            url=f"http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original",
+            url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original",
             json=[["original"], ["http://asdf.blacklanternsecurity.com"]],
         )
 

bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py

@@ -120,7 +120,7 @@ class TestSubdomainEnumWildcardBaseline(ModuleTestBase):
 
     def check(self, module_test, events):
         assert self.queries == ["walmart.cn"]
-        assert len(events) == 6
+        assert len(events) == 7
         assert 2 == len(
             [
                 e
@@ -185,7 +185,7 @@ def custom_lookup(query, rdtype):
     def check(self, module_test, events):
         # no subdomain enum should happen on this domain!
         assert self.queries == []
-        assert len(events) == 6
+        assert len(events) == 7
         assert 2 == len(
             [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1]
         )

bbot/wordlists/devops_mutations.txt

@@ -105,4 +105,4 @@ store
 home
 production
 auto
-cn
+cn

bbot/wordlists/ffuf_shortname_candidates.txt

@@ -107979,4 +107979,4 @@ zzz
 zzzindex
 zzztest
 zzzz
-zzzzz
+zzzzz

bbot/wordlists/nameservers.txt

@@ -2370,4 +2370,4 @@
 8.25.185.131
 203.39.3.133
 118.69.187.252
-108.56.80.135
+108.56.80.135

bbot/wordlists/paramminer_headers.txt

@@ -1147,4 +1147,4 @@ http_sm_userdn
 http_sm_usermsg
 x-remote-ip
 traceparent
-tracestate
+tracestate

bbot/wordlists/paramminer_parameters.txt

@@ -6520,4 +6520,4 @@ shell_path
 user_token
 adminCookie
 fullapp
-LandingUrl
+LandingUrl

bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt

@@ -830,4 +830,4 @@
 .z
 .zdat
 .zif
-.zip
+.zip

bbot/wordlists/valid_url_schemes.txt

@@ -379,4 +379,4 @@ xri
 ymsgr
 z39.50
 z39.50r
-z39.50s
+z39.50s