bbot 2.0.1.4654rc0__py3-none-any.whl → 2.3.0.5397rc0__py3-none-any.whl

bbot/test/test_step_2/module_tests/test_module_passivetotal.py

@@ -2,15 +2,17 @@ from .base import ModuleTestBase
 
 
 class TestPassiveTotal(ModuleTestBase):
-    config_overrides = {"modules": {"passivetotal": {"username": "jon@bls.fakedomain", "api_key": "asdf"}}}
+    config_overrides = {"modules": {"passivetotal": {"api_key": "jon@bls.fakedomain:asdf"}}}
 
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://api.passivetotal.org/v2/account/quota",
+            match_headers={"Authorization": "Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY="},
             json={"user": {"counts": {"search_api": 10}, "limits": {"search_api": 20}}},
         )
         module_test.httpx_mock.add_response(
             url="https://api.passivetotal.org/v2/enrichment/subdomains?query=blacklanternsecurity.com",
+            match_headers={"Authorization": "Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY="},
             json={"subdomains": ["asdf"]},
         )
 

bbot/test/test_step_2/module_tests/test_module_portscan.py

@@ -21,7 +21,6 @@ class TestPortscan(ModuleTestBase):
     masscan_output_ping = """{   "ip": "8.8.8.8",   "timestamp": "1719862594", "ports": [ {"port": 0, "proto": "icmp", "status": "open", "reason": "none", "ttl": 54} ] }"""
 
     async def setup_after_prep(self, module_test):
-
         from bbot.modules.base import BaseModule
 
         class DummyModule(BaseModule):
@@ -109,10 +108,12 @@ class TestPortscan(ModuleTestBase):
                 if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module"
             ]
         )
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3
-        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3
+        # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first
+        # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port.
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 4
+        assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 4
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"])
@@ -121,7 +122,7 @@ class TestPortscan(ModuleTestBase):
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.net:80"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.asdf.evilcorp.net:80"])
         assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.evilcorp.com:631"])
-        assert not any([e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com"])
+        assert not any(e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com")
 
 
 class TestPortscanPingFirst(TestPortscan):
@@ -135,7 +136,7 @@ class TestPortscanPingFirst(TestPortscan):
         assert self.ping_runs == 1
         open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"]
         assert len(open_port_events) == 3
-        assert set([e.data for e in open_port_events]) == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"}
+        assert {e.data for e in open_port_events} == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"}
 
 
 class TestPortscanPingOnly(TestPortscan):
@@ -153,4 +154,4 @@ class TestPortscanPingOnly(TestPortscan):
         assert len(open_port_events) == 0
         ip_events = [e for e in events if e.type == "IP_ADDRESS"]
         assert len(ip_events) == 1
-        assert set([e.data for e in ip_events]) == {"8.8.8.8"}
+        assert {e.data for e in ip_events} == {"8.8.8.8"}

bbot/test/test_step_2/module_tests/test_module_postgres.py

@@ -0,0 +1,74 @@
+import time
+import asyncio
+
+from .base import ModuleTestBase
+
+
+class TestPostgres(ModuleTestBase):
+    targets = ["evilcorp.com"]
+    skip_distro_tests = True
+
+    async def setup_before_prep(self, module_test):
+        process = await asyncio.create_subprocess_exec(
+            "docker",
+            "run",
+            "--name",
+            "bbot-test-postgres",
+            "--rm",
+            "-e",
+            "POSTGRES_PASSWORD=bbotislife",
+            "-e",
+            "POSTGRES_USER=postgres",
+            "-p",
+            "5432:5432",
+            "-d",
+            "postgres",
+        )
+
+        import asyncpg
+
+        # wait for the container to start
+        start_time = time.time()
+        while True:
+            try:
+                # Connect to the default 'postgres' database to create 'bbot'
+                conn = await asyncpg.connect(
+                    user="postgres", password="bbotislife", database="postgres", host="127.0.0.1"
+                )
+                await conn.execute("CREATE DATABASE bbot")
+                await conn.close()
+                break
+            except asyncpg.exceptions.DuplicateDatabaseError:
+                # If the database already exists, break the loop
+                break
+            except Exception as e:
+                if time.time() - start_time > 60:  # timeout after 60 seconds
+                    self.log.error("PostgreSQL server did not start in time.")
+                    raise e
+                await asyncio.sleep(1)
+
+        if process.returncode != 0:
+            self.log.error("Failed to start PostgreSQL server")
+
+    async def check(self, module_test, events):
+        import asyncpg
+
+        # Connect to the PostgreSQL database
+        conn = await asyncpg.connect(user="postgres", password="bbotislife", database="bbot", host="127.0.0.1")
+
+        try:
+            events = await conn.fetch("SELECT * FROM event")
+            assert len(events) == 3, "No events found in PostgreSQL database"
+            scans = await conn.fetch("SELECT * FROM scan")
+            assert len(scans) == 1, "No scans found in PostgreSQL database"
+            targets = await conn.fetch("SELECT * FROM target")
+            assert len(targets) == 1, "No targets found in PostgreSQL database"
+        finally:
+            await conn.close()
+            process = await asyncio.create_subprocess_exec(
+                "docker", "stop", "bbot-test-postgres", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                raise Exception(f"Failed to stop PostgreSQL server: {stderr.decode()}")

bbot/test/test_step_2/module_tests/test_module_postman.py

@@ -2,281 +2,112 @@ from .base import ModuleTestBase
 
 
 class TestPostman(ModuleTestBase):
-    config_overrides = {
-        "omit_event_types": [],
-        "scope": {"report_distance": 1},
-    }
-
-    modules_overrides = ["postman", "httpx", "excavate"]
+    modules_overrides = ["postman", "speculate"]
 
     async def setup_after_prep(self, module_test):
+        await module_test.mock_dns(
+            {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}}
+        )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
             json={
                 "data": [
                     {
-                        "score": 499.22498,
-                        "normalizedScore": 8.43312276976538,
+                        "score": 611.41156,
+                        "normalizedScore": 23,
                         "document": {
-                            "isPublisherVerified": False,
-                            "publisherType": "user",
-                            "curatedInList": [],
-                            "publisherId": "28329861",
-                            "publisherHandle": "",
-                            "publisherLogo": "",
-                            "isPublic": True,
-                            "customHostName": "",
-                            "id": "28329861-28329861-f6ef-4f23-9f3a-8431f3567ac1",
-                            "workspaces": [
-                                {
-                                    "visibilityStatus": "public",
-                                    "name": "BlackLanternSecuritySpilledSecrets",
-                                    "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
-                                    "slug": "blacklanternsecurityspilledsecrets",
-                                }
-                            ],
-                            "collectionForkLabel": "",
-                            "method": "POST",
-                            "entityType": "request",
-                            "url": "www.example.com/index",
-                            "isBlacklisted": False,
-                            "warehouse__updated_at_collection": "2023-12-11 02:00:00",
-                            "isPrivateNetworkEntity": False,
-                            "warehouse__updated_at_request": "2023-12-11 02:00:00",
-                            "publisherName": "NA",
-                            "name": "A test post request",
-                            "privateNetworkMeta": "",
+                            "watcherCount": 6,
+                            "apiCount": 0,
+                            "forkCount": 0,
+                            "isblacklisted": "false",
+                            "createdAt": "2021-06-15T14:03:51",
+                            "publishertype": "team",
+                            "publisherHandle": "blacklanternsecurity",
+                            "id": "11498add-357d-4bc5-a008-0a2d44fb8829",
+                            "slug": "bbot-public",
+                            "updatedAt": "2024-07-30T11:00:35",
+                            "entityType": "workspace",
+                            "visibilityStatus": "public",
+                            "forkcount": "0",
+                            "tags": [],
+                            "createdat": "2021-06-15T14:03:51",
+                            "forkLabel": "",
+                            "publisherName": "blacklanternsecurity",
+                            "name": "BlackLanternSecurity BBOT [Public]",
+                            "dependencyCount": 7,
+                            "collectionCount": 6,
+                            "warehouse__updated_at": "2024-07-30 11:00:00",
                             "privateNetworkFolders": [],
-                            "documentType": "request",
-                            "collection": {
-                                "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                "name": "Secret Collection",
-                                "tags": [],
-                                "forkCount": 0,
-                                "watcherCount": 0,
-                                "views": 31,
-                                "apiId": "",
-                                "apiName": "",
-                            },
-                        },
-                    },
-                    {
-                        "score": 498.22398,
-                        "normalizedScore": 8.43312266976538,
-                        "document": {
                             "isPublisherVerified": False,
-                            "publisherType": "user",
+                            "publisherType": "team",
                             "curatedInList": [],
-                            "publisherId": "28329861",
-                            "publisherHandle": "",
+                            "creatorId": "6900157",
+                            "description": "",
+                            "forklabel": "",
+                            "publisherId": "299401",
                             "publisherLogo": "",
+                            "popularity": 5,
                             "isPublic": True,
-                            "customHostName": "",
-                            "id": "b7fa2137-b7fa2137-23bf-45d1-b176-35359af30ded",
-                            "workspaces": [
-                                {
-                                    "visibilityStatus": "public",
-                                    "name": "SpilledSecrets",
-                                    "id": "92d0451b-119d-4ef0-b74c-22c400e5ce05",
-                                    "slug": "spilledsecrets",
-                                }
-                            ],
-                            "collectionForkLabel": "",
-                            "method": "POST",
-                            "entityType": "request",
-                            "url": "www.example.com/index",
+                            "categories": [],
+                            "universaltags": "",
+                            "views": 5788,
+                            "summary": "BLS public workspaces.",
+                            "memberCount": 2,
                             "isBlacklisted": False,
-                            "warehouse__updated_at_collection": "2023-12-11 02:00:00",
+                            "publisherid": "299401",
                             "isPrivateNetworkEntity": False,
-                            "warehouse__updated_at_request": "2023-12-11 02:00:00",
-                            "publisherName": "NA",
-                            "name": "A test post request",
+                            "isDomainNonTrivial": True,
                             "privateNetworkMeta": "",
-                            "privateNetworkFolders": [],
-                            "documentType": "request",
-                            "collection": {
-                                "id": "007e8d67-007e8d67-932b-46ff-b95c-a2aa216edaf3",
-                                "name": "Secret Collection",
-                                "tags": [],
-                                "forkCount": 0,
-                                "watcherCount": 0,
-                                "views": 31,
-                                "apiId": "",
-                                "apiName": "",
-                            },
+                            "updatedat": "2021-10-20T16:19:29",
+                            "documentType": "workspace",
                         },
-                    },
+                        "highlight": {"summary": "<b>BLS</b> BBOT api test."},
+                    }
                 ],
-            },
-        )
-        module_test.httpx_mock.add_response(
-            url="https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f",
-            json={
-                "model_id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
-                "meta": {"model": "workspace", "action": "find"},
-                "data": {
-                    "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f",
-                    "name": "SpilledSecrets",
-                    "description": "A Mock workspace environment filled with fake secrets to act as a testing ground for secret scanners",
-                    "summary": "A Public workspace with mock secrets",
-                    "createdBy": "28329861",
-                    "updatedBy": "28329861",
-                    "team": "28329861",
-                    "createdAt": "2023-12-13T19:12:21.000Z",
-                    "updatedAt": "2023-12-13T19:15:07.000Z",
-                    "visibilityStatus": "public",
-                    "profileInfo": {
-                        "slug": "spilledsecrets",
-                        "profileType": "team",
-                        "profileId": "28329861",
-                        "publicHandle": "https://www.postman.com/lunar-pizza-28329861",
-                        "publicImageURL": "https://res.cloudinary.com/postman/image/upload/t_team_logo/v1/team/default-L4",
-                        "publicName": "lunar-pizza-28329861",
-                        "isVerified": False,
+                "meta": {
+                    "queryText": "blacklanternsecurity",
+                    "total": {
+                        "collection": 0,
+                        "request": 0,
+                        "workspace": 1,
+                        "api": 0,
+                        "team": 0,
+                        "user": 0,
+                        "flow": 0,
+                        "apiDefinition": 0,
+                        "privateNetworkFolder": 0,
                     },
-                    "user": "28329861",
-                    "type": "team",
-                    "dependencies": {
-                        "collections": ["28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"],
-                        "environments": ["28129865-fa7edca0-2df6-4187-9805-11845912f567"],
-                        "globals": ["28735eff-5ecd-43e6-8473-387f160f220f"],
+                    "state": "AQ4",
+                    "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None},
+                    "featureFlags": {
+                        "enabledPublicResultCuration": True,
+                        "boostByPopularity": True,
+                        "reRankPostNormalization": True,
+                        "enableUrlBarHostNameSearch": True,
                     },
-                    "members": {"users": {"28129865": {"id": "28129865"}}},
                 },
             },
         )
-        module_test.httpx_mock.add_response(
-            url="https://www.postman.com/_api/list/collection?workspace=afa061be-9cb0-4520-9d4d-fe63361daf0f",
-            json={
-                "data": [
-                    {
-                        "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                        "name": "Secret Collection",
-                        "folders_order": ["d6dd1092-94d4-462c-be48-4e3cad3b8612"],
-                        "order": ["67c9db4c-d0ed-461c-86d2-9a8c5a5de896"],
-                        "attributes": {
-                            "permissions": {
-                                "userCanUpdate": False,
-                                "userCanDelete": False,
-                                "userCanShare": False,
-                                "userCanCreateMock": False,
-                                "userCanCreateMonitor": False,
-                                "anybodyCanView": True,
-                                "teamCanView": True,
-                            },
-                            "fork": None,
-                            "parent": {"type": "workspace", "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f"},
-                            "flags": {"isArchived": False, "isFavorite": False},
-                        },
-                        "folders": [
-                            {
-                                "id": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
-                                "name": "Nested folder",
-                                "folder": None,
-                                "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                "folders_order": ["73a999c1-4246-4805-91bd-0232cc75958a"],
-                                "order": ["3aa78b71-2c4f-4299-94df-287ed1036409"],
-                                "folders": [
-                                    {
-                                        "id": "28129865-73a999c1-4246-4805-91bd-0232cc75958a",
-                                        "name": "Another Nested Folder",
-                                        "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
-                                        "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                        "folders_order": [],
-                                        "order": ["987c8ac8-bfa9-4bab-ade9-88ccf0597862"],
-                                        "folders": [],
-                                        "requests": [
-                                            {
-                                                "id": "28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862",
-                                                "name": "Delete User",
-                                                "method": "DELETE",
-                                                "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                                "folder": "28129865-73a999c1-4246-4805-91bd-0232cc75958a",
-                                                "responses_order": [],
-                                                "responses": [],
-                                            }
-                                        ],
-                                    }
-                                ],
-                                "requests": [
-                                    {
-                                        "id": "28129865-3aa78b71-2c4f-4299-94df-287ed1036409",
-                                        "name": "Login",
-                                        "method": "POST",
-                                        "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                        "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612",
-                                        "responses_order": [],
-                                        "responses": [],
-                                    }
-                                ],
-                            }
-                        ],
-                        "requests": [
-                            {
-                                "id": "28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896",
-                                "name": "Example Basic HTTP request",
-                                "method": "GET",
-                                "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205",
-                                "folder": None,
-                                "responses_order": [],
-                                "responses": [],
-                            }
-                        ],
-                    }
-                ]
-            },
-        )
-
-        old_emit_event = module_test.module.emit_event
-
-        async def new_emit_event(event_data, event_type, **kwargs):
-            if event_data.startswith("https://www.postman.com"):
-                event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888")
-            await old_emit_event(event_data, event_type, **kwargs)
-
-        module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event)
-        await module_test.mock_dns(
-            {"blacklanternsecurity.com": {"A": ["127.0.0.1"]}, "asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}}
-        )
-
-        request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862")
-        respond_args = dict(response_data="https://asdf.blacklanternsecurity.com")
-        module_test.set_expect_requests(request_args, respond_args)
 
     def check(self, module_test, events):
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events
-        ), "Failed to detect workspace"
-        assert any(
-            e.data != "http://127.0.0.1:8888/_api/workspace/92d0451b-119d-4ef0-b74c-22c400e5ce05" for e in events
-        ), "Workspace should not be detected"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals"
-            for e in events
-        ), "Failed to detect workspace globals"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567"
-            for e in events
-        ), "Failed to detect workspace environment"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"
-            for e in events
-        ), "Failed to detect collection"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
-            for e in events
-        ), "Failed to detect collection request #1"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409"
-            for e in events
-        ), "Failed to detect collection request #2"
-        assert any(
-            e.data == "http://127.0.0.1:8888/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896"
-            for e in events
-        ), "Failed to detect collection request #3"
-        assert any(
-            e.type == "HTTP_RESPONSE"
-            and e.data["url"] == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862"
-            for e in events
-        ), "Failed to emit HTTP_RESPONSE"
-        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain"
+        assert len(events) == 5
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0
+            ]
+        ), "Failed to emit target DNS_NAME"
+        assert 1 == len(
+            [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0]
+        ), "Failed to find ORG_STUB"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "postman" in e.tags
+                and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public"
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find blacklanternsecurity postman workspace"