bbot 2.6.0.6879rc0__py3-none-any.whl → 2.7.2.7254rc0__py3-none-any.whl

bbot/modules/censys.py

@@ -1,98 +0,0 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
-
-
-class censys(subdomain_enum_apikey):
-    """
-    thanks to https://github.com/owasp-amass/amass/blob/master/resources/scripts/cert/censys.ads
-    """
-
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    flags = ["subdomain-enum", "passive", "safe"]
-    meta = {
-        "description": "Query the Censys API",
-        "created_date": "2022-08-04",
-        "author": "@TheTechromancer",
-        "auth_required": True,
-    }
-    options = {"api_key": "", "max_pages": 5}
-    options_desc = {
-        "api_key": "Censys.io API Key in the format of 'key:secret'",
-        "max_pages": "Maximum number of pages to fetch (100 results per page)",
-    }
-
-    base_url = "https://search.censys.io/api"
-
-    async def setup(self):
-        self.max_pages = self.config.get("max_pages", 5)
-        return await super().setup()
-
-    async def ping(self):
-        url = f"{self.base_url}/v1/account"
-        resp = await self.api_request(url, retry_on_http_429=False)
-        d = resp.json()
-        assert isinstance(d, dict), f"Invalid response from {url}: {resp}"
-        quota = d.get("quota", {})
-        used = int(quota.get("used", 0))
-        allowance = int(quota.get("allowance", 0))
-        assert used < allowance, "No quota remaining"
-
-    def prepare_api_request(self, url, kwargs):
-        api_id, api_secret = self.api_key.split(":", 1)
-        kwargs["auth"] = (api_id, api_secret)
-        return url, kwargs
-
-    async def query(self, query):
-        results = set()
-        cursor = ""
-        for i in range(self.max_pages):
-            url = f"{self.base_url}/v2/certificates/search"
-            json_data = {
-                "q": f"names: {query}",
-                "per_page": 100,
-            }
-            if cursor:
-                json_data.update({"cursor": cursor})
-            resp = await self.api_request(
-                url,
-                method="POST",
-                json=json_data,
-            )
-
-            if resp is None:
-                break
-
-            try:
-                d = resp.json()
-            except Exception as e:
-                self.warning(f"Failed to parse JSON from {url} (response: {resp}): {e}")
-
-            if resp.status_code < 200 or resp.status_code >= 400:
-                if isinstance(d, dict):
-                    error = d.get("error", "")
-                    if error:
-                        self.warning(error)
-                self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i + 1}')
-                self.debug(f"Response: {resp.text}")
-                break
-            else:
-                if d is None:
-                    break
-                elif not isinstance(d, dict):
-                    break
-                status = d.get("status", "").lower()
-                result = d.get("result", {})
-                hits = result.get("hits", [])
-                if status != "ok" or not hits:
-                    break
-
-                for h in hits:
-                    names = h.get("names", [])
-                    for n in names:
-                        results.add(n.strip(".*").lower())
-
-                cursor = result.get("links", {}).get("next", "")
-                if not cursor:
-                    break
-
-        return results

bbot/modules/gitlab.py

@@ -1,141 +0,0 @@
-from bbot.modules.base import BaseModule
-
-
-class gitlab(BaseModule):
-    watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
-    produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"]
-    flags = ["active", "safe", "code-enum"]
-    meta = {
-        "description": "Detect GitLab instances and query them for repositories",
-        "created_date": "2024-03-11",
-        "author": "@TheTechromancer",
-    }
-    options = {"api_key": ""}
-    options_desc = {"api_key": "Gitlab access token"}
-
-    scope_distance_modifier = 2
-
-    async def setup(self):
-        await self.require_api_key()
-        return True
-
-    async def filter_event(self, event):
-        # only accept out-of-scope SOCIAL events
-        if event.type == "HTTP_RESPONSE":
-            if event.scope_distance > self.scan.scope_search_distance:
-                return False, "event is out of scope distance"
-        elif event.type == "TECHNOLOGY":
-            if not event.data["technology"].lower().startswith("gitlab"):
-                return False, "technology is not gitlab"
-            if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == "gitlab":
-                return False, "gitlab instance is not self-hosted"
-        elif event.type == "SOCIAL":
-            if event.data["platform"] != "gitlab":
-                return False, "platform is not gitlab"
-        return True
-
-    async def handle_event(self, event):
-        if event.type == "HTTP_RESPONSE":
-            await self.handle_http_response(event)
-        elif event.type == "TECHNOLOGY":
-            await self.handle_technology(event)
-        elif event.type == "SOCIAL":
-            await self.handle_social(event)
-
-    async def handle_http_response(self, event):
-        # identify gitlab instances from HTTP responses
-        # HTTP_RESPONSE --> TECHNOLOGY
-        # HTTP_RESPONSE --> FINDING
-        headers = event.data.get("header", {})
-        if "x_gitlab_meta" in headers:
-            url = event.parsed_url._replace(path="/").geturl()
-            await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url},
-                "TECHNOLOGY",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
-            )
-            description = f"GitLab server at {event.host}"
-            await self.emit_event(
-                {"host": str(event.host), "description": description},
-                "FINDING",
-                parent=event,
-                context=f"{{module}} detected {{event.type}}: {description}",
-            )
-
-    async def handle_technology(self, event):
-        # retrieve gitlab groups from gitlab instances
-        # TECHNOLOGY --> SOCIAL
-        # TECHNOLOGY --> URL
-        # TECHNOLOGY --> CODE_REPOSITORY
-        base_url = self.get_base_url(event)
-        projects_url = self.helpers.urljoin(base_url, "api/v4/projects?simple=true")
-        await self.handle_projects_url(projects_url, event)
-        groups_url = self.helpers.urljoin(base_url, "api/v4/groups?simple=true")
-        await self.handle_groups_url(groups_url, event)
-
-    async def handle_social(self, event):
-        # retrieve repositories from gitlab user
-        # SOCIAL --> CODE_REPOSITORY
-        # SOCIAL --> SOCIAL
-        username = event.data.get("profile_name", "")
-        if not username:
-            return
-        base_url = self.get_base_url(event)
-        urls = [
-            # group
-            self.helpers.urljoin(base_url, f"api/v4/users/{username}/projects?simple=true"),
-            # user
-            self.helpers.urljoin(base_url, f"api/v4/groups/{username}/projects?simple=true"),
-        ]
-        for url in urls:
-            await self.handle_projects_url(url, event)
-
-    async def handle_projects_url(self, projects_url, event):
-        for project in await self.gitlab_json_request(projects_url):
-            project_url = project.get("web_url", "")
-            if project_url:
-                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
-                await self.emit_event(
-                    code_event, context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}"
-                )
-            namespace = project.get("namespace", {})
-            if namespace:
-                await self.handle_namespace(namespace, event)
-
-    async def handle_groups_url(self, groups_url, event):
-        for group in await self.gitlab_json_request(groups_url):
-            await self.handle_namespace(group, event)
-
-    async def gitlab_json_request(self, url):
-        response = await self.api_request(url)
-        if response is not None:
-            try:
-                json = response.json()
-            except Exception:
-                return []
-            if json and isinstance(json, list):
-                return json
-        return []
-
-    async def handle_namespace(self, namespace, event):
-        namespace_name = namespace.get("path", "")
-        namespace_url = namespace.get("web_url", "")
-        namespace_path = namespace.get("full_path", "")
-        if namespace_name and namespace_url and namespace_path:
-            namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f"/{namespace_path}").geturl()
-            social_event = self.make_event(
-                {"platform": "gitlab", "profile_name": namespace_path, "url": namespace_url},
-                "SOCIAL",
-                parent=event,
-            )
-            await self.emit_event(
-                social_event,
-                context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
-            )
-
-    def get_base_url(self, event):
-        base_url = event.data.get("url", "")
-        if not base_url:
-            base_url = f"https://{event.host}"
-        return self.helpers.urlparse(base_url)._replace(path="/").geturl()

bbot/modules/zoomeye.py

@@ -1,77 +0,0 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
-
-
-class zoomeye(subdomain_enum_apikey):
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    flags = ["affiliates", "subdomain-enum", "passive", "safe"]
-    meta = {
-        "description": "Query ZoomEye's API for subdomains",
-        "created_date": "2022-08-03",
-        "author": "@TheTechromancer",
-        "auth_required": True,
-    }
-    options = {"api_key": "", "max_pages": 20, "include_related": False}
-    options_desc = {
-        "api_key": "ZoomEye API key",
-        "max_pages": "How many pages of results to fetch",
-        "include_related": "Include domains which may be related to the target",
-    }
-
-    base_url = "https://api.zoomeye.hk"
-
-    async def setup(self):
-        self.max_pages = self.config.get("max_pages", 20)
-        self.include_related = self.config.get("include_related", False)
-        return await super().setup()
-
-    def prepare_api_request(self, url, kwargs):
-        kwargs["headers"]["API-KEY"] = self.api_key
-        return url, kwargs
-
-    async def ping(self):
-        url = f"{self.base_url}/resources-info"
-        r = await self.api_request(url, retry_on_http_429=False)
-        assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining"
-
-    async def handle_event(self, event):
-        query = self.make_query(event)
-        results = await self.query(query)
-        if results:
-            for hostname in results:
-                if hostname == event:
-                    continue
-                tags = []
-                if not hostname.endswith(f".{query}"):
-                    tags = ["affiliate"]
-                await self.emit_event(
-                    hostname,
-                    "DNS_NAME",
-                    event,
-                    tags=tags,
-                    context=f'{{module}} searched ZoomEye API for "{query}" and found {{event.type}}: {{event.data}}',
-                )
-
-    async def query(self, query):
-        results = set()
-        query_type = 0 if self.include_related else 1
-        url = f"{self.base_url}/domain/search?q={self.helpers.quote(query)}&type={query_type}&page=" + "{page}"
-        i = 0
-        agen = self.api_page_iter(url)
-        try:
-            async for j in agen:
-                r = list(await self.parse_results(j))
-                if r:
-                    results.update(set(r))
-                if not r or i >= (self.max_pages - 1):
-                    break
-                i += 1
-        finally:
-            await agen.aclose()
-        return results
-
-    async def parse_results(self, r):
-        results = set()
-        for entry in r.get("list", []):
-            results.add(entry["name"])
-        return results

bbot/test/test_step_2/module_tests/test_module_censys.py

@@ -1,83 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestCensys(ModuleTestBase):
-    config_overrides = {"modules": {"censys": {"api_key": "api_id:api_secret"}}}
-
-    async def setup_before_prep(self, module_test):
-        module_test.httpx_mock.add_response(
-            url="https://search.censys.io/api/v1/account",
-            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            json={
-                "email": "info@blacklanternsecurity.com",
-                "login": "nope",
-                "first_login": "1917-08-03 20:03:55",
-                "last_login": "1918-05-19 01:15:22",
-                "quota": {"used": 26, "allowance": 250, "resets_at": "1919-06-03 16:30:32"},
-            },
-        )
-        module_test.httpx_mock.add_response(
-            url="https://search.censys.io/api/v2/certificates/search",
-            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            method="POST",
-            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100},
-            json={
-                "code": 200,
-                "status": "OK",
-                "result": {
-                    "query": "names: blacklanternsecurity.com",
-                    "total": 196,
-                    "duration_ms": 1046,
-                    "hits": [
-                        {
-                            "parsed": {
-                                "validity_period": {
-                                    "not_before": "2021-11-18T00:09:46Z",
-                                    "not_after": "2022-11-18T00:09:46Z",
-                                },
-                                "issuer_dn": "C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com\\, Inc., OU=http://certs.godaddy.com/repository/, CN=Go Daddy Secure Certificate Authority - G2",
-                                "subject_dn": "CN=asdf.blacklanternsecurity.com",
-                            },
-                            "fingerprint_sha256": "590ad51b8db62925f0fd3f300264c6a36692e20ceec2b5a22e7e4b41c1575cdc",
-                            "names": ["asdf.blacklanternsecurity.com", "asdf2.blacklanternsecurity.com"],
-                        },
-                    ],
-                    "links": {"next": "NextToken", "prev": ""},
-                },
-            },
-        )
-        module_test.httpx_mock.add_response(
-            url="https://search.censys.io/api/v2/certificates/search",
-            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            method="POST",
-            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"},
-            json={
-                "code": 200,
-                "status": "OK",
-                "result": {
-                    "query": "names: blacklanternsecurity.com",
-                    "total": 196,
-                    "duration_ms": 1046,
-                    "hits": [
-                        {
-                            "parsed": {
-                                "validity_period": {
-                                    "not_before": "2021-11-18T00:09:46Z",
-                                    "not_after": "2022-11-18T00:09:46Z",
-                                },
-                                "issuer_dn": "C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com\\, Inc., OU=http://certs.godaddy.com/repository/, CN=Go Daddy Secure Certificate Authority - G2",
-                                "subject_dn": "CN=zzzz.blacklanternsecurity.com",
-                            },
-                            "fingerprint_sha256": "590ad51b8db62925f0fd3f300264c6a36692e20ceec2b5a22e7e4b41c1575cdc",
-                            "names": ["zzzz.blacklanternsecurity.com"],
-                        },
-                    ],
-                    "links": {"next": "", "prev": ""},
-                },
-            },
-        )
-
-    def check(self, module_test, events):
-        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect asdf subdomain"
-        assert any(e.data == "asdf2.blacklanternsecurity.com" for e in events), "Failed to detect asdf2 subdomain"
-        assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect zzzz subdomain"

bbot/test/test_step_2/module_tests/test_module_zoomeye.py

@@ -1,35 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestZoomEye(ModuleTestBase):
-    config_overrides = {"modules": {"zoomeye": {"api_key": "asdf", "include_related": True, "max_pages": 3}}}
-
-    async def setup_before_prep(self, module_test):
-        module_test.httpx_mock.add_response(
-            url="https://api.zoomeye.hk/resources-info",
-            match_headers={"API-KEY": "asdf"},
-            json={"quota_info": {"remain_total_quota": 5}},
-        )
-        module_test.httpx_mock.add_response(
-            url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=1",
-            json={"list": [{"name": "asdf.blacklanternsecurity.com"}]},
-        )
-        module_test.httpx_mock.add_response(
-            url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=2",
-            json={"list": [{"name": "zzzz.blacklanternsecurity.com"}]},
-        )
-        module_test.httpx_mock.add_response(
-            url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=3",
-            json={"list": [{"name": "ffff.blacklanternsecurity.com"}, {"name": "affiliate.bls"}]},
-        )
-        module_test.httpx_mock.add_response(
-            url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=4",
-            json={"list": [{"name": "nope.blacklanternsecurity.com"}]},
-        )
-
-    def check(self, module_test, events):
-        assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #1"
-        assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #2"
-        assert any(e.data == "ffff.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #3"
-        assert any(e.data == "affiliate.bls" and "affiliate" in e.tags for e in events), "Failed to detect affiliate"
-        assert not any(e.data == "nope.blacklanternsecurity.com" for e in events), "Failed to obey max_pages"