intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/tests/bots/collectors/blackkite/test_collector.py

@@ -0,0 +1,287 @@
+"""Tests for BlackKite Collector
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+import unittest
+from unittest import mock
+
+import intelmq.lib.message as message
+import requests
+
+from intelmq_extensions.bots.collectors.blackkite.collector import BlackKiteCollectorBot
+
+from ....base import BotTestCase
+from .base import BlackKite_APIMockMixIn
+
+
+class TestBlackKiteCollectorBot(BotTestCase, BlackKite_APIMockMixIn, unittest.TestCase):
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = BlackKiteCollectorBot
+        cls.sysconfig = {
+            "url": "https://blackkite.example.at/v1",
+            "client_id": "client1",
+            "client_secret": "secret1",
+            "code": "feed-code",
+            "categories": {"PATCH": {}},
+        }
+
+    def catch_http_session(self):
+        session = requests.Session()
+        session_mock = mock.patch(
+            "intelmq_extensions.bots.collectors.blackkite.collector.create_request_session",
+            return_value=session,
+        )
+        session_mock.start()
+        self.addCleanup(session_mock.stop)
+        return session
+
+    def setUp(self) -> None:
+        super().setUp()
+        session = self.catch_http_session()
+        self.setup_api_mock(session=session)
+
+    def test_static_check_validates_filters(self):
+        correct_params = {
+            "severities": ["Info", "Low", "Medium", "High", "Critical"],
+            "statuses": [
+                "Active",
+                "FalsePositive",
+                "Suppressed",
+                "Acknowledged",
+                "Deleted",
+            ],
+            "outputs": ["Info", "Passed", "Warning", "Failed"],
+            "categories": {
+                "PATCH": {
+                    "statuses": ["Active"],
+                    "severities": ["Critical"],
+                    "include": ["PATCH-001"],
+                },
+                "APPSEC": {},
+                "DNS": {"outputs": ["Failed"]},
+                "LEAK": None,
+            },
+        }
+        self.assertIsNone(self.bot_reference.check(correct_params))
+
+    def test_static_check_invalid_params(self):
+        invalid_params = {
+            "severities": ["Some", "Other"],
+            "statuses": ["NonActive"],
+            "outputs": ["Other"],
+            "categories": {
+                "PATCH": {
+                    "include": ["PATCH-001"],
+                    "exclude": ["PATCH-002"],
+                },
+                "DNS": {
+                    "statuses": ["NonActive"],
+                },
+                "FRADOM": {
+                    "outputs": ["Failed"],
+                },
+                "non-existing": {},
+                "SMTP": {"not-existing-key": []},
+            },
+        }
+        results = self.bot_reference.check(invalid_params)
+        self.assertEqual(8, len([r for r in results if r[0] == "error"]))
+
+    def _create_report_dict(self, company_data: dict, finding_data: dict, **kwargs):
+        report = message.Report(kwargs, harmonization=self.harmonization)
+        data = {"company": company_data, "finding": finding_data}
+        report.add("feed.name", "Test Bot")
+        report.add("feed.accuracy", 100.0)
+        report.add("feed.code", "feed-code")
+        report.add("raw", json.dumps(data))
+        return report.to_dict(with_type=True)
+
+    def _mock_companies(self, number: int = 2):
+        self.mock_request(
+            "companies",
+            json=[{"CompanyId": id_ + 1} for id_ in range(number)],
+            headers={"X-Total-Items": str(number)},
+        )
+
+    def test_get_data_for_all_companies(self):
+        self._mock_companies()
+        self.mock_request(
+            "companies/1/findings/patchmanagement?" "status=Active&severity=Critical",
+            json=[{"FindingId": 1}, {"FindingId": 2}],
+        )
+        self.mock_request(
+            "companies/2/findings/patchmanagement?&status=Active&severity=Critical",
+            json=[{"FindingId": 3}],
+        )
+
+        self.run_bot()
+
+        self.assertOutputQueueLen(3)
+        self.assertMessageEqual(
+            0, self._create_report_dict({"CompanyId": 1}, {"FindingId": 1})
+        )
+        self.assertMessageEqual(
+            1, self._create_report_dict({"CompanyId": 1}, {"FindingId": 2})
+        )
+        self.assertMessageEqual(
+            2, self._create_report_dict({"CompanyId": 2}, {"FindingId": 3})
+        )
+
+    def test_single_exception_doesnt_stop_processing(self):
+        self._mock_companies()
+        self.mock_request(
+            "companies/1/findings/patchmanagement?status=Active&severity=Critical",
+            status_code=500,
+        )
+        self.mock_request(
+            "companies/1/findings/applicationsecurity?status=Active&severity=Critical",
+            json=[{"FindingId": 1}],
+        )
+        self.mock_request(
+            "companies/2/findings/patchmanagement?status=Active&severity=Critical",
+            json=[{"FindingId": 2}],
+        )
+        self.mock_request(
+            "companies/2/findings/applicationsecurity?status=Active&severity=Critical",
+            json=[{"FindingId": 3}],
+        )
+
+        self.run_bot(
+            parameters={**self.sysconfig, "categories": {"PATCH": {}, "APPSEC": {}}},
+            allowed_error_count=2,  # log from collector + from client
+        )
+
+        self.assertOutputQueueLen(3)
+        self.assertMessageEqual(
+            0, self._create_report_dict({"CompanyId": 1}, {"FindingId": 1})
+        )
+        self.assertMessageEqual(
+            1, self._create_report_dict({"CompanyId": 2}, {"FindingId": 2})
+        )
+        self.assertMessageEqual(
+            2, self._create_report_dict({"CompanyId": 2}, {"FindingId": 3})
+        )
+
+    def test_overriding_filters(self):
+        self._mock_companies(1)
+        self.mock_request(
+            "companies/1/findings/patchmanagement?status=Acknowledged&severity=Critical",
+            json=[{"FindingId": 1}],
+        )
+        self.mock_request(
+            "companies/1/findings/applicationsecurity?status=Active&severity=Low,Critical",
+            headers={"X-Total-Items": "1"},
+            json=[{"FindingId": 2}],
+        )
+
+        self.run_bot(
+            parameters={
+                **self.sysconfig,
+                "severities": ["Critical"],
+                "statuses": ["Active"],
+                "categories": {
+                    "PATCH": {"statuses": ["Acknowledged"]},
+                    "APPSEC": {"severities": ["Low", "Critical"]},
+                },
+            }
+        )
+
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0, self._create_report_dict({"CompanyId": 1}, {"FindingId": 1})
+        )
+        self.assertMessageEqual(
+            1, self._create_report_dict({"CompanyId": 1}, {"FindingId": 2})
+        )
+
+    def test_category_with_no_settings(self):
+        self._mock_companies(1)
+        self.mock_request(
+            "companies/1/findings/patchmanagement?status=Active&severity=Critical",
+            json=[{"FindingId": 1}],
+        )
+
+        self.run_bot(
+            parameters={
+                **self.sysconfig,
+                "categories": {
+                    "PATCH": None,
+                },
+            }
+        )
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0, self._create_report_dict({"CompanyId": 1}, {"FindingId": 1})
+        )
+
+    def test_include_and_exclude_findings(self):
+        self._mock_companies(1)
+        self.mock_request(
+            "companies/1/findings/patchmanagement",
+            json=[
+                {"ControlId": "PATCH-001"},
+                {"ControlId": "PATCH-002"},
+                {"ControlId": "PATCH-003"},
+            ],
+        )
+        self.mock_request(
+            "companies/1/findings/applicationsecurity",
+            headers={"X-Total-Items": "1"},
+            json=[
+                {"ControlId": "APPSEC-001"},
+                {"ControlId": "APPSEC-002"},
+                {"ControlId": "APPSEC-003"},
+            ],
+        )
+
+        self.run_bot(
+            parameters={
+                **self.sysconfig,
+                "categories": {
+                    "PATCH": {"include": ["PATCH-001", "PATCH-003"]},
+                    "APPSEC": {"exclude": ["APPSEC-003"]},
+                },
+            }
+        )
+
+        self.assertOutputQueueLen(4)
+        self.assertMessageEqual(
+            0, self._create_report_dict({"CompanyId": 1}, {"ControlId": "PATCH-001"})
+        )
+        self.assertMessageEqual(
+            1, self._create_report_dict({"CompanyId": 1}, {"ControlId": "PATCH-003"})
+        )
+        self.assertMessageEqual(
+            2, self._create_report_dict({"CompanyId": 1}, {"ControlId": "APPSEC-001"})
+        )
+        self.assertMessageEqual(
+            3, self._create_report_dict({"CompanyId": 1}, {"ControlId": "APPSEC-002"})
+        )
+
+    def test_acknowledge_findings(self):
+        self._mock_companies(1)
+        self.mock_request(
+            "companies/1/findings/credentialmanagement",
+            json=[
+                {"ControlId": "LEAK-001", "FindingId": 1},
+                {"ControlId": "LEAK-002", "FindingId": 2},
+            ],
+            headers={"X-Total-Items": "2"},
+        )
+        self.mock_request(
+            "companies/1/findings/1", json={"Status": "Acknowledged"}, method="patch"
+        )
+        self.mock_request(
+            "companies/1/findings/2", json={"Status": "Acknowledged"}, method="patch"
+        )
+
+        self.run_bot(
+            parameters={**self.sysconfig, "categories": {"LEAK": {"acknowledge": True}}}
+        )
+
+        self.assertEqual(2 + 2, self.requests.call_count)

intelmq_extensions/tests/bots/collectors/disp/base.py

@@ -0,0 +1,147 @@
+"""Base for DISP client and collector tests
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+from datetime import datetime, timedelta
+
+import requests
+from requests_mock import MockerCore
+
+from ....lib.base import OAuthAccess_APIMockMixIn
+
+_DEFAULT_TAGS = ["_BotnetCreds", "family: Redline", "type: InfoStealer", "_Parsed"]
+_DEFAULT_ASSETS = ["nic.at"]
+
+
+class DISP_APIMockMixIn(OAuthAccess_APIMockMixIn):
+    def setup_api_mock(
+        self,
+        url: str = "https://disp.example.at/v1",
+        auth_token: str = "XXX",
+        oauth_clientid: str = "client1",
+        oauth_clientsecret: str = "secret1",
+        oauth_url: str = "https://oauthip.example.at/v1",
+        session: requests.Session = None,
+    ):
+        self._api_url = url
+        self._auth_token = auth_token
+        self.session = session
+        self.requests = MockerCore(session=self.session)
+        self.requests.start()
+        self.addCleanup(self.requests.stop)
+
+        self.setup_oauth_mock(
+            oauth_clientid=oauth_clientid,
+            oauth_clientsecret=oauth_clientsecret,
+            oauth_url=oauth_url,
+            oauth_scope="https://gateway.disp.deloitte.com/.default",
+            oauth_grant_type="client_credentials",
+            session=self.session,
+        )
+
+        self._incident_counter = 0
+        self._incidents = []
+
+    def create_requests_mock(self):
+        # create nested requests mock to separate tested requests
+        mocker = MockerCore(session=self.session, real_http=True)
+        mocker.start()
+        self.addCleanup(mocker.stop)
+        return mocker
+
+    def mock_request(
+        self, path: str, mocker: MockerCore = None, method: str = "get", **kwargs
+    ):
+        mocker = mocker or self.requests
+        mocking_method = getattr(mocker, method)
+        mocking_method(
+            f"{self._api_url}/{path}",
+            request_headers={
+                "Authorization": f"Bearer {self._auth_token}",
+                "OAuth": self.mocked_access_token,
+            },
+            **kwargs,
+        )
+
+    def add_incident(
+        self,
+        *,
+        viewed: bool = False,
+        validation_date: datetime = None,
+        title: str = "Credentials compromised by a botnet",
+        category: str = "Information discovery",
+        type_: str = "Credentials",
+        tags: list = None,
+        related_assets: list = None,
+        evidence_id: str = None,
+        other: dict = None,
+    ):
+        """Add mocked incident to the API responses, but only selected used fields fields"""
+        incident_id = f"incident-{self._incident_counter}"
+        self._incident_counter += 1
+        if tags is None:
+            tags = _DEFAULT_TAGS
+        if related_assets is None:
+            related_assets = _DEFAULT_ASSETS
+        if validation_date is None:
+            validation_date = datetime.utcnow() - timedelta(days=1)
+
+        incident = {
+            "id": incident_id,
+            "metadata": {
+                "viewed": viewed,
+            },
+            "validationDate": int(validation_date.timestamp() * 1000),
+            "title": title,
+            "category": category,
+            "type": type_,
+            "tags": tags,
+            "relatedAssets": related_assets,
+            "url": f"https://disp.app/document/incident/{incident_id}",
+        }
+
+        if evidence_id:
+            incident.update(
+                {
+                    "evidences": [
+                        {"idStoredFile": evidence_id, "name": f"{incident_id}.json.txt"}
+                    ]
+                }
+            )
+        else:
+            incident.update({"evidences": []})
+
+        if other:
+            incident.update(other)
+
+        self._incidents.append(incident)
+        return incident_id, incident
+
+    def mock_evidence(
+        self,
+        incident_id: str,
+        file_id: str,
+        data: dict = None,
+        mocker: MockerCore = None,
+    ):
+        mocker = mocker or self.requests
+        self.mock_request(
+            f"incident/{incident_id}/file/{file_id}", json=data, mocker=mocker
+        )
+
+    def _mock_incidents(self, mocker: MockerCore = None, after: datetime = None):
+        after = after or datetime.utcnow() - timedelta(days=7)
+        mocker = mocker or self.requests
+        self.mock_request(
+            (
+                f"incident/?query=validationDate%20%3E%20{int(after.timestamp() * 1000)}"
+                "%20AND%20UNREAD&page=0&size=10"
+            ),
+            mocker=mocker,
+            json={
+                "content": self._incidents,
+                "last": True,
+            },
+        )

intelmq_extensions/tests/bots/collectors/disp/test_client.py

@@ -0,0 +1,134 @@
+"""Testing DISP Collector
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+from datetime import datetime, timezone
+from unittest import TestCase
+
+import requests
+
+from intelmq_extensions.bots.collectors.disp._client import DISPClient
+
+from .base import DISP_APIMockMixIn
+
+
+class DISPClientTestCase(DISP_APIMockMixIn, TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.config = {
+            "api_url": "https://disp.example.at/v1",
+            "auth_token": "XXX",
+            "oauth_clientid": "client1",
+            "oauth_clientsecret": "secret1",
+            "oauth_url": "https://oauthip.example.at/v1",
+        }
+        self.setup_api_mock(
+            url=self.config["api_url"],
+            auth_token=self.config["auth_token"],
+            oauth_clientid=self.config["oauth_clientid"],
+            oauth_clientsecret=self.config["oauth_clientsecret"],
+            oauth_url=self.config["oauth_url"],
+            session=requests.Session(),
+        )
+
+        self.client = DISPClient(session=self.session, **self.config)
+
+    def test_get_simple_response(self):
+        self.mock_request(
+            "incident/",
+            json={"id": "some-id"},
+        )
+
+        response = self.client.get("incident/")
+        self.assertEqual({"id": "some-id"}, response)
+
+    def test_get_paginated_response_1_page(self):
+        self.mock_request(
+            "incident/?page=0&size=10",
+            json={
+                "content": [{"id": "id-1"}, {"id": "id-2"}],
+                "last": True,
+            },
+        )
+
+        responses = list(self.client.get_paginated("incident/"))
+        self.assertEqual([{"id": "id-1"}, {"id": "id-2"}], responses)
+
+    def test_get_paginated_response_3_pages(self):
+        self.mock_request(
+            "incident/?page=0&size=10",
+            json={
+                "content": [{"id": "id-1"}, {"id": "id-2"}],
+                "last": False,
+            },
+        )
+        self.mock_request(
+            "incident/?page=1&size=10",
+            json={
+                "content": [{"id": "id-3"}],
+                "last": False,
+            },
+        )
+        self.mock_request(
+            "incident/?page=2&size=10",
+            json={
+                "content": [{"id": "id-4"}],
+                "last": True,
+            },
+        )
+
+        responses = list(self.client.get_paginated("incident/"))
+        self.assertEqual(
+            [{"id": "id-1"}, {"id": "id-2"}, {"id": "id-3"}, {"id": "id-4"}], responses
+        )
+
+    def test_get_incidents(self):
+        after = datetime(2023, 1, 1, 19, 10, tzinfo=timezone.utc)
+
+        self.mock_request(
+            "incident/?query=validationDate%20%3E%201672600200000%20AND%20UNREAD&page=0&size=10",
+            json={"content": [{"id": "id-1"}]},
+        )
+        self.mock_request(
+            "incident/?query=validationDate%20%3E%201672600200000&page=0&size=10",
+            json={"content": [{"id": "id-2"}]},
+        )
+        self.mock_request(
+            "incident/?query=NOT%20SOMETHING&page=0&size=10",
+            json={"content": [{"id": "id-3"}]},
+        )
+
+        self.assertEqual([{"id": "id-1"}], list(self.client.incidents(after, True)))
+        self.assertEqual([{"id": "id-2"}], list(self.client.incidents(after, False)))
+        self.assertEqual(
+            [{"id": "id-3"}], list(self.client.incidents(query="NOT SOMETHING"))
+        )
+
+    def test_get_evidence(self):
+        # For credential tracking, the expected file is in JSON format.
+        # For other incidents (not covered now), the format may differ
+        self.mock_request("incident/id-1/file/f-1", text='{"credentials": []}')
+
+        result = self.client.download_evidence_json("id-1", "f-1")
+
+        self.assertEqual({"credentials": []}, result)
+
+    def test_mark_read(self):
+        self.mock_request("incident/read?id=id-1&read=true", method="post")
+
+        self.client.mark_incident_read("id-1")
+
+        self.assertEqual(1, self.requests.call_count)
+
+    def test_raise_on_connection_issue(self):
+        self.mock_request("/", status_code=500)
+
+        with self.assertRaises(RuntimeError):
+            self.client.get("/")
+
+        self.mock_request("/", method="post", status_code=500)
+
+        with self.assertRaises(RuntimeError):
+            self.client.post("/")