intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/tests/bots/parsers/disp/test_parser.py

@@ -0,0 +1,282 @@
+"""Tests for DISPParserBot
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+import unittest
+from copy import deepcopy
+from datetime import datetime
+
+import intelmq.lib.message as message
+from dateutil.tz import UTC
+from intelmq.lib.test import skip_internet
+
+from intelmq_extensions.bots.parsers.disp.parser import DISPParserBot
+
+from ....base import BotTestCase
+
+
+class TestDISPParserBot(BotTestCase, unittest.TestCase):
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = DISPParserBot
+        cls.sysconfig = {
+            "default_fields": {
+                "classification.taxonomy": "information-content-security",
+                "classification.type": "data-leak",
+                "classification.identifier": "leaked-credentials",
+                "comment": "A comment",
+            }
+        }
+
+    def _create_report_dict(self, credentials, id_: str = "id-1", **kwargs):
+        incident_data = {
+            "id": id_,
+            "creationDate": 1693391274125,
+            "publishedDate": 1693391312517,
+            "title": "Credentials compromised by malware-1 botnet",
+            "category": "Information discovery",
+            "type": "Credentials",
+            "detectionDate": 1693391031733,
+            "validationDate": 1693391274125,
+            "updateDate": 1693391446948,
+            "riskLevel": "High",
+            "tags": [
+                "family: malware-1",
+                "type: InfoStealer",
+            ],
+            "relatedAssets": ["example.at"],
+        }
+        default_credentials = {
+            "malware": "malware-1",
+            "date": "",
+            "application": "Some App XXX",
+            "url": "https://some.example.at/",
+            "username": "my-user-1",
+            "password": "pas*****",
+        }
+        credentials_data = []
+        for credential in credentials:
+            c_data = deepcopy(default_credentials)
+            c_data.update(credential)
+            credentials_data.append(c_data)
+        evidence_data = {"credentials": credentials_data}
+
+        raw = {"incident": incident_data, "evidences": evidence_data}
+        data = {
+            "feed.name": "Test Bot",
+            "feed.accuracy": 100.0,
+            "feed.code": "feed-code",
+            "time.observation": datetime(2023, 1, 1, tzinfo=UTC).isoformat(),
+        }
+        data.update(kwargs)
+        report = message.Report(data, harmonization=self.harmonization)
+        report.add("raw", json.dumps(raw))
+        return report.to_dict(with_type=True)
+
+    def _create_event_dict(
+        self,
+        username: str = "my-user-1",
+        password: str = "pas*****",
+        fqdn: str = "some.example.at",
+        url: str = "https://some.example.at/",
+        urlpath: str = "/",
+        full_url: str = "hxxps://some.example.at/",
+        compromise_time: str = "",
+        incident_data: dict = None,
+        evidences_data: dict = None,
+        compromise_time_full: str = None,
+        ip: str = None,
+        **kwargs,
+    ):
+        data = {
+            "classification.identifier": "leaked-credentials",
+            "classification.taxonomy": "information-content-security",
+            "classification.type": "data-leak",
+            "comment": "A comment",
+            "event_description.text": "Credentials compromised by malware-1 botnet",
+            "source.fqdn": fqdn,
+            "source.urlpath": urlpath,
+            "source.url": url,
+            "source.ip": ip,
+            "source.account": username,
+            "extra.account": username,  # Intentionally copied
+            "extra.password": password,
+            "extra.compromise_time": compromise_time,
+            "extra.compromise_time_full": compromise_time_full,
+            "extra.application": "Some App XXX",
+            "extra.full_url": full_url,
+            "extra.feed_event_id": "id-1",
+            "malware.name": "malware-1",
+            "feed.code": "feed-code",
+            "feed.name": "Test Bot",
+            "feed.accuracy": 100.0,
+            "extra.monitored_asset": "example.at",  # Domain we monitor in DISP
+            "time.observation": datetime(2023, 1, 1, tzinfo=UTC).isoformat(),
+        }
+        data.update(kwargs)
+        event = message.Event(data, harmonization=self.harmonization)
+        event.add(
+            "time.source",
+            datetime.fromtimestamp(1693391274125 // 1000, tz=UTC).isoformat(),
+        )
+        event.add(
+            "raw", json.dumps({"incident": incident_data, "evidences": evidences_data})
+        )
+        return event.to_dict(with_type=True)
+
+    def test_parsing_report_default(self):
+        self.input_message = self._create_report_dict(
+            [
+                {
+                    "malware": "malware-1",
+                    "date": "",
+                    "application": "Some App XXX",
+                    "url": "https://some.example.at/",
+                    "username": "my-user-1",
+                    "password": "pas*****",
+                },
+                {
+                    "malware": "malware-1",
+                    "date": "09.27.2021 16:17:18",
+                    "application": "Some App XXX",
+                    "url": "https://example.at/some/sub/page",
+                    "username": "my-user-2",
+                    "password": "lea*******",
+                },
+            ],
+        )
+
+        self.run_bot()
+
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(username="my-user-1", password="pas*****"),
+            compare_raw=False,
+        )
+        self.assertMessageEqual(
+            1,
+            self._create_event_dict(
+                username="my-user-2",
+                password="lea*******",
+                fqdn="example.at",
+                url="https://example.at/[REDACTED]",
+                urlpath="/[REDACTED]",
+                full_url="hxxps://example.at/some/sub/page",
+                compromise_time="2021-09",
+                compromise_time_full="09.27.2021 16:17:18",
+            ),
+            compare_raw=False,
+        )
+
+    def test_compromise_time_settings(self):
+        report = self._create_report_dict([{"date": "09.27.2021 16:17:18"}])
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"compromise_time_format": "original"})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                compromise_time="09.27.2021 16:17:18",
+                compromise_time_full="09.27.2021 16:17:18",
+            ),
+            compare_raw=False,
+        )
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"compromise_time_format": ""})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                compromise_time="",
+                compromise_time_full="09.27.2021 16:17:18",
+            ),
+            compare_raw=False,
+        )
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"compromise_time_format": "%Y-%m-%d"})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                compromise_time="2021-09-27",
+                compromise_time_full="09.27.2021 16:17:18",
+            ),
+            compare_raw=False,
+        )
+
+        self.input_message = self._create_report_dict([{"date": "invalid"}])
+        self.run_bot(
+            parameters={"compromise_time_format": "%Y-%m-%d"}, allowed_warning_count=1
+        )
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                compromise_time="",
+                compromise_time_full="invalid",
+            ),
+            compare_raw=False,
+        )
+
+    def test_redacting_url_settings(self):
+        report = self._create_report_dict(
+            [{"url": "https://some.example.at/my/sub?url=1"}]
+        )
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"redact_url_path": True})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                fqdn="some.example.at",
+                url="https://some.example.at/[REDACTED]",
+                urlpath="/[REDACTED]",
+                full_url="hxxps://some.example.at/my/sub?url=1",
+            ),
+            compare_raw=False,
+        )
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"redact_url_path": False})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                fqdn="some.example.at",
+                url="https://some.example.at/my/sub",
+                urlpath="/my/sub",
+                full_url="hxxps://some.example.at/my/sub?url=1",
+            ),
+            compare_raw=False,
+        )
+
+    @skip_internet()
+    def test_resolve_ip_settings(self):
+        report = self._create_report_dict([{"url": "https://cert.at/"}])
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"resolve_ip": True})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                fqdn="cert.at",
+                ip="131.130.249.234",
+                url="https://cert.at/",
+                full_url="hxxps://cert.at/",
+            ),
+            compare_raw=False,
+        )
+
+        self.input_message = deepcopy(report)
+        self.run_bot(parameters={"resolve_ip": False})
+        self.assertMessageEqual(
+            0,
+            self._create_event_dict(
+                fqdn="cert.at",
+                ip="",
+                url="https://cert.at/",
+                full_url="hxxps://cert.at/",
+            ),
+            compare_raw=False,
+        )

intelmq_extensions/tests/bots/parsers/malwaredomains/test_parser.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+import base64
+import os
+import unittest
+
+import intelmq.lib.test as test
+
+from intelmq_extensions.bots.parsers.malwaredomains.parser import (
+    MalwareDomainsParserBot,
+)
+
+with open(os.path.join(os.path.dirname(__file__), "domains.txt"), "rb") as fh:
+    RAW = base64.b64encode(fh.read()).decode()
+
+OUTPUT1 = {
+    "__type": "Event",
+    "classification.type": "phishing",
+    "event_description.text": "phishing",
+    "classification.identifier": "phishing",
+    "raw": "CQlleGFtcGxlLmNvbQlwaGlzaGluZwlvcGVucGhpc2guY29tCTIwMTYwNTI3CTIwMTYwMTA4",
+    "source.fqdn": "example.com",
+    "time.source": "2016-05-27T00:00:00+00:00",
+}
+OUTPUT2 = {
+    "__type": "Event",
+    "classification.type": "phishing",
+    "event_description.text": "phishing",
+    "classification.identifier": "phishing",
+    "raw": "CQlleGFtcGxlLmludmFsaWQJcGhpc2hpbmcJb3BlbnBoaXNoLmNvbQkyMDE2MDUyNwkyMDE2MDEwOA==",
+    "source.fqdn": "example.invalid",
+    "time.source": "2016-05-27T00:00:00+00:00",
+}
+OUTPUT3 = {
+    "__type": "Event",
+    "classification.type": "c2-server",
+    "event_description.text": "C&C",
+    "classification.identifier": "C&C",
+    "raw": "CQlleGFtcGxlLm5ldAlDJkMJc291cmNlLmV4YW1wbGUuY29tCTIwMTcxMjAxCTIwMTYwNzE5CTIwMTYwMzEw",
+    "source.fqdn": "example.net",
+    "time.source": "2017-12-01T00:00:00+00:00",
+}
+
+
+class TestMalwareDomainsParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for MalwareDomainsParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = MalwareDomainsParserBot
+        cls.default_input_message = {"__type": "Report", "raw": RAW}
+
+    def test_event(self):
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT1)
+        self.assertMessageEqual(1, OUTPUT2)
+        self.assertMessageEqual(2, OUTPUT3)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

intelmq_extensions/tests/cli/test_create_reports.py

@@ -0,0 +1,97 @@
+from intelmq.lib.test import skip_database
+
+from intelmq_extensions.cli.create_reports import IntelMQCLIContoller
+
+from ..base import CLITestCase
+
+
+@skip_database()
+class TestCreatingReports(CLITestCase):
+    CLI_CONTROLLER = IntelMQCLIContoller
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.event_ids = [
+            self.db_add_event({"feed.code": "feed1", "constituency": "energy"}),
+            self.db_add_event({"feed.code": "feed2", "constituency": "energy"}),
+            self.db_add_event({"feed.code": "feed3", "constituency": "national"}),
+            self.db_add_event({"feed.code": "feed4", "constituency": "national"}),
+            self.db_add_event({"feed.code": "feed5", "constituency": None}),
+        ]
+
+    def test_list_open_feeds(self):
+        self.config.update({"constituency": {"key": "energy"}})
+
+        self.run_cli(["-l"])
+
+        self.assertEqual(2, len(self.stdout))
+        self.assertIn("feed1\n", self.stdout)
+        self.assertIn("feed2\n", self.stdout)
+
+        self.config.update({"constituency": {"key": "national", "default": True}})
+
+        self.run_cli(["-l"])
+        self.assertEqual(3, len(self.stdout))
+        self.assertIn("feed3\n", self.stdout)
+        self.assertIn("feed4\n", self.stdout)
+        self.assertIn("feed5\n", self.stdout)
+
+    def test_run_create_reports(self):
+        self.config.update({"constituency": {"key": "energy"}})
+
+        self.run_cli([])
+
+        self.assertInLogs("Handling feedcode 'feed1'")
+        self.assertInLogs("Handling feedcode 'feed2'")
+        self.assertRTTicketCount(2)
+
+        event_feed1 = self.db_get_event(self.event_ids[0])
+        self.assertEqual(event_feed1["rtir_report_id"], 1)
+
+        event_feed2 = self.db_get_event(self.event_ids[1])
+        self.assertEqual(event_feed2["rtir_report_id"], 2)
+
+        self.assertIn("Owner", self.rt_mock.create_ticket.call_args.kwargs)
+
+    def test_create_internal_only_notifications(self):
+        self.config.update({"constituency": {"key": "national", "default": True}})
+        id_1 = self.db_add_event(
+            {"feed.code": "internal-only"},
+            notify=True,
+            abuse_contact="",
+            cc="PL",
+            extra={"monitored_asset": "something"},
+        )
+        id_2 = self.db_add_event(
+            {"feed.code": "internal-only2"}, notify=True, abuse_contact="", cc="AT"
+        )
+        id_3 = self.db_add_event(
+            {"feed.code": "internal-only2"},
+            notify=True,
+            abuse_contact="",
+            cc="US",
+            extra={"monitored_asset": "something"},
+        )
+
+        self.run_cli(
+            [
+                "--internal-notify",
+                "--monitored-assets",
+                "--feed",
+                "internal-only",
+                "internal-only2",
+            ]
+        )
+
+        self.assertRTTicketCount(2)
+        self.assertEqual(self.db_get_event(id_1)["rtir_report_id"], 1)
+        for event_id in [id_2, id_3]:
+            event = self.db_get_event(event_id)
+            self.assertEqual(event["rtir_report_id"], 2)
+
+        self.assertNotIn("Owner", self.rt_mock.create_ticket.call_args.kwargs)
+
+        # Other events are untouched
+        for event_id in self.event_ids:
+            event = self.db_get_event(event_id)
+            self.assertIsNone(event["rtir_report_id"])

intelmq_extensions/tests/cli/test_intelmqcli.py

@@ -0,0 +1,158 @@
+from intelmq.lib.test import skip_database
+
+from intelmq_extensions.cli.intelmqcli import IntelMQCLIContoller
+
+from ..base import CLITestCase
+
+
+@skip_database()
+class TestIntelMQCLI(CLITestCase):
+    CLI_CONTROLLER = IntelMQCLIContoller
+
+    def test_list_feeds(self):
+        self.db_add_event({"feed.code": "my-feed"})
+        self.db_add_event({"feed.code": "feed-2"})
+
+        self.run_cli(["-l"])
+
+        self.assertIn("my-feed\n", self.stdout)
+        self.assertIn("feed-2\n", self.stdout)
+
+    def test_add_event_description_to_text(self):
+        self.add_boilerplate(
+            "test", "my generic description and \n{event_descriptions}"
+        )
+        self.add_boilerplate("event-description-divider", "\n ### \n")
+        self.db_add_event(
+            {
+                "feed.code": "feed1",
+                "rtir_report_id": 10,
+                "event_description.text": "descrpt1",
+            }
+        )
+        self.db_add_event(
+            {
+                "feed.code": "feed1",
+                "rtir_report_id": 10,
+                "event_description.text": "description with \\n encoded new line",
+            }
+        )
+
+        self.run_cli(["--batch", "--quiet"])
+
+        self.assertRTTicketCount(2)
+
+        text = self.get_rt_text(2)
+        self.assertIn(
+            "description with \n encoded new line",
+            text,
+        )
+        self.assertIn("\n ### \n", text)
+        self.assertIn("descrpt1", text)
+
+    def test_add_event_description_to_text_default_divider(self):
+        self.add_boilerplate(
+            "test", "my generic description and \n{event_descriptions}"
+        )
+        self.db_add_event(
+            {
+                "feed.code": "feed1",
+                "rtir_report_id": 10,
+                "event_description.text": "descrpt1",
+            }
+        )
+        self.db_add_event(
+            {
+                "feed.code": "feed1",
+                "rtir_report_id": 10,
+                "event_description.text": "description with \\n encoded new line",
+            }
+        )
+
+        self.run_cli(["--batch", "--quiet"])
+
+        self.assertRTTicketCount(2)
+
+        text = self.get_rt_text(2)
+        self.assertIn("\n --- \n", text)
+
+
+@skip_database()
+class TestConstituencySupport(CLITestCase):
+    CLI_CONTROLLER = IntelMQCLIContoller
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.config.update({"constituency": {"key": "energy"}})
+
+        self.add_boilerplate("test", "some_body")
+
+        self.event_ids = [
+            self.db_add_event(
+                {"feed.code": "feed1", "constituency": "energy", "rtir_report_id": 10}
+            ),
+            self.db_add_event(
+                {"feed.code": "feed2", "constituency": "energy", "rtir_report_id": 20}
+            ),
+            self.db_add_event(
+                {"feed.code": "feed3", "constituency": "national", "rtir_report_id": 30}
+            ),
+            self.db_add_event(
+                {"feed.code": "feed4", "constituency": "national", "rtir_report_id": 40}
+            ),
+            self.db_add_event(
+                {"feed.code": "feed5", "constituency": None, "rtir_report_id": 50}
+            ),
+        ]
+
+    def test_all_list_feeds(self):
+        """intelmqcli ignore constituency when listing - FIXME"""
+        self.run_cli(["-l"])
+
+        self.assertEqual(5, len(self.stdout))
+
+    def test_run_create_reports(self):
+        self.run_cli(["--batch", "--quiet"])
+
+        self.assertInLogs("All taxonomies: test")
+
+        # 1 new incident + 1 new investigation
+        # events have the same abuse contact and taxonomy, so they are processed together
+        self.assertRTTicketCount(2)
+
+        for event_id in self.event_ids[:2]:
+            event = self.db_get_event(event_id)
+            self.assertEqual(event["rtir_incident_id"], 1)
+            self.assertEqual(event["rtir_investigation_id"], 2)
+            self.assertIsNotNone(event["sent_at"])
+
+        # Other events untouched
+        for event_id in self.event_ids[2:]:
+            event = self.db_get_event(event_id)
+            self.assertIsNone(event["rtir_incident_id"])
+            self.assertIsNone(event["rtir_investigation_id"])
+            self.assertIsNone(event["sent_at"])
+
+    def test_creating_attachment_removes_new_lines_from_description(self):
+        self.db_add_event(
+            {
+                "feed.code": "feed2",
+                "constituency": "energy",
+                "rtir_report_id": 20,
+                "event_description.text": (
+                    "Some very long description \n with \n\n multiple \n new lines."
+                ),
+            }
+        )
+
+        self.run_cli(["--batch", "--quiet"])
+
+        self.assertInLogs("All taxonomies: test")
+
+        self.assertRTTicketCount(2)
+
+        _, content_io, __ = self.get_rt_attachment(investigation_id=2)
+        content: str = content_io.getvalue()
+        self.assertIn(
+            "Some very long description   with    multiple   new lines.", content
+        )

intelmq_extensions/tests/lib/base.py

@@ -0,0 +1,81 @@
+"""Base for testing APIs with OAuth token
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+from base64 import urlsafe_b64encode
+from datetime import datetime, timedelta
+from urllib.parse import parse_qs
+
+import requests
+from requests_mock import MockerCore
+
+
+def _get_form_matcher(expected: dict):
+    def _matcher(request: requests.Request):
+        sent_data = parse_qs(request.body)
+        for key, value in sent_data.items():
+            if len(value) == 1:
+                sent_data[key] = value[0]
+        return expected == sent_data
+
+    return _matcher
+
+
+class OAuthAccess_APIMockMixIn:
+    def setup_oauth_mock(
+        self,
+        oauth_clientid: str = "client1",
+        oauth_clientsecret: str = "secret1",
+        oauth_url: str = "https://oauthip.example.at/v1",
+        oauth_scope: str = "",
+        oauth_grant_type: str = "client_credentials",
+        session: requests.Session = None,
+    ):
+        self._oauth_clientid = oauth_clientid
+        self._oauth_clientsecret = oauth_clientsecret
+        self._oauth_url = oauth_url
+        self._oauth_scope = oauth_scope
+        self._oauth_grant_type = oauth_grant_type
+        self.__session = session
+        self._mock_access_token()
+
+    def _mock_access_token(self):
+        self.auth_requests = MockerCore(session=self.__session, real_http=True)
+        self.auth_requests.start()
+        self.addCleanup(self.auth_requests.stop)
+
+        expected = {
+            "scope": self._oauth_scope,
+            "client_id": self._oauth_clientid,
+            "client_secret": self._oauth_clientsecret,
+            "grant_type": self._oauth_grant_type,
+        }
+        expected = dict(filter(lambda kv: kv[1], expected.items()))
+        self.mocked_access_token = self.gen_fake_token()
+
+        def _gen_response(*_, **__):
+            return {
+                "token_type": "Bearer",
+                "expires_in": 3599,
+                "ext_expires_in": 3599,
+                "access_token": self.mocked_access_token,
+            }
+
+        # TODO: matcher
+
+        self.auth_requests.post(
+            url=self._oauth_url,
+            json=_gen_response,
+            additional_matcher=_get_form_matcher(expected),
+        )
+
+    @staticmethod
+    def gen_fake_token(expiry: datetime = None):
+        expiry = expiry or (datetime.utcnow() + timedelta(hours=1))
+        data = urlsafe_b64encode(
+            json.dumps({"exp": int(expiry.timestamp())}).encode()
+        ).decode()
+        return f"xxxxx.{data}.xxx"