intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/tests/bots/collectors/disp/test_collector.py

@@ -0,0 +1,137 @@
+"""Tests for DISP Collector
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+import unittest
+from unittest import mock
+
+import intelmq.lib.message as message
+import requests
+from freezegun import freeze_time
+
+from intelmq_extensions.bots.collectors.disp.collector import DISPCollectorBot
+
+from ....base import BotTestCase
+from .base import DISP_APIMockMixIn
+
+
+@freeze_time("2023-09-19 12:18")
+class TestDISPCollectorBot(BotTestCase, DISP_APIMockMixIn, unittest.TestCase):
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = DISPCollectorBot
+        cls.sysconfig = {
+            "api_url": "https://disp.example.at/v1",
+            "auth_token": "XXX",
+            "oauth_clientid": "client1",
+            "oauth_clientsecret": "secret1",
+            "ouath_url": "https://oauthip.example.at/v1",
+            "mark_as_read": False,
+            "wait_for_evidences": True,
+            "check_last": "7 days",
+            "code": "feed-code",
+        }
+
+    def catch_http_session(self):
+        session = requests.Session()
+        session_mock = mock.patch(
+            "intelmq_extensions.bots.collectors.disp.collector.create_request_session",
+            return_value=session,
+        )
+        session_mock.start()
+        self.addCleanup(session_mock.stop)
+        return session
+
+    def setUp(self) -> None:
+        super().setUp()
+        session = self.catch_http_session()
+        self.setup_api_mock(session=session)
+        self._mock_incidents()
+
+    def test_no_incidents(self):
+        self.run_bot()
+
+        self.assertOutputQueueLen(0)
+        self.assertEqual(1, self.requests.call_count)
+
+    def test_ignore_incidents_without_evidences(self):
+        self.add_incident(evidence_id=None)
+
+        self.run_bot()
+
+        self.assertOutputQueueLen(0)
+        self.assertEqual(1, self.requests.call_count)
+
+    def test_generate_event_without_waiting_for_evidence(self):
+        self.add_incident(evidence_id=None)
+
+        self.run_bot(parameters={"wait_for_evidences": False})
+
+        self.assertOutputQueueLen(1)
+
+    def _create_report_dict(self, incident_data: str, evidence_data: str, **kwargs):
+        report = message.Report(kwargs, harmonization=self.harmonization)
+        data = {"incident": incident_data, "evidences": evidence_data}
+        report.add("feed.name", "Test Bot")
+        report.add("feed.accuracy", 100.0)
+        report.add("feed.code", "feed-code")
+        report.add("raw", json.dumps(data))
+        return report.to_dict(with_type=True)
+
+    def test_generate_event(self):
+        incident_id, incident_data = self.add_incident(evidence_id="file-1")
+        self.mock_evidence(incident_id, "file-1", {"my": "evidence"})
+
+        self.run_bot()
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0, self._create_report_dict(incident_data, {"my": "evidence"})
+        )
+
+    def test_mark_as_read(self):
+        incident_id, _ = self.add_incident(evidence_id="file-1")
+        self.mock_evidence(incident_id, "file-1", {"my": "evidence"})
+        self.mock_request(f"incident/read?id={incident_id}&read=true", method="post")
+
+        self.run_bot(parameters={"mark_as_read": True})
+
+        self.assertOutputQueueLen(1)
+        self.assertIn("incident/read", self.requests.last_request.path)
+        self.assertIn(f"id={incident_id}&read=true", self.requests.last_request.query)
+
+    def test_mask_password(self):
+        incident_id, incident_data = self.add_incident(evidence_id="file-1")
+        self.mock_evidence(
+            incident_id,
+            "file-1",
+            {
+                "credentials": [
+                    {"password": "123"},
+                    {"password": "ab"},
+                    {"password": ""},
+                    {"password": "mysuperpassword"},
+                ]
+            },
+        )
+
+        self.run_bot()
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0,
+            self._create_report_dict(
+                incident_data,
+                {
+                    "credentials": [
+                        {"password": "***"},
+                        {"password": "**"},
+                        {"password": ""},
+                        {"password": "mys************"},
+                    ]
+                },
+            ),
+        )

intelmq_extensions/tests/bots/collectors/xmpp/test_collector.py

@@ -0,0 +1,10 @@
+"""
+Test for the XMPP Collector Bot
+"""
+
+import os
+
+if os.environ.get("INTELMQ_TEST_EXOTIC"):
+    import intelmq_extensions.bots.collectors.xmpp.collector  # noqa
+
+# This file is a stub.

intelmq_extensions/tests/bots/experts/certat_contact_intern/test_expert.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+"""
+Testing certat_contact
+"""
+import os
+import unittest
+
+import intelmq.lib.test as test
+
+from intelmq_extensions.bots.experts.certat_contact_intern.expert import (
+    CERTatContactExpertBot,
+)
+
+from ....base import POSTGRES_CONFIG, BotTestCase
+
+if os.environ.get("INTELMQ_TEST_DATABASES"):
+    import psycopg2
+    from psycopg2 import sql
+
+INPUT1 = {
+    "__type": "Event",
+    "source.asn": 64496,
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "another-feed-code",
+}
+OUTPUT1 = {
+    "__type": "Event",
+    "source.asn": 64496,
+    "source.abuse_contact": "cert@example.com",
+    "feed.code": "another-feed-code",
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "destination_visible": True,
+}
+INPUT2 = {
+    "__type": "Event",
+    "source.asn": 64496,
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "example-feed",
+}
+OUTPUT2 = {
+    "__type": "Event",
+    "source.asn": 64496,
+    "source.abuse_contact": "cert@example.com",
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "example-feed",
+    "destination_visible": False,
+}
+OUTPUT3 = {
+    "__type": "Event",
+    "source.asn": 64496,
+    "source.abuse_contact": "cert@example.com",
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "example-feed",
+    "destination_visible": True,
+}
+INPUT4 = {
+    "__type": "Event",
+    "source.asn": 64497,
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "another-feed-code",
+}
+OUTPUT4 = {
+    "__type": "Event",
+    "source.asn": 64497,
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "feed.code": "another-feed-code",
+    "destination_visible": True,
+}
+
+
+@test.skip_database()
+class TestCERTatContactExpertBot(BotTestCase, unittest.TestCase):
+    """
+    A TestCase for CERTatContactExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = CERTatContactExpertBot
+        cls.default_input_message = INPUT1
+        if not os.environ.get("INTELMQ_TEST_DATABASES"):
+            return
+        cls.sysconfig = {
+            "autocommit": True,
+            "ascolumn": "asn",
+            "column": "contact",
+            "feed_code": "example-feed",
+            "table": "test_contacts",
+            "overwrite": False,
+        }
+        cls.sysconfig.update(POSTGRES_CONFIG)
+        cls.con = psycopg2.connect(
+            database=cls.sysconfig["database"],
+            user=cls.sysconfig["user"],
+            password=cls.sysconfig["password"],
+            host=cls.sysconfig["host"],
+            port=cls.sysconfig["port"],
+            sslmode=cls.sysconfig["sslmode"],
+        )
+        cls.con.autocommit = True
+        cls.cur = cls.con.cursor()
+        cls.create_if_not_exists(cls)
+        cls.truncate(cls)
+
+    def create_if_not_exists(self):
+        self.cur.execute(
+            sql.SQL(
+                "CREATE TABLE IF NOT EXISTS {} ({} integer, {} text, {} boolean);"
+            ).format(
+                sql.Identifier(self.sysconfig["table"]),
+                sql.Identifier(self.sysconfig["ascolumn"]),
+                sql.Identifier(self.sysconfig["column"]),
+                sql.Identifier(
+                    "can-see-tlp-amber_{}".format(self.sysconfig["feed_code"])
+                ),
+            ),
+        )
+
+    def truncate(self):
+        self.cur.execute("TRUNCATE TABLE {}".format(self.sysconfig["table"]))
+
+    def insert(self, asn, contact, tlp_amber):
+        query = """
+INSERT INTO {table}(
+    "{ascolumn}", "{column}", "can-see-tlp-amber_{feed_code}"
+) VALUES (%s, %s, %s)
+""".format(
+            table=self.sysconfig["table"],
+            column=self.sysconfig["column"],
+            feed_code=self.sysconfig["feed_code"],
+            ascolumn=self.sysconfig["ascolumn"],
+        )
+        self.cur.execute(query, (asn, contact, tlp_amber))
+
+    def test_simple(self):
+        "simple query"
+        self.insert(64496, "cert@example.com", False)
+        self.input_message = INPUT1
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT1)
+
+    def test_special_feed(self):
+        "query with special feed code"
+        self.insert(64496, "cert@example.com", False)
+        self.input_message = INPUT2
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT2)
+
+    def test_special_feed_amber(self):
+        "query with special feed code"
+        self.insert(64496, "cert@example.com", True)
+        self.input_message = INPUT2
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT3)
+
+    def test_no_result_destination_visible(self):
+        "check if destination_visible to set to true if there's no match in the DB"
+        self.input_message = INPUT4
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT4)
+
+    def tearDown(self):
+        self.truncate()
+        super().tearDown()
+
+    @classmethod
+    def tearDownClass(cls):
+        if not os.environ.get("INTELMQ_TEST_DATABASES"):
+            return
+        cls.truncate(cls)
+        cls.cur.close()
+        cls.con.close()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

intelmq_extensions/tests/bots/experts/copy_extra/test_expert.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+Testing Copy Extra expert bot.
+"""
+
+import unittest
+
+from intelmq_extensions.bots.experts.copy_extra.expert import CopyExtraExpertBot
+
+from ....base import BotTestCase
+
+INPUT = {
+    "__type": "Event",
+    "time.observation": "2015-01-01T00:00:00+00:00",
+    "extra.deviceid": "foo",
+    "extra.devicerev": "bar",
+    "extra.firmwarerev": 1,
+}
+OUTPUT = INPUT.copy()
+OUTPUT["shareable_extra_info"] = '{"deviceid": "foo", "firmwarerev": 1}'
+
+
+class TestCopyExtraExpertBot(BotTestCase, unittest.TestCase):
+    """
+    A TestCase for CopyExtraExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = CopyExtraExpertBot
+        cls.sysconfig = {"keys": ["deviceid", "firmwarerev"]}
+        cls.default_input_message = {"__type": "Event"}
+
+    def test_events(self):
+        """Test if correct Events have been produced."""
+        self.input_message = INPUT
+        self.run_bot()
+        self.assertMessageEqual(0, OUTPUT)
+
+
+if __name__ == "__main__":
+    unittest.main()

intelmq_extensions/tests/bots/experts/event_group_splitter/test_expert.py

@@ -0,0 +1,302 @@
+"""Testing events splitter
+
+SPDX-FileCopyrightText: 2024 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+import unittest
+
+from intelmq_extensions.bots.experts.event_group_splitter.expert import (
+    EventGroupSplitterExpertBot,
+)
+
+from ....base import BotTestCase
+
+INPUT = {
+    "__type": "Event",
+    "classification.identifier": "zeus",
+    "classification.type": "infected-system",
+    "notify": False,
+    "source.asn": 1,
+    "source.ip": "192.0.2.1",
+    "feed.name": "Example Feed",
+    "feed.code": "feed-1",
+}
+
+
+class TestEventGroupSplitter(BotTestCase, unittest.TestCase):
+    "Test cases prepared to handle tagging in ShadowServer Compromised Website report"
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = EventGroupSplitterExpertBot
+        cls.sysconfig = {
+            "look_in": "extra.tag",
+            "copy_to": ["classification.identifier"],
+            "regex": r"([a-zA-Z0-9\-]+)[;]{0,1}",
+            "ignore": ["tag2", "tag4"],
+            "groups_file": None,
+        }
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.create_dynamic_groups_file()
+
+    def create_dynamic_groups_file(self):
+        self.groups_file_path = f"{self.tmp_dir}/groups.json"
+        groups = {
+            "feed-1": [
+                ["citrix", "injected-code"],
+                ["backdoor-activity", "ivanti-connect-secure"],
+                ["ivanti-connect-secure", "credential-stealer", "injected-code"],
+            ],
+            "feed-3": [["test"]],
+        }
+        with open(self.groups_file_path, "w+") as f:
+            json.dump(groups, f)
+
+    def test_ignore_selected_tags(self):
+        message = {
+            **INPUT,
+            "extra.tag": "tag1,tag2,tag3,tag4",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "tag1-tag3",
+            },
+        )
+
+    def test_handle_configured_groups(self):
+        message = {
+            **INPUT,
+            "extra.tag": "citrix;injected-code;backdoor-activity;ivanti-connect-secure",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "citrix-injected-code",
+            },
+        )
+        self.assertMessageEqual(
+            1,
+            {
+                **message,
+                "classification.identifier": "backdoor-activity-ivanti-connect-secure",
+            },
+        )
+
+    def test_generate_new_groups(self):
+        message = {
+            **INPUT,
+            "extra.tag": "injected-code;cisco;backdoor-activity;ivanti-connect-secure",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "backdoor-activity-ivanti-connect-secure",
+            },
+        )
+        self.assertMessageEqual(
+            1,
+            {
+                **message,
+                "classification.identifier": "cisco-injected-code",
+            },
+        )
+
+    def test_generate_new_groups_when_duplicates(self):
+        message = {
+            **INPUT,
+            "extra.tag": "citrix;injected-code;injected-code;cisco",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "citrix-injected-code",
+            },
+        )
+        self.assertMessageEqual(
+            1,
+            {
+                **message,
+                "classification.identifier": "cisco-injected-code",
+            },
+        )
+
+        with open(self.groups_file_path) as f:
+            new_groups = json.load(f)["feed-1"]
+        self.assertEqual(4, len(new_groups))
+        self.assertIn(set(["cisco", "injected-code"]), (set(g) for g in new_groups))
+
+    def test_new_groups_are_kept(self):
+        message = {
+            **INPUT,
+            "extra.tag": "injected-code;cisco",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "cisco-injected-code",
+            },
+        )
+
+        with open(self.groups_file_path) as f:
+            new_groups = json.load(f)["feed-1"]
+        self.assertIn(set(["cisco", "injected-code"]), (set(g) for g in new_groups))
+
+        message = {
+            **INPUT,
+            "extra.tag": "cisco;injected-code;new-tag",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "cisco-injected-code",
+            },
+        )
+        self.assertMessageEqual(
+            1,
+            {
+                **message,
+                "classification.identifier": "new-tag",
+            },
+        )
+
+        with open(self.groups_file_path) as f:
+            new_groups = json.load(f)["feed-1"]
+        self.assertEqual(5, len(new_groups))
+        self.assertIn(set(["cisco", "injected-code"]), (set(g) for g in new_groups))
+        self.assertIn(set(["new-tag"]), (set(g) for g in new_groups))
+
+    def test_groups_are_separated_between_feeds(self):
+        message = {
+            **INPUT,
+            "extra.tag": "citrix,injected-code",
+            "feed.code": "feed-2",
+        }
+        self.input_message = message
+
+        self.run_bot(parameters={"groups_file": self.groups_file_path})
+
+        self.assertOutputQueueLen(1)
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "citrix-injected-code",
+            },
+        )
+
+        with open(self.groups_file_path) as f:
+            new_groups = json.load(f)
+        self.assertEqual(3, len(new_groups["feed-1"]))
+        self.assertEqual(1, len(new_groups["feed-2"]))
+        self.assertIn(
+            set(["citrix", "injected-code"]), (set(g) for g in new_groups["feed-2"])
+        )
+
+    def test_groups_are_not_separated_between_feeds_if_set(self):
+        message = {
+            **INPUT,
+            "extra.tag": "citrix,injected-code",
+            "feed.code": "feed-2",
+        }
+        self.input_message = message
+
+        self.prepare_bot(
+            parameters={
+                "groups_file": self.groups_file_path,
+                "treat_feeds_separately": False,
+            },
+            destination_queues=["_default", "new_tag_groups"],
+        )
+        self.run_bot(prepare=False)
+
+        self.assertOutputQueueLen(1)
+        self.assertOutputQueueLen(0, "new_tag_groups")
+        self.assertMessageEqual(
+            0,
+            {
+                **message,
+                "classification.identifier": "citrix-injected-code",
+            },
+        )
+
+        with open(self.groups_file_path) as f:
+            new_groups = json.load(f)
+        self.assertEqual(3, len(new_groups["feed-1"]))
+        self.assertNotIn("feed-2", new_groups)
+
+    def test_events_with_new_tag_groups_are_sent_to_specific_path(self):
+        message_with_old_tag = {**INPUT, "extra.tag": "citrix,injected-code"}
+        self.input_message = message_with_old_tag
+
+        self.prepare_bot(
+            parameters={"groups_file": self.groups_file_path},
+            destination_queues=["_default", "new_tag_groups"],
+        )
+        self.run_bot(prepare=False)
+
+        self.assertOutputQueueLen(1)
+        self.assertOutputQueueLen(0, "new_tag_groups")
+
+        message_with_new_tags = {**INPUT, "extra.tag": "new,group"}
+        self.input_message = message_with_new_tags
+
+        self.prepare_bot(
+            parameters={"groups_file": self.groups_file_path},
+            destination_queues=["_default", "new_tag_groups"],
+        )
+        self.run_bot(prepare=False)
+
+        self.assertOutputQueueLen(1)
+        self.assertOutputQueueLen(1, "new_tag_groups")
+
+        # new_tag_groups should be notified only once
+        self.input_message = message_with_new_tags
+
+        self.prepare_bot(
+            parameters={"groups_file": self.groups_file_path},
+            destination_queues=["_default", "new_tag_groups"],
+        )
+        self.run_bot(prepare=False)
+
+        self.assertOutputQueueLen(1)
+        self.assertOutputQueueLen(0, "new_tag_groups")