intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/bots/collectors/xmpp/collector.py

@@ -0,0 +1,210 @@
+"""
+XMPP Collector Bot
+Connects to a XMPP Server and a Room and reads data from the room.
+If no room is provided, which is equivalent to an empty string,
+it only collects events which were sent to the xmpp user directly.
+
+TLS is used by default.
+
+Tested with Python >= 3.4
+Tested with slixmpp >= 1.0.0-beta5
+
+Copyright (C) 2016 by Bundesamt für Sicherheit in der Informationstechnik
+Software engineering by Intevation GmbH
+
+Parameters:
+ca_certs: string to a CA-bundle file or false/empty string for no checks
+strip_message: boolean
+xmpp_user: string
+xmpp_server: string
+xmpp_password: boolean
+xmpp_room: string
+xmpp_room_password: string
+xmpp_room_nick: string
+pass_full_xml: boolean
+strip_message: boolean
+xmpp_userlist: array
+xmpp_whitelist_mode: boolean
+"""
+
+from intelmq.lib.bot import CollectorBot
+from intelmq.lib.exceptions import MissingDependencyError
+
+try:
+    import slixmpp
+
+    class XMPPClient(slixmpp.ClientXMPP):
+        def __init__(self, jid, password, room, room_nick, room_password, logger):
+            slixmpp.ClientXMPP.__init__(self, jid, password)
+
+            self.logger = logger
+            self.logger.info("Initiated.")
+            self.xmpp_room = room
+            self.xmpp_room_nick = room_nick
+            self.xmpp_room_password = room_password
+
+            self.add_event_handler("session_start", self.session_start)
+
+        def session_start(self, event):
+            self.send_presence()
+            self.logger.debug("Session started.")
+
+            try:
+                self.get_roster()
+            except slixmpp.exceptions.IqError as err:
+                self.logger.error("There was an error getting the roster.")
+                self.logger.error(err.iq["error"]["condition"])
+                self.disconnect()
+            except slixmpp.exceptions.IqTimeout:
+                self.logger.error("Server is taking too long to respond.")
+                self.disconnect()
+
+            if (
+                self.xmpp_room
+            ):  # and self.plugin.get('xep_0045') # this check should also exist!
+                self.logger.debug("Joining room: %s.", self.xmpp_room)
+                pwd = self.xmpp_room_password if self.xmpp_room_password else ""
+                self.plugin["xep_0045"].joinMUC(
+                    self.xmpp_room, self.xmpp_room_nick, password=pwd, wait=True
+                )
+
+except ImportError:
+    slixmpp = None
+
+
+class XMPPCollectorBot(CollectorBot):
+    name = "XMPP"
+    xmpp = None
+    collector_empty_process = True
+
+    def init(self):
+        self.logger.warning(
+            "The output bot 'intelmq.bots.collectors.xmpp.output' "
+            "is deprecated. It will be removed in version 3.0."
+            "Please see https://github.com/certtools/intelmq/blob/"
+            "develop/NEWS.md#xmpp-bots for more details."
+        )
+        if slixmpp is None:
+            raise MissingDependencyError("slixmpp")
+
+        # Retrieve Parameters from configuration
+        xmpp_user = getattr(self.parameters, "xmpp_user", None)
+        xmpp_server = getattr(self.parameters, "xmpp_server", None)
+        xmpp_password = getattr(self.parameters, "xmpp_password", None)
+
+        if None in (xmpp_user, xmpp_server, xmpp_password):
+            raise ValueError("No User / Password provided.")
+        else:
+            xmpp_login = xmpp_user + "@" + xmpp_server
+
+        self.userlist = getattr(self.parameters, "xmpp_userlist", [])
+        # When configured in manager this is most likely a ,-separated string, we'd like an array
+        if type(self.userlist) is str:
+            self.userlist = [u.strip() for u in self.userlist.split(",")]
+        elif self.userlist is None:  # if value is unset, set to empty list
+            self.userlist = []
+
+        self.whitelist_mode = getattr(self.parameters, "xmpp_whitelist_mode", False)
+
+        self.muc = getattr(self.parameters, "use_muc", None)
+        xmpp_room = getattr(self.parameters, "xmpp_room", None) if self.muc else None
+        xmpp_room_nick = (
+            getattr(self.parameters, "xmpp_room_nick", None) if self.muc else None
+        )
+        xmpp_room_password = (
+            getattr(self.parameters, "xmpp_room_password", None) if self.muc else None
+        )
+
+        self.pass_full_xml = getattr(self.parameters, "pass_full_xml", None)
+        self.strip_message = getattr(self.parameters, "strip_message", None)
+
+        ca_certs = getattr(self.parameters, "ca_certs", None)
+
+        if self.muc and not xmpp_room:
+            raise ValueError("No room provided.")
+
+        if self.muc:
+            if not xmpp_room_nick:
+                # create the room_nick from user and server
+                xmpp_room_nick = xmpp_login
+
+        self.xmpp = XMPPClient(
+            xmpp_login,
+            xmpp_password,
+            xmpp_room,
+            xmpp_room_nick,
+            xmpp_room_password,
+            self.logger,
+        )
+
+        if ca_certs:
+            # Set CA-Certificates
+            self.xmpp.ca_certs = ca_certs
+
+        if self.xmpp.connect(reattempt=False):
+            self.xmpp.process()
+            # Add Handlers and register Plugins
+            self.xmpp.register_plugin("xep_0030")  # Service Discovery
+            self.xmpp.register_plugin("xep_0045")  # Multi-User Chat
+
+            self.xmpp.add_event_handler("message", self.log_message)
+
+        else:
+            raise ValueError("Could not connect to XMPP-Server.")
+
+    def process(self):
+        # Processing is done by function called from the eventhandler...
+        pass
+
+    def shutdown(self):
+        if self.xmpp:
+            if self.xmpp.disconnect():
+                self.logger.info("Disconnected from XMPP Server.")
+            else:
+                self.logger.error("Could not disconnect from XMPP Server.")
+        else:
+            self.logger.info("There was no XMPPClient I could stop.")
+
+    def log_message(self, msg):
+        # If some exception happens here, the bot would silently fail.
+        # We want to know the reason, so we will log the exception manually.
+        try:
+            # Check if the message was sent by a users that is on
+            # the white or blacklist, determine if the message shall
+            # be processed.
+            if self.muc:
+                if msg["mucnick"] not in self.userlist and self.whitelist_mode:
+                    # Whitelist-Case
+                    return
+                elif msg["mucnick"] in self.userlist and not self.whitelist_mode:
+                    # Blacklist Case
+                    return
+
+            if self.pass_full_xml:
+                body = str(msg)
+            else:
+                if self.strip_message:
+                    body = msg["body"].strip()
+                else:
+                    body = msg["body"]
+
+            if len(body) > 400:
+                tmp_body = body[:397] + "..."
+            else:
+                tmp_body = body
+
+            self.logger.debug("Received Stanza: %r from %r.", tmp_body, msg["from"])
+
+            raw_msg = body
+            # Read msg-body and add as raw to a new report.
+            # now it's up to a parser to do the interpretation of the message.
+            if raw_msg:
+                report = self.new_report()
+                report.add("raw", raw_msg)
+                self.send_message(report)
+        except Exception:
+            self.logger.exception("Error during message handling.")
+            raise
+
+
+BOT = XMPPCollectorBot

intelmq_extensions/bots/experts/certat_contact_intern/expert.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+"""
+CERT.at geolocate the national CERT abuse service
+"""
+from intelmq.lib.bot import Bot
+
+try:
+    import psycopg2
+except ImportError:
+    psycopg2 = None
+
+
+class CERTatContactExpertBot(Bot):
+    connect_timeout = 5
+    database: str = ""
+    user: str = ""
+    password: str = ""
+    host: str = ""
+    port: str = ""
+    sslmode: str = ""
+    autocommit: bool = True
+    table: str = "contacts"
+    column: str = ""
+    feed_code: str = ""
+    ascolumn: str = ""
+    overwrite: bool = False
+
+    def init(self):
+        self.logger.debug("Connecting to database.")
+        if psycopg2 is None:
+            self.logger.error("Could not import psycopg2. Please install it.")
+            self.stop()
+
+        try:
+            self.con = psycopg2.connect(
+                database=self.database,
+                user=self.user,
+                password=self.password,
+                host=self.host,
+                port=self.port,
+                sslmode=self.sslmode,
+                connect_timeout=self.connect_timeout,
+            )
+            self.cur = self.con.cursor()
+            self.con.autocommit = self.autocommit
+
+        except Exception:
+            self.logger.exception("Failed to connect to database.")
+            self.stop()
+        self.logger.info("Connected to PostgreSQL.")
+
+        self.query = (
+            'SELECT "{column}", "can-see-tlp-amber_{feed_code}"'
+            ' FROM "{table}" WHERE "{ascolumn}" = %s'
+            "".format(
+                table=self.table,
+                column=self.column,
+                feed_code=self.feed_code,
+                ascolumn=self.ascolumn,
+            )
+        )
+
+    def process(self):
+        event = self.receive_message()
+        default_destination_visible = (
+            True if event.get("feed.code") != self.feed_code else False
+        )
+
+        if "source.asn" not in event:
+            self.logger.info("source.asn not present in event. Skipping event.")
+            event.add(
+                "destination_visible",
+                default_destination_visible,
+                overwrite=self.overwrite,
+            )
+            self.send_message(event)
+            self.acknowledge_message()
+            return
+
+        if "source.abuse_contact" in event and not self.overwrite:
+            event.add(
+                "destination_visible",
+                default_destination_visible,
+                overwrite=self.overwrite,
+            )
+            self.send_message(event)
+            self.acknowledge_message()
+            return
+
+        try:
+            self.logger.debug(
+                "Executing %r." % self.cur.mogrify(self.query, (event["source.asn"],))
+            )
+            self.cur.execute(self.query, (event["source.asn"],))
+        except (
+            psycopg2.InterfaceError,
+            psycopg2.InternalError,
+            psycopg2.OperationalError,
+            AttributeError,
+        ):
+            self.logger.exception("Database connection problem, connecting again.")
+            self.init()
+        else:
+            if self.cur.rowcount > 1:
+                raise ValueError(
+                    "Lookup returned more than one result. Please inspect."
+                )
+            elif self.cur.rowcount == 1:
+                result = self.cur.fetchone()
+                self.logger.debug(
+                    "Changing `source.abuse_contact` from %r to %r."
+                    % (event.get("source.abuse_contact"), result[0])
+                )
+
+                event.add("source.abuse_contact", result[0], overwrite=self.overwrite)
+
+                if event["feed.code"] == self.feed_code:
+                    if result[1]:
+                        event.add("destination_visible", True, overwrite=self.overwrite)
+                    else:
+                        event.add(
+                            "destination_visible", False, overwrite=self.overwrite
+                        )
+                else:
+                    event.add("destination_visible", True, overwrite=self.overwrite)
+
+            else:
+                self.logger.debug("No contact found.")
+                event.add(
+                    "destination_visible",
+                    default_destination_visible,
+                    overwrite=self.overwrite,
+                )
+
+            self.send_message(event)
+            self.acknowledge_message()
+
+
+BOT = CERTatContactExpertBot

intelmq_extensions/bots/experts/copy_extra/expert.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+Modify Expert bot let's you manipulate all fields with a config file.
+"""
+import json
+
+from intelmq.lib.bot import Bot
+
+
+class CopyExtraExpertBot(Bot):
+    keys: list = []
+
+    def process(self):
+        event = self.receive_message()
+
+        if "extra" in event:
+            extra = json.loads(event["extra"])
+
+            share = {key: extra[key] for key in self.keys if key in extra}
+            if share:
+                event.add("shareable_extra_info", share, overwrite=False)
+
+        self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = CopyExtraExpertBot

intelmq_extensions/bots/experts/event_group_splitter/expert.py

@@ -0,0 +1,117 @@
+"""Event Group Splitter can split events based on tag groups,
+and automatically recognize new groups.
+
+The bot is designed with an intention to handle ShadowServer Compromise Website report.
+
+SPDX-FileCopyrightText: 2024 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import itertools
+import json
+import re
+
+from intelmq.lib.bot import ExpertBot
+
+
+class EventGroupSplitterExpertBot(ExpertBot):
+    look_in: str = "extra.tag"
+    copy_to: list[str] = ["classification.identifier", "extra.vulnerabilities"]
+    regex: str = ""  # has to match one group meaning matched duplication value
+    ignore: list[str] = []  # tags to ignore
+    # path to a JSON file holding groups; the file will be updated with new groups
+    # the structure is a dict with keys as feed codes, and values as list of list of tags
+    # e.g.:
+    # {
+    #   "feed-1": [["t1", "t2"], ["t1", "t3"]],
+    #   "feed-2": [["t6"]]
+    # }
+    groups_file: str = None
+    # whether groups should be analysed separately for every feed or not;
+    # if not, the bot will not use "new group" path for events matching
+    # group already seen in any other feed
+    treat_feeds_separately: bool = True
+
+    def init(self):
+        self.matcher = re.compile(self.regex)
+
+        self._tag_groups: dict[str, list[list[str]]] = []
+        self._reload_groups()
+
+    def _reload_groups(self, dump_current=False):
+        if dump_current:
+            with open(self.groups_file, "w+") as f:
+                json.dump(
+                    {
+                        feed: [list(g) for g in groups]
+                        for feed, groups in self._tag_groups.items()
+                    },
+                    f,
+                    indent=4,
+                )
+
+        with open(self.groups_file) as f:
+            new_groups = json.load(f)
+
+        self._tag_groups = {
+            feed: sorted(
+                (set(group) for group in groups), key=lambda g: len(g), reverse=True
+            )
+            for feed, groups in new_groups.items()
+        }
+
+    def process(self):
+        event = self.receive_message()
+
+        lookup_data = event.get(self.look_in, "")
+        matches = self.matcher.findall(lookup_data)
+        if not matches:
+            self.send_message(event)
+        else:
+            matches = [m for m in matches if m not in self.ignore]
+            matches_set = set(matches)
+            feed = event.get("feed.code", "")
+            if self.treat_feeds_separately:
+                feed_groups = self._tag_groups.get(feed, [])
+            else:
+                feed_groups = itertools.chain(*self._tag_groups.values())
+
+            for group in feed_groups:
+                if not matches_set >= group:
+                    # The group is not in event's tags
+                    continue
+                self._generate_event(event, group)
+
+                # Tags can be duplicated in event. Respect it.
+                for tag in group:
+                    matches.remove(tag)
+                matches_set = set(matches)
+
+                if not matches:
+                    break
+
+            # Something is left - new group
+            if matches:
+                new_group = set(matches)
+                self.logger.info("New tag group was discovered: %s.", new_group)
+                self._generate_event(event, new_group, new_group=True)
+
+                if feed not in self._tag_groups:
+                    self._tag_groups[feed] = list()
+                self._tag_groups[feed].append(new_group)
+                self._reload_groups(dump_current=True)
+
+        self.acknowledge_message()
+
+    def _generate_event(self, event, group: set, new_group: bool = False):
+        grouped_value = "-".join(sorted(group))
+        self.logger.debug("Found: %s.", group)
+        sub_event = self.new_event(event)
+        for key in self.copy_to:
+            sub_event.add(key, grouped_value, overwrite=True)
+        self.send_message(sub_event)
+        if new_group:
+            self.send_message(sub_event, path="new_tag_groups", path_permissive=True)
+
+
+BOT = EventGroupSplitterExpertBot

intelmq_extensions/bots/experts/event_splitter/expert.py

@@ -0,0 +1,41 @@
+"""Event Splitter can produce multiple events from a one, based
+on event's tag.
+
+Currently implemented with intention of splitting CVE events.
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import re
+
+from intelmq.lib.bot import ExpertBot
+
+
+class EventSplitterExpertBot(ExpertBot):
+    look_in: str = "extra.tag"
+    copy_to: list[str] = ["classification.identifier", "extra.vulnerabilities"]
+    regex: str = ""  # has to match one group meaning matched duplication value
+
+    def init(self):
+        self.matcher = re.compile(self.regex)
+
+    def process(self):
+        event = self.receive_message()
+
+        lookup_data = event.get(self.look_in, "")
+        matches = self.matcher.findall(lookup_data)
+        if not matches:
+            self.send_message(event)
+        else:
+            for matched in matches:
+                self.logger.debug("Found: %s.", matched)
+                sub_event = self.new_event(event)
+                for key in self.copy_to:
+                    sub_event.add(key, matched, overwrite=True)
+                self.send_message(sub_event)
+
+        self.acknowledge_message()
+
+
+BOT = EventSplitterExpertBot