intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/bots/outputs/xmpp/output.py

@@ -0,0 +1,180 @@
+"""
+XMPP Output Bot
+Connects to a XMPP Server and sends data to a user.
+
+TLS is used by default.
+
+Tested with Python >= 3.4
+Tested with slixmpp >= 1.0.0-beta5
+
+Copyright (C) 2016 by Bundesamt für Sicherheit in der Informationstechnik
+Software engineering by Intevation GmbH
+
+Parameters:
+ca_certs: string to a CA-bundle file or false/empty string for no checks
+hierarchical_output: boolean (false by default)
+xmpp_user: string
+xmpp_server: string
+xmpp_password: boolean
+xmpp_to_user: string
+xmpp_to_server: string
+xmpp_room: string
+xmpp_room_nick: string
+xmpp_room_password: string
+use_muc: boolean
+"""
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.exceptions import MissingDependencyError
+
+try:
+    import slixmpp
+
+    class XMPPClient(slixmpp.ClientXMPP):
+        def __init__(self, jid, password, room, room_nick, room_password, logger):
+            slixmpp.ClientXMPP.__init__(self, jid, password)
+
+            self.xmpp_room = room
+            self.xmpp_room_nick = room_nick
+            self.xmpp_room_password = room_password
+
+            self.logger = logger
+            self.add_event_handler("session_start", self.session_start)
+
+            self.logger.info("Initialized XMPP Client.")
+
+        def session_start(self, event):
+            self.send_presence()
+            self.logger.debug("Session started.")
+
+            try:
+                self.get_roster()
+            except slixmpp.exceptions.IqError as err:
+                self.logger.error("There was an error getting the roster.")
+                self.logger.error(err.iq["error"]["condition"])
+                self.disconnect()
+            except slixmpp.exceptions.IqTimeout:
+                self.logger.error("Server is taking too long to respond.")
+                self.disconnect()
+
+            if (
+                self.xmpp_room
+            ):  # and self.plugin.get('xep_0045') # this check should also exist!
+                self.logger.debug("Joining room: %s.", self.xmpp_room)
+                pwd = self.xmpp_room_password if self.xmpp_room_password else ""
+                self.plugin["xep_0045"].joinMUC(
+                    self.xmpp_room, self.xmpp_room_nick, password=pwd, wait=True
+                )
+
+except ImportError:
+    slixmpp = None
+
+
+class XMPPOutputBot(Bot):
+    xmpp = None
+
+    def init(self):
+        self.logger.warning(
+            "The output bot 'intelmq.bots.outputs.xmpp.output' "
+            "is deprecated. It will be removed in version 3.0."
+            "Please see https://github.com/certtools/intelmq/blob/"
+            "develop/NEWS.md#xmpp-bots for more details."
+        )
+        if slixmpp is None:
+            raise MissingDependencyError("slixmpp")
+
+        # Retrieve Parameters from configuration
+        xmpp_user = getattr(self.parameters, "xmpp_user", None)
+        xmpp_server = getattr(self.parameters, "xmpp_server", None)
+        xmpp_password = getattr(self.parameters, "xmpp_password", None)
+
+        if None in (xmpp_user, xmpp_server, xmpp_password):
+            raise ValueError("No User / Password provided.")
+        else:
+            xmpp_login = xmpp_user + "@" + xmpp_server
+
+        self.muc = getattr(self.parameters, "use_muc", None)
+        xmpp_to_user = getattr(self.parameters, "xmpp_to_user", None)
+        xmpp_to_server = getattr(self.parameters, "xmpp_to_server", None)
+        xmpp_room = getattr(self.parameters, "xmpp_room", None) if self.muc else None
+        xmpp_room_nick = (
+            getattr(self.parameters, "xmpp_room_nick", None) if self.muc else None
+        )
+        xmpp_room_password = (
+            getattr(self.parameters, "xmpp_room_password", None) if self.muc else None
+        )
+
+        ca_certs = getattr(self.parameters, "ca_certs", None)
+
+        # Be sure the receiver was set up
+        if not self.muc and None in (xmpp_to_user, xmpp_to_server):
+            raise ValueError("No receiver for direct messages provided.")
+        else:
+            self.xmpp_receiver = xmpp_to_user + "@" + xmpp_to_server
+
+        if self.muc and not xmpp_room:
+            raise ValueError("No room provided.")
+        else:
+            self.xmpp_receiver = xmpp_room
+
+        if self.muc:
+            if not xmpp_room_nick:
+                # create the room_nick from user and server
+                xmpp_room_nick = xmpp_login
+
+        self.xmpp = XMPPClient(
+            xmpp_login,
+            xmpp_password,
+            xmpp_room,
+            xmpp_room_nick,
+            xmpp_room_password,
+            self.logger,
+        )
+
+        if ca_certs:
+            # Set CA-Certificates
+            self.xmpp.ca_certs = ca_certs
+
+        if self.xmpp.connect(reattempt=False):
+            self.xmpp.process()
+            # Add Handlers and register Plugins
+            self.xmpp.register_plugin("xep_0030")  # Service Discovery
+            self.xmpp.register_plugin("xep_0045")  # Multi-User Chat
+        else:
+            raise ValueError("Could not connect to XMPP-Server.")
+
+    def process(self):
+        event = self.receive_message()
+
+        jevent = event.to_json(
+            hierarchical=self.parameters.hierarchical_output, with_type=True
+        )
+
+        try:
+            # TODO: proper error handling.
+            # Right now it cannot be detected if the message was sent successfully.
+            if self.muc:
+                self.logger.debug("Trying to send to room %s.", self.xmpp_receiver)
+                self.xmpp.send_message(
+                    mto=self.xmpp_receiver, mbody=jevent, mtype="groupchat"
+                )
+            else:
+                self.logger.debug("Trying to send to %s.", self.xmpp_receiver)
+                self.xmpp.send_message(mto=self.xmpp_receiver, mbody=jevent)
+        except slixmpp.exceptions.XMPPError as err:
+            self.logger.error("There was an error when sending the event.")
+            self.logger.error(err.iq["error"]["condition"])
+
+        self.acknowledge_message()
+
+    def shutdown(self):
+        if self.xmpp:
+            if self.xmpp.disconnect():
+                self.logger.info("Disconnected from XMPP Server.")
+            else:
+                self.logger.error("Could not disconnect from XMPP Server.")
+        else:
+            self.logger.info("There was no XMPPClient I could stop.")
+
+
+BOT = XMPPOutputBot

intelmq_extensions/bots/parsers/blackkite/_transformers.py

@@ -0,0 +1,202 @@
+"""Transformers for the BlackKite
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+from abc import ABC
+from datetime import datetime
+from typing import Iterable, Sequence
+
+import dateutil.parser
+from dateutil.tz import UTC
+from intelmq.lib.harmonization import IPAddress
+
+from intelmq_extensions.lib.blackkite import Category
+
+
+class BaseTransformer(ABC):
+    DEFAULT_CLASSIFICATION = ("", "", "")
+    CLASSIFICATION_MAP: dict[str, tuple] = {}
+    _COMMON_FIELDS = (
+        ("extra.feed_event_id", "FindingId"),
+        # In general, the Domain field does not represent the affected domain.
+        # It can be a primary domain of the affected URL, or just the domain
+        # configured for the given company, without direct relation to the finding.
+        # FIXME: waiting for BlackKite response on how to get the affected URL
+        ("source.fqdn", "", "get_domain"),
+        ("source.ip", "IpAddress", "get_ip"),
+        ("event_description.text", "Detail", "get_description"),
+        # The date that Black Kite first seen the finding.
+        ("time.source", "FindingDate"),
+        ("feed.documentation", "ControlId", "get_documentation_url"),
+    )
+    SPECIFIC_FIELDS = ()
+
+    def __init__(self) -> None:
+        self._data = None
+        self._transformed = {}
+
+    def to_events_data(self, incident_data: dict) -> Iterable[dict]:
+        self._data = incident_data
+        self._transformed = {}
+        self._finding_id = incident_data.get("ControlId")
+        self.transform()
+        for event_data in self.transform_single():
+            yield event_data
+
+    def _map(self, event_key: str, finding_key: str, mapper: str = None):
+        if finding_key in self._data:
+            if not mapper:
+                self._transformed[event_key] = self._data[finding_key]
+            else:
+                self._transformed[event_key] = getattr(self, mapper)(finding_key)
+
+    def _map_many(self, mappings: Sequence[tuple]):
+        for mapping in mappings:
+            self._map(*mapping)
+
+    def _map_classification(self):
+        taxonomy, type_, identifier = self.CLASSIFICATION_MAP.get(
+            self._finding_id, self.DEFAULT_CLASSIFICATION
+        )
+        self._transformed["classification.taxonomy"] = taxonomy
+        self._transformed["classification.type"] = type_
+        self._transformed["classification.identifier"] = identifier
+
+    def _map_feed_data(self):
+        category = self._data.get("ControlId", "").split("-")[0]
+        self._transformed["feed.code"] = f"blackkite-{category.lower()}"
+        self._transformed["feed.name"] = f"BlackKite {category.upper()}"
+
+    def get_description(self, _):
+        description = []
+        if title := self._data.get("Title"):
+            description.append(title)
+        if detail := self._data.get("Detail"):
+            description.append(detail)
+        return ". ".join(description)
+
+    def transform(self):
+        self._map_classification()
+        self._map_feed_data()
+        self._map_many(self._COMMON_FIELDS)
+        self._map_many(self.SPECIFIC_FIELDS)
+
+    def transform_single(self) -> Iterable[dict]:
+        """If an incident produce multiple events,
+        this method should do the extraction and transformation"""
+
+        yield self._transformed
+
+    def get_documentation_url(self, _):
+        return f"https://cyber.riskscore.cards/kb/{self._finding_id}"
+
+    def lower(self, field: str) -> str:
+        return (self._data.get(field) or "").lower()
+
+    def get_ip(self, field: str):
+        # BlackKite can send domain in the IPAddress field
+        ip_data = self._data.get(field)
+        if IPAddress.is_valid(ip_data, sanitize=True):
+            return ip_data
+        return None
+
+    def get_domain(self, field: str):
+        # BlackKite can send domain in the IPAddress field
+        # as it's then a better domain, this should be used
+        if ip_data := self._data.get("IpAddress"):
+            if not IPAddress.is_valid(ip_data):
+                return ip_data
+
+        if field:
+            return self._data.get(field)
+
+    def get_first(self, field: str) -> str:
+        """Get first item from the list"""
+        items = self._data.get(field)
+        if items:
+            return items[0]
+        return None
+
+
+class PatchManagementTransformer(BaseTransformer):
+    DEFAULT_CLASSIFICATION = ("vulnerable", "vulnerable-system", "bk-patchmanagement")
+    CLASSIFICATION_MAP = {
+        "PATCH-010": ("vulnerable", "vulnerable-system", "end-of-live")
+    }
+    SPECIFIC_FIELDS = (
+        ("extra.product_name", "ProductName"),
+        # lower to match with data from other sources
+        ("extra.vulnerabilities", "CveId", "lower"),
+        ("event_description.url", "References", "get_first"),
+    )
+
+    def _map_cpe(self):
+        cpes = self._data.get("Cpes")
+        if not cpes:
+            return
+        cpe_data = cpes[0].split(":")
+        self._transformed["extra.vendor"] = cpe_data[3]
+        self._transformed["extra.product"] = cpe_data[4]
+
+    def _map_cve_to_identifier(self):
+        if cve_id := self._data.get("CveId"):
+            self._transformed["classification.identifier"] = cve_id.lower()
+
+    def transform(self) -> Iterable[dict]:
+        super().transform()
+        self._map_cpe()
+        self._map_cve_to_identifier()
+
+
+class ApplicationSecurityTransformer(BaseTransformer):
+    CLASSIFICATION_MAP = {
+        "APPSEC-014": ("vulnerable", "potentially-unwanted-accessible", "bk-appsec")
+    }
+
+
+class CredentialManagementTransformer(BaseTransformer):
+    DEFAULT_CLASSIFICATION = (
+        "information-content-security",
+        "data-leak",
+        "leaked-credentials",
+    )
+    SPECIFIC_FIELDS = (
+        ("source.fqdn", "EmailorUsername", "get_domain_from_email"),
+        ("source.account", "EmailorUsername"),
+        ("extra.account", "EmailorUsername"),
+        ("extra.password", "PasswordType"),
+        ("extra.compromise_time_full", "LeakDate"),
+        ("extra.compromise_time", "LeakDate", "get_compromise_time"),
+        ("extra.leak_source", "Source"),
+    )
+    _DESCRIPTION = (
+        "A user with email in your domain"
+        " was found in leaked credentials related to: {source}"
+    )
+    _COMPROMISE_TIME_FORMAT = "%Y-%m"
+
+    def get_description(self, _):
+        return self._DESCRIPTION.format(source=self._data.get("Source"))
+
+    def get_compromise_time(self, _):
+        # return the month only, as in DISP parser
+        date_str = self._data.get("LeakDate")
+        if not date_str:
+            return None
+        date: datetime = dateutil.parser.parse(date_str).astimezone(tz=UTC)
+        return date.strftime(self._COMPROMISE_TIME_FORMAT)
+
+    def get_domain_from_email(self, field):
+        data = self._data.get(field)
+        if "@" in data:
+            return data.split("@")[-1]
+        return None
+
+
+TRANSFORMERS_MAPPER: dict[Category, BaseTransformer] = {
+    Category.PatchManagement: PatchManagementTransformer(),
+    Category.ApplicationSecurity: ApplicationSecurityTransformer(),
+    Category.CredentialManagement: CredentialManagementTransformer(),
+}

intelmq_extensions/bots/parsers/blackkite/parser.py

@@ -0,0 +1,65 @@
+"""Parser for BlackKite feeds
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Union
+
+import intelmq.lib.message as message
+from intelmq.lib import utils
+from intelmq.lib.bot import ParserBot
+
+from intelmq_extensions.lib.blackkite import Category
+
+from ._transformers import TRANSFORMERS_MAPPER
+
+
+@dataclass
+class PreparedData:
+    raw_report: str
+    company: dict
+    event_data: dict
+
+
+class BlackKiteParserBot(ParserBot):
+    def init(self):
+        pass
+
+    def parse(self, report: message.Report):
+        raw_report = utils.base64_decode(report.get("raw"))
+        self._current_line = raw_report
+
+        data = json.loads(raw_report)
+        company = data["company"]
+        finding = data["finding"]
+
+        category = Category(finding["ControlId"].split("-")[0])
+        for event_data in TRANSFORMERS_MAPPER[category].to_events_data(finding):
+            self._current_line = PreparedData(raw_report, company, event_data)
+            yield self._current_line
+
+    def parse_line(self, data: PreparedData, report: message.Report):
+        event = self.new_event(report)
+        event.add("raw", data.raw_report)
+
+        event.add("extra.monitored_asset", data.company.get("DomainName"))
+        event.add("extra.blackkite_company_id", data.company.get("CompanyId"))
+
+        for key, value in data.event_data.items():
+            event.add(key, value, overwrite=True)
+
+        if "time.source" not in event:
+            event.add("time.source", event.get("time.observation"))
+
+        return event
+
+    def recover_line(self, line: Union[str, None, PreparedData] = None) -> str:
+        if isinstance(line, str):
+            return super().recover_line(line)
+        return super().recover_line(line.raw_report)
+
+
+BOT = BlackKiteParserBot

intelmq_extensions/bots/parsers/disp/parser.py

@@ -0,0 +1,125 @@
+"""Parsing DISP data feed.
+
+Currently only for credentials tracking
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+from copy import deepcopy
+from datetime import datetime
+from urllib import parse
+
+import intelmq.lib.message as message
+from dateutil.parser import ParserError
+from dateutil.parser import parse as dt_parse
+from dateutil.tz import UTC
+from dns.exception import DNSException
+from intelmq.lib import utils
+from intelmq.lib.bot import ParserBot
+from intelmq.lib.exceptions import IntelMQException
+from intelmq.lib.harmonization import URL, DateTime
+
+
+class InvalidData(IntelMQException, ValueError):
+    """Given message is invalid comparing with parser requirements"""
+
+
+class DISPParserBot(ParserBot):
+    compromise_time_format: str = "%Y-%m"  # empty, 'original' or format string
+    redact_url_path: bool = True
+    resolve_ip: bool = False
+
+    def parse(self, report: message.Report):
+        raw_report = utils.base64_decode(report.get("raw"))
+        report_data = json.loads(raw_report)
+
+        incident = report_data.get("incident")
+        evidences = report_data.get("evidences", {}).get("credentials")
+
+        if not evidences:
+            self.logger.error("Report doesn't contain any evidences")
+            raise InvalidData("No evidences in DISP report")
+
+        for evidence in evidences:
+            current = {"incident": deepcopy(incident), "evidence": evidence}
+            self._current_line = json.dumps(current)
+            yield current
+
+    def parse_line(self, line: dict, report: message.Report):
+        event = self.new_event(report)
+        event.add("raw", json.dumps(line))
+
+        incident, evidence = line.get("incident"), line.get("evidence")
+        self._map_incident(incident, event)
+        self._map_evidence(evidence, event)
+        return event
+
+    def _map_incident(self, incident: dict, event: message.Event):
+        event.add("event_description.text", incident.get("title"))
+        event.add("extra.feed_event_id", incident.get("id"))
+        event.add(
+            "time.source", DateTime.from_epoch_millis(incident.get("validationDate"))
+        )
+        event.add("extra.monitored_asset", incident.get("relatedAssets", [""])[0])
+
+    def _map_evidence(self, evidence: dict, event: message.Event):
+        url = evidence.get("url")
+        parsed_url = parse.urlsplit(url)
+        event.add("source.fqdn", parsed_url.netloc)
+
+        if self.resolve_ip:
+            try:
+                event.add("source.ip", URL.to_ip(url))
+            except DNSException:
+                self.logger.warning("Cannot get IP for domain %s.", url, exc_info=True)
+
+        path = parsed_url.path
+        if self.redact_url_path and parsed_url.path and parsed_url.path != "/":
+            path = "/[REDACTED]"
+        event.add(
+            "source.url",
+            f"{parsed_url.scheme}://{parsed_url.netloc}{path}",
+            sanitize=True,
+        )
+        event.add("source.urlpath", path)
+        event.add("extra.full_url", self._disarm_url(url))
+
+        event.add("source.account", evidence.get("username"))
+        event.add("extra.account", evidence.get("username"))
+        event.add("extra.password", evidence.get("password"))
+        event.add("extra.application", evidence.get("application"))
+        event.add("extra.compromise_time", self._parse_compromise_time(evidence))
+        event.add("extra.compromise_time_full", evidence.get("date"))
+
+        event.add("malware.name", evidence.get("malware"))
+
+    @staticmethod
+    def _disarm_url(url: str):
+        """Prevents URLs being usable"""
+        url = url.replace("http://", "hxxp://")
+        return url.replace("https://", "hxxps://")
+
+    def _parse_compromise_time(self, evidence: dict):
+        compromise_time = evidence.get("date")
+        if not compromise_time:
+            return None
+
+        if self.compromise_time_format == "original":
+            return compromise_time
+
+        if not self.compromise_time_format:
+            return ""
+
+        try:
+            dt: datetime = dt_parse(compromise_time).astimezone(tz=UTC)
+        except ParserError:
+            self.logger.warning(
+                "Error parsing compromise time %s.", compromise_time, exc_info=True
+            )
+            return None
+        return dt.strftime(self.compromise_time_format)
+
+
+BOT = DISPParserBot

intelmq_extensions/bots/parsers/malwaredomains/parser.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+The descriptions give a hint about what the entry is about and is very mixed.
+Most prominent description is "phishing", most of them are malware names.
+More types could be mapped better, only the most obvious ones are done currently.
+"""
+import datetime
+
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+
+
+class MalwareDomainsParserBot(Bot):
+    def is_valid_date(self, strd):
+        try:
+            datetime.datetime.strptime(strd, "%Y%m%d")
+            return True
+        except Exception:
+            return False
+
+    def process(self):
+        report = self.receive_message()
+
+        raw_report = utils.base64_decode(report.get("raw"))
+
+        for row in raw_report.splitlines():
+            row = row.rstrip()
+
+            if row.startswith("#") or len(row) == 0:
+                continue
+
+            values = row.split("\t")[1:]
+
+            event = self.new_event(report)
+
+            event.add("source.fqdn", values[1])
+            if values[2] == "phishing":
+                event.add("classification.identifier", values[2])
+                event.add("classification.type", "phishing")
+            elif values[2] == "C&C":
+                event.add("classification.identifier", values[2])
+                event.add("classification.type", "c2-server")
+            else:
+                event.add("classification.identifier", values[2])
+                event.add("classification.type", "malware")
+            event.add("event_description.text", values[2])
+
+            for i in range(4, len(values)):
+                if self.is_valid_date(values[i]):
+                    event.add(
+                        "time.source",  # times are GMT, verified via email
+                        values[i] + "T00:00:00+00:00",
+                        overwrite=True,
+                    )
+                    break
+
+            event.add("raw", row)
+
+            self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = MalwareDomainsParserBot