intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/bots/experts/squelcher/expert.py

@@ -0,0 +1,316 @@
+# -*- coding: utf-8 -*-
+"""
+Squelcher Expert marks events as new or old depending on a TTL(ASN, Net, IP).
+"""
+
+from __future__ import unicode_literals
+
+from ipaddress import ip_address, ip_network
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+from intelmq.lib.utils import load_configuration
+
+try:
+    import psycopg2
+except ImportError:
+    psycopg2 = None
+try:
+    import netaddr
+except ImportError:
+    netaddr = None
+
+"""
+If the event in the DB is older than 2 days, then we also check if it has been sent out.
+If this is not the case, we assume the event will be sent out, thus we squelch the new event.
+"""
+SELECT_QUERY = """
+SELECT COUNT(*) FROM {table}
+WHERE
+"time.source" >= LOCALTIMESTAMP - INTERVAL '%(ttl)s SECONDS' AND
+"classification.type" = %(type)s AND
+"classification.identifier" = %(identifier)s AND
+{source_filters}
+notify IS TRUE AND
+("time.source" >= LOCALTIMESTAMP - INTERVAL %(sending_interval)s OR
+ (sent_at IS NOT NULL AND "time.source" < LOCALTIMESTAMP - INTERVAL %(sending_interval)s)
+)
+"""
+
+# If the event is newer than sending interval, assume it will be sent soon or already has been,
+# regardless of the report id. If the source time is older, ignore events without report id as
+# they were most probably forgotten
+OPEN_REPORT_QUERY = """
+SELECT COUNT(*) FROM {table}
+WHERE
+"time.source" >= LOCALTIMESTAMP - INTERVAL '%(ttl)s SECONDS' AND
+"classification.type" = %(type)s AND
+"classification.identifier" = %(identifier)s AND
+{source_filters}
+notify IS TRUE AND
+(
+    "time.source" >= LOCALTIMESTAMP - INTERVAL %(sending_interval)s OR
+    (rtir_report_id IS NOT NULL AND "time.source" < LOCALTIMESTAMP - INTERVAL %(sending_interval)s)
+)
+"""
+
+QUERY_MAP = {"base": SELECT_QUERY, "open_report": OPEN_REPORT_QUERY}
+JSON_FIELDS = ["extra."]
+
+
+class SquelcherExpertBot(Bot):
+    configuration_path: str = ""
+    connect_timeout: int = 5
+    database: str = ""
+    user: str = ""
+    password: str = ""
+    host: str = ""
+    port: str = ""
+    sslmode: str = ""
+    autocommit: bool = True
+    table: str = "contacts"
+    sending_time_interval: int = 1
+    overwrite: bool = False
+    query: str = "base"  # base, open_report
+    source_fields: str = "source.ip"
+    filter_ip_only: str = True
+    use_ttl_field: str = "extra.ttl"
+
+    _filters_mapping: dict = None
+
+    def init(self):
+        self.config = load_configuration(self.configuration_path)
+
+        self.logger.debug("Connecting to PostgreSQL.")
+        if psycopg2 is None:
+            raise ValueError("Could not import psycopg2. Please install it.")
+        if netaddr is None:
+            raise ValueError("Could not import netaddr. Please install it.")
+
+        try:
+            self.con = psycopg2.connect(
+                database=self.database,
+                user=self.user,
+                password=self.password,
+                host=self.host,
+                port=self.port,
+                sslmode=self.sslmode,
+                connect_timeout=self.connect_timeout,
+            )
+            self.cur = self.con.cursor()
+            self.con.autocommit = self.autocommit
+
+        except Exception:
+            self.logger.exception("Failed to connect to database.")
+            self.stop()
+        self.logger.info("Connected to PostgreSQL.")
+
+        self.query_tpl = self._build_query()
+        self.convert_config()
+
+    def _build_query(self):
+        # TODO: Build it using sql-safe syntax
+        # TODO: ensure source field is in the harmonization
+        template = QUERY_MAP[self.query]
+        self._filters_mapping = dict()
+        for idx, field in enumerate(self.source_fields.split(",")):
+            self._filters_mapping[field] = f"source_{idx}"
+
+        conditions = []
+        for field, filter_key in self._filters_mapping.items():
+            json_fields = [f for f in JSON_FIELDS if field.startswith(f)]
+            if json_fields:
+                conditions.append(
+                    (
+                        f'"{json_fields[0][:-1]}" ->> \'{field.replace(json_fields[0], "")}\''
+                        f" IS NOT DISTINCT FROM %({filter_key})s AND"
+                    )
+                )
+            else:
+                # IS NOT DISTINCT makes a good job, but doesn't play well with
+                # unique indexes unless probably PostgreSQL 15 and UNIQUE NULLS NOT DISTINCT
+                # index configuration
+                conditions.append(
+                    f"""CASE WHEN %({filter_key})s IS null THEN
+                            "{field}" is null
+                            ELSE "{field}" = %({filter_key})s
+                        END AND
+                    """
+                )
+        return template.format(table=self.table, source_filters="\n".join(conditions))
+
+    def convert_config(self):
+        for rule_index, ruleset in enumerate(self.config):
+            for key, value in ruleset[0].items():
+                if isinstance(value, list):
+                    self.config[rule_index][0][key] = tuple(value)
+                if isinstance(value, dict):
+                    self.config[rule_index][0][key] = tuple(value.items())
+
+    def convert_event(self, event):
+        event_copy = event.to_dict()
+        for key, value in event_copy.items():
+            if isinstance(value, list):
+                event_copy[key] = tuple(value)
+            if isinstance(value, dict):
+                event_copy[key] = tuple(value.items())
+        return event_copy
+
+    def process(self):
+        event = self.receive_message()
+
+        if "notify" in event and not self.overwrite:
+            self.logger.debug(
+                "Notify field present and not allowed to overwrite, skipping."
+            )
+            self.modify_end(event)
+            return
+
+        if self.filter_ip_only:
+            if "source.ip" not in event and "source.fqdn" in event:
+                self.logger.debug(
+                    "Filtering restricted to IPs, setting notify=true for domain event."
+                )
+                event.add("notify", True, overwrite=True)
+                self.modify_end(event)
+                return
+            if "source.asn" not in event:
+                self.logger.debug("Discarding event as it lacks AS number.")
+                event.add("notify", False, overwrite=True)
+                self.modify_end(event)
+                return
+        ttl = None
+
+        if self.use_ttl_field:
+            try:
+                ttl = int(event[self.use_ttl_field])
+            except KeyError:
+                pass
+
+        if ttl is None:
+            for ruleset in self.config:
+                condition = ruleset[0].copy()
+                conditions = []
+                if "source.network" in condition and "source.ip" in event:
+                    conditions.append(
+                        ip_address(event["source.ip"])
+                        in ip_network(condition["source.network"])
+                    )
+                    del condition["source.network"]
+                if "source.iprange" in condition and "source.ip" in event:
+                    conditions.append(
+                        event["source.ip"]
+                        in netaddr.IPRange(*condition["source.iprange"])
+                    )
+                    del condition["source.iprange"]
+                if set(condition.items()).issubset(
+                    self.convert_event(event).items()
+                ) and all(conditions):
+                    ttl = ruleset[1]["ttl"]
+                    break
+
+        self.logger.debug(
+            "Found TTL {} for ({}, {})." "".format(
+                ttl, event.get("source.asn"), event.get("source.ip")
+            )
+        )
+
+        try:
+            if ttl >= 0:
+                source_filters = {
+                    filter_key: event.get(field)
+                    for field, filter_key in self._filters_mapping.items()
+                }
+                self.cur.execute(
+                    self.query_tpl,
+                    {
+                        "ttl": ttl,
+                        "type": event["classification.type"],
+                        "identifier": event["classification.identifier"],
+                        "sending_interval": self.sending_time_interval,
+                        **source_filters,
+                    },
+                )
+                result = self.cur.fetchone()[0]
+            else:  # never notify with ttl -1
+                result = 1
+        except (
+            psycopg2.InterfaceError,
+            psycopg2.InternalError,
+            psycopg2.OperationalError,
+            AttributeError,
+        ):
+            self.logger.exception("Cursor has been closed, connecting again.")
+            self.init()
+        else:
+            if result == 0:
+                notify = True
+            else:
+                notify = False
+
+            event.add("notify", notify, overwrite=True)
+            self.modify_end(event)
+
+    def shutdown(self):
+        try:
+            self.cur.close()
+        except Exception:
+            pass
+        try:
+            self.con.close()
+        except Exception:
+            pass
+
+    def modify_end(self, event):
+        self.send_message(event)
+        self.acknowledge_message()
+
+    @staticmethod
+    def check(parameters):
+        retval = []
+        try:
+            config = load_configuration(parameters["configuration_path"])
+        except ValueError as exc:
+            return [["error", "Could not load configuration: %r." % exc]]
+        for ruleset in config:
+            condition = ruleset[0].copy()
+            if "source.network" in condition:
+                try:
+                    ip_network(condition["source.network"])
+                except ValueError as exc:
+                    retval += [
+                        [
+                            "warning",
+                            "%r is not a valid IP network: %r."
+                            % (condition["source.network"], exc),
+                        ]
+                    ]
+                del condition["source.network"]
+            if "source.iprange" in condition:
+                try:
+                    netaddr.IPRange(*condition["source.iprange"])
+                except ValueError as exc:
+                    retval += [
+                        [
+                            "warning",
+                            "%r is not a valid IP range: %r."
+                            % (condition["source.iprange"], exc),
+                        ]
+                    ]
+                del condition["source.iprange"]
+            try:
+                Event(condition)
+            except Exception as exc:
+                retval += [
+                    ["warning", "Failed to parse conditions as Event: %r." % (exc)]
+                ]
+            try:
+                int(ruleset[1]["ttl"])
+            except ValueError as exc:
+                retval += [
+                    ["error", "%r is not a valid TTL: %r." % (ruleset[1]["ttl"], exc)]
+                ]
+        return retval if retval else None
+
+
+BOT = SquelcherExpertBot

intelmq_extensions/bots/experts/vulnerability_lookup/expert.py

@@ -0,0 +1,136 @@
+import json
+from typing import Optional
+
+from intelmq.lib.bot import ExpertBot
+from intelmq.lib.mixins import CacheMixin
+from intelmq.lib.utils import create_request_session
+
+CACHE_FORMAT = "vuln:{identifier}"
+CACHE_NOT_FOUND = "VULN_NOT_FOUND"
+
+
+class VulnerabilityLookupExpertBot(ExpertBot, CacheMixin):
+    url = "https://vulnerability.circl.lu"
+    vulnerability_field = "classification.identifier"
+    description_length = 500
+    overwrite = False
+
+    filter_classification_type = ["vulnerable-system"]
+
+    redis_cache_ttl = 86400  # 1 day
+
+    def init(self):
+        self.set_request_parameters()
+        self.http_session = create_request_session(self)
+
+    def _get_vulnerability_data(self, vuln_id: Optional[str]) -> Optional[dict]:
+        vuln_id = (vuln_id or "").strip().lower()
+        if not vuln_id:
+            return None
+
+        cache_key = CACHE_FORMAT.format(identifier=vuln_id)
+        cached_data = self.cache_get(cache_key)
+        if cached_data:
+            if cached_data == CACHE_NOT_FOUND:
+                return None
+            return json.loads(cached_data)
+
+        response = self.http_session.get(f"{self.url}/api/vulnerability/{vuln_id}")
+        if response.status_code != 200:
+            response.raise_for_status()
+
+        vuln_raw_data = response.json()
+        if response.status_code == 404 or not vuln_raw_data:
+            self.cache_set(cache_key, CACHE_NOT_FOUND)
+            return None
+
+        epss = None
+        response = self.http_session.get(f"{self.url}/api/epss/{vuln_id}")
+        if response.status_code == 200:
+            epss_data = response.json().get("data", [])
+            if epss_data:
+                epss = epss_data[0].get("epss", None)
+        else:
+            self.logger.info(
+                "Cannot get EPSS score, status code: %d", response.status_code
+            )
+
+        vuln_data = {"url": f"{self.url}/vuln/{vuln_id}"}
+        if epss:
+            vuln_data["epss"] = epss
+
+        description = None
+        cvss3_1 = None
+        cvss3_0 = None
+        cvss4_0 = None
+
+        # Every source queried by Vulnerability Lookup provides different data format
+
+        # CVE records
+        if vuln_raw_data.get("dataType", "") == "CVE_RECORD":
+            cna = vuln_raw_data.get("containers", {}).get("cna")
+            if cna:
+                for item in cna.get("descriptions", []):
+                    if item.get("lang", "") in ["en"]:
+                        description = item.get("value", "")
+                        break
+
+                for item in cna.get("metrics", []):
+                    if "cvssV3_1" in item:
+                        cvss3_1 = item["cvssV3_1"].get("baseScore")
+                    if "cvssV3_0" in item:
+                        cvss3_0 = item["cvssV3_0"].get("baseScore")
+                    if "cvssV4_0" in item:
+                        cvss4_0 = item["cvssV4_0"].get("baseScore")
+
+        # GitHub Security Advisories
+        if vuln_id.startswith("ghsa-"):
+            description = vuln_raw_data.get("details")
+            # TODO: Calculate CVSS
+
+        if description:
+            # Some CSV readers do not understand multi-line texts
+            description = description.replace("\n", " ")
+            vuln_data["description"] = description[: self.description_length]
+            if len(description) > self.description_length:
+                vuln_data["description"] += "..."
+
+        if cvss3_0:
+            vuln_data["cvss3_0"] = cvss3_0
+
+        if cvss3_1:
+            vuln_data["cvss3_1"] = cvss3_1
+
+        if cvss4_0:
+            vuln_data["cvss4_0"] = cvss4_0
+
+        self.cache_set(cache_key, json.dumps(vuln_data))
+        return vuln_data
+
+    def process(self):
+        event = self.receive_message()
+
+        if (
+            not self.filter_classification_type
+            or event.get("classification.type") in self.filter_classification_type
+        ):
+            vuln_id = event.get(self.vulnerability_field)
+            vuln_data = self._get_vulnerability_data(vuln_id) or {}
+
+            if description := vuln_data.get("description"):
+                event.add(
+                    "event_description.text", description, overwrite=self.overwrite
+                )
+
+            if url := vuln_data.get("url"):
+                event.add("event_description.url", url, overwrite=self.overwrite)
+
+            for score_type in ["cvss3_0", "cvss3_1", "cvss4_0", "epss"]:
+                if score := vuln_data.get(score_type):
+                    event.add(f"extra.{score_type}", score, overwrite=self.overwrite)
+
+        self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = VulnerabilityLookupExpertBot

intelmq_extensions/bots/outputs/mattermost/output.py

@@ -0,0 +1,113 @@
+from copy import deepcopy
+
+from intelmq.lib.bot import OutputBot
+from intelmq.lib.utils import create_request_session
+
+
+class MattermostOutputBot(OutputBot):
+    mm_url: str
+    bot_token: str
+    channel_id: str
+
+    message: str = None
+
+    # https://developers.mattermost.com/integrate/reference/message-attachments/
+    fallback: str = None
+    pretext: str = None
+    text: str = None
+
+    title: str = None
+    title_link: str = None
+
+    author_name: str = "IntelMQ"
+    author_icon: str = None
+    author_link: str = None
+
+    color: str = None
+
+    fields: list[dict] = None
+
+    footer: str = None
+
+    # https://developers.mattermost.com/integrate/webhooks/incoming/#parameters
+    card: str = None
+
+    _template_fields = [
+        "fallback",
+        "message",
+        "pretext",
+        "text",
+        "title",
+        "value",
+        "footer",
+    ]
+    _attachment_fields = [
+        "fallback",
+        "pretext",
+        "text",
+        "title",
+        "title_link",
+        "author_name",
+        "author_icon",
+        "author_link",
+        "color",
+        "fields",
+        "footer",
+    ]
+    _is_attachment = False
+
+    def init(self):
+        self.set_request_parameters()
+        self.session = create_request_session(self)
+
+        if not self.message and not self.text:
+            raise ValueError("Either message or text have to be configured")
+
+        if any(getattr(self, f, None) for f in self._attachment_fields):
+            self._is_attachment = True
+
+    def process(self):
+        event = self.receive_message()
+        event.set_default_value()
+
+        request_data = {"channel_id": self.channel_id, "props": {}}
+
+        if self.message:
+            request_data["message"] = self.message.format(ev=event)
+
+        if self._is_attachment:
+            request_data["props"]["attachments"] = [self._prepare_attachment(event)]
+
+        if self.card:
+            request_data["props"]["card"] = self.card.format(ev=event)
+
+        result = self.session.post(
+            f"{self.mm_url}/api/v4/posts",
+            json=request_data,
+            headers={"Authorization": f"Bearer {self.bot_token}"},
+        )
+        result.raise_for_status()
+
+        self.acknowledge_message()
+
+    def _prepare_attachment(self, event) -> dict:
+        attachment = {}
+        for field in self._attachment_fields:
+            data = getattr(self, field, None)
+            if data is None:
+                continue
+            if field == "fields":
+                data: list[dict] = deepcopy(data)
+                for item in data:
+                    if "title" in item:
+                        item["title"] = item["title"].format(ev=event)
+                    if "value" in item:
+                        item["value"] = item["value"].format(ev=event)
+            elif field in self._template_fields:
+                data = data.format(ev=event)
+
+            attachment[field] = data
+        return attachment
+
+
+BOT = MattermostOutputBot

intelmq_extensions/bots/outputs/to_logs/output.py

@@ -0,0 +1,12 @@
+from intelmq.lib.bot import Bot
+
+
+class ToLogOutput(Bot):
+    def process(self):
+        event = self.receive_message()
+        jevent = event.to_json()
+        self.logger.info("Got message %s", jevent)
+        self.acknowledge_message()
+
+
+BOT = ToLogOutput