intelmq-extensions 1.8.1__tar.gz → 1.10.0__tar.gz

{intelmq_extensions-1.8.1 → intelmq_extensions-1.10.0}/PKG-INFO

@@ -1,25 +1,25 @@
 Metadata-Version: 2.4
 Name: intelmq_extensions
-Version: 1.8.1
+Version: 1.10.0
 Summary: Additional bots for IntelMQ
 Author: CERT.at Data & Development Team
 License: AGPLv3
 Project-URL: Repository, https://github.com/certat/intelmq-extensions
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: slixmpp
+Requires-Dist: importlib_metadata; python_version < "3.8"
+Requires-Dist: psycopg2-binary
 Requires-Dist: rt<3.0.0,>=1.0.9
-Requires-Dist: netaddr>=0.7.14
 Requires-Dist: psycopg2-binary>=2.5.5
 Requires-Dist: mergedeep
-Requires-Dist: intelmq
-Requires-Dist: slixmpp
-Requires-Dist: tabulate>=0.7.5
-Requires-Dist: psycopg2-binary
 Requires-Dist: netaddr>=0.7.14
 Requires-Dist: python-termstyle>=0.1.10
-Requires-Dist: importlib_metadata; python_version < "3.8"
+Requires-Dist: tabulate>=0.7.5
+Requires-Dist: netaddr>=0.7.14
+Requires-Dist: intelmq
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: tox>=4; extra == "dev"
@@ -33,14 +33,25 @@ Dynamic: requires-dist
 
 # IntelMQ Extensions
 
-This project collects customized bots used primary by CERT.at.
+[![Running tests](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml/badge.svg)](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
+
+This project collects customized bots and some helper scripts for
+[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
+
+It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
+as well as newer solutions.
 
 ## Usage
 
-Install the package on the machine. Then, it's enough to just declare the bot's module
-pointing to this package, e.g. `intelmq_extensions.bots.collectors.xmpp`
+Install the package on the machine or virtualenv, where you have the IntelMQ, using
+`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
+bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
+`intelmq.bots.experts.certat.vulnerability_lookup.expert`
 
+## Documentation
 
+There is a limited documentation available. Consult bot Python code to see information
+about the usage and available configuration.
 
 ## Running tests
 
@@ -61,3 +72,9 @@ This package comes with test runners configured using `tox`. To use them:
     tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
 
 ```
+
+---
+
+Part of the development was financed by the European Union.
+
+![CEF-Logo](https://github.com/certat/intelmq-extensions/blob/main/docs/cef_logo.png?raw=true)

intelmq_extensions-1.10.0/README.md

@@ -0,0 +1,47 @@
+# IntelMQ Extensions
+
+[![Running tests](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml/badge.svg)](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
+
+This project collects customized bots and some helper scripts for
+[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
+
+It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
+as well as newer solutions.
+
+## Usage
+
+Install the package on the machine or virtualenv, where you have the IntelMQ, using
+`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
+bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
+`intelmq.bots.experts.certat.vulnerability_lookup.expert`
+
+## Documentation
+
+There is a limited documentation available. Consult bot Python code to see information
+about the usage and available configuration.
+
+## Running tests
+
+This package comes with test runners configured using `tox`. To use them:
+
+```bash
+
+    tox -elint  # run code style checks
+    tox -epy310  # run simple unittests against Python 3.10
+
+    # For running all unittests, including connecting to external services / database
+    # use on of the following:
+    tox -efull  # assuming you run redis, postgres etc. on your own
+    tox -efull-with-docker  # this will use docker compose to provision services for tests;
+                            # please note it uses default ports
+
+    # You can pass arguments to the pytest, e.g. to run a specific test:
+    tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
+
+```
+
+---
+
+Part of the development was financed by the European Union.
+
+![CEF-Logo](https://github.com/certat/intelmq-extensions/blob/main/docs/cef_logo.png?raw=true)

intelmq_extensions-1.10.0/intelmq_extensions/bots/collectors/modat/collector.py

@@ -0,0 +1,87 @@
+"""Collector of data from Modat API
+
+SPDX-FileCopyrightText: 2025 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+https://api.magnify.modat.io/docs
+
+Parameters:
+
+api_key
+# TODO: multiple queries in one bot?
+query
+type [service|host]
+
+# with defaults
+url = "https://api.magnify.modat.io/"
+page_size = 10
+max_results = 100
+
+# Standard Collector parameters
+name: Optional[str] = None
+accuracy: int = 100
+code: Optional[str] = None
+provider: Optional[str] = None
+documentation: Optional[str] = None
+"""
+
+import json
+from urllib.parse import urljoin
+
+from intelmq.lib.bot import CollectorBot
+from intelmq.lib.utils import create_request_session
+
+
+class ModatCollectorBot(CollectorBot):
+    url: str = "https://api.magnify.modat.io/"
+    page_size = 10
+    max_results = 100
+
+    api_key: str
+    query: str
+    type: str  # service|host
+
+    def init(self):
+        self.set_request_parameters()
+        self.session = create_request_session(self)
+        self.session.headers = {"Authorization": f"Bearer {self.api_key}"}
+
+    def process(self):
+        page = 1
+        collected_results = 0
+        total_available = self.page_size
+
+        if self.type == "host":
+            url = urljoin(self.url, "/host/search/v1")
+        else:
+            url = urljoin(self.url, "/service/search/v1")
+
+        while (
+            total_available > collected_results and collected_results < self.max_results
+        ):
+            result = self.session.post(
+                url,
+                json={"page": page, "page_size": self.page_size, "query": self.query},
+            )
+            if result.status_code != 200:
+                self.logger.error("Modat responded with error %d.", result.status_code)
+                self.logger.debug("Modat response: %s.", result.text)
+                raise RuntimeError(
+                    "Cannot retrieve data from Modat.", detail=result.text
+                )
+
+            data = result.json()
+            report = self.new_report()
+            report.add("raw", json.dumps(data["page"]))
+            self.send_message(report)
+
+            page += 1
+            collected_results += len(data["page"])
+            total_available = data["total_records"]
+
+    # @staticmethod
+    # def check(parameters: dict) -> list[list[str]] or None:
+    #     pass
+
+
+BOT = ModatCollectorBot

intelmq_extensions-1.10.0/intelmq_extensions/bots/experts/replace_in_dict/expert.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+ReplaceInDict allow replacing pattern in any text field in a dict field(s)
+"""
+
+from intelmq.lib.bot import ExpertBot
+from intelmq.lib.exceptions import ConfigurationError, KeyNotExists
+
+
+class ReplaceInDictExpertBot(ExpertBot):
+    old_value: str = None
+    new_value: str = None
+    fields: str = None  # actually str | list on newer Python
+
+    def init(self):
+        if isinstance(self.fields, str):
+            self.fields = self.fields.split(",")
+        for field in self.fields:
+            definition = self.harmonization["event"][field]
+            if definition["type"] != "JSONDict":
+                raise ConfigurationError("Field is not a JSONDict", field)
+
+    def process(self):
+        event = self.receive_message()
+
+        for field in self.fields:
+            for name, value in event.finditems(f"{field}."):
+                if isinstance(value, str):
+                    try:
+                        event.change(
+                            name, value.replace(self.old_value, self.new_value)
+                        )
+                    except KeyNotExists:
+                        # Safeguard for an edge case if we would get default value
+                        # of an non-existing field
+                        pass
+
+        self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = ReplaceInDictExpertBot

intelmq_extensions-1.10.0/intelmq_extensions/bots/parsers/modat/parser.py

@@ -0,0 +1,92 @@
+"""Parsing Modat search requests
+
+Currently supporting only host search results
+
+SPDX-FileCopyrightText: 2025 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+from datetime import datetime, timezone
+
+import intelmq.lib.message as message
+from intelmq.lib import utils
+from intelmq.lib.bot import ParserBot
+
+
+class ModatParserBot(ParserBot):
+    def parse(self, report: message.Report):
+        raw_report = utils.base64_decode(report.get("raw"))
+        report_data = json.loads(raw_report)
+
+        for entry in report_data:
+            self._current_line = json.dumps(entry)
+            yield entry
+
+    def parse_line(self, line: dict, report: message.Report):
+        event = self.new_event(report)
+        event.add("raw", self._current_line)
+
+        event.add("source.ip", line.get("ip"), raise_failure=False)
+        event.add(
+            "source.geolocation.cc",
+            line.get("geo", {}).get("country_iso_code"),
+            raise_failure=False,
+        )
+        event.add(
+            "source.geolocation.country",
+            line.get("geo", {}).get("country_name"),
+            raise_failure=False,
+        )
+        event.add(
+            "source.geolocation.city",
+            line.get("geo", {}).get("city_name"),
+            raise_failure=False,
+        )
+
+        event.add("source.asn", line.get("asn", {}).get("number"), raise_failure=False)
+        event.add("source.as_name", line.get("asn", {}).get("org"), raise_failure=False)
+        fqdns = line.get("fqdns", [])
+        if fqdns:
+            event.add("source.fqdn", fqdns[0], raise_failure=False)
+        if len(fqdns) > 1:
+            event.add("extra.fqdns", ";".join(fqdns), raise_failure=False)
+        event.add("extra.tag", ";".join(line.get("tags", [])), raise_failure=False)
+
+        cves = ";".join(cve.get("id", "").lower() for cve in line.get("cves", []))
+        if cves:
+            event.add("product.vulnerabilities", cves, raise_failure=False)
+
+        services = line.get("services", [])
+        last_scanned = datetime.now(tz=timezone.utc)
+        if services:
+            last_scanned = max(s["scanned_at"] for s in services)
+            event.add("extra.services", services, raise_failure=False)
+
+            for service in services:
+                # This is what we were looking for
+                if service["is_match"]:
+                    event.add(
+                        "protocol.application",
+                        service.get("protocol"),
+                        ignore=(None, "unknown"),
+                        raise_failure=False,
+                    )
+                    event.add(
+                        "protocol.transport",
+                        service.get("transport"),
+                        raise_failure=False,
+                    )
+                    event.add("source.port", service.get("port"), raise_failure=False)
+
+                    last_scanned = service.get("scanned_at") or last_scanned
+
+                    # TODO: emit more events?
+                    break
+
+        event.add("time.source", last_scanned, raise_failure=False)
+
+        return event
+
+
+BOT = ModatParserBot

{intelmq_extensions-1.8.1 → intelmq_extensions-1.10.0}/intelmq_extensions/etc/harmonization.conf

@@ -1,5 +1,33 @@
 {
     "event": {
+        "destination_visible": {
+            "description": "If destination fields are visible. CERT.at-specific",
+            "type": "Boolean"
+        },
+        "notify": {
+            "description": "If mail will be sent out to affected or responsible contact. CERT.at-specific",
+            "type": "Boolean"
+        },
+        "rtir_incident_id": {
+            "description": "Request Tracker Incident Response incident id. CERT.at-specific",
+            "type": "Integer"
+        },
+        "rtir_investigation_id": {
+            "description": "Request Tracker Incident Response investigation id. CERT.at-specific",
+            "type": "Integer"
+        },
+        "rtir_report_id": {
+            "description": "Request Tracker Incident Response incident report id. CERT.at-specific",
+            "type": "Integer"
+        },
+        "sent_at": {
+            "description": "Time when the report has been sent to the responsible recipient. CERT.at-specific",
+            "type": "DateTime"
+        },
+        "shareable_extra_info": {
+            "description": "Fields from extra which can be shared. CERT.at-specific",
+            "type": "JSON"
+        },
         "classification.identifier": {
             "description": "The lowercase identifier defines the actual software or service (e.g. ``heartbleed`` or ``ntp_version``) or standardized malware name (e.g. ``zeus``). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.",
             "type": "String"
@@ -7,7 +35,7 @@
         "classification.taxonomy": {
             "description": "We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check `ENISA taxonomies <http://www.enisa.europa.eu/activities/cert/support/incident-management/browsable/incident-handling-process/incident-taxonomy/existing-taxonomies>`_.",
             "length": 100,
-            "type": "LowercaseString"
+            "type": "ClassificationTaxonomy"
         },
         "classification.type": {
             "description": "The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid *type explosion*, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.",
@@ -17,6 +45,10 @@
             "description": "Free text commentary about the abuse event inserted by an analyst.",
             "type": "String"
         },
+        "constituency": {
+            "description": "Internal identifier for multi-constituency setup",
+            "type": "String"
+        },
         "destination.abuse_contact": {
             "description": "Abuse contact for destination address. A comma separated list.",
             "type": "LowercaseString"
@@ -81,11 +113,11 @@
             "type": "IPAddress"
         },
         "destination.local_hostname": {
-            "description": "Some sources report a internal hostname within a NAT related to the name configured for a compromized system",
+            "description": "Some sources report an internal hostname within a NAT related to the name configured for a compromised system",
             "type": "String"
         },
         "destination.local_ip": {
-            "description": "Some sources report a internal (NATed) IP address related a compromized system. N.B. RFC1918 IPs are OK here.",
+            "description": "Some sources report an internal (NATed) IP address related a compromised system. N.B. RFC1918 IPs are OK here.",
             "type": "IPAddress"
         },
         "destination.network": {
@@ -118,10 +150,6 @@
             "description": "The path portion of an HTTP or related network request.",
             "type": "String"
         },
-        "destination_visible": {
-            "description": "If destination fields are visible.",
-            "type": "Boolean"
-        },
         "event_description.target": {
             "description": "Some sources denominate the target (organization) of a an attack.",
             "type": "String"
@@ -144,6 +172,10 @@
             "description": "All anecdotal information, which cannot be parsed into the data harmonization elements. E.g. os.name, os.version, etc.  **Note**: this is only intended for mapping any fields which can not map naturally into the data harmonization. It is not intended for extending the data harmonization with your own fields.",
             "type": "JSONDict"
         },
+        "extra.min_amplification": {
+            "description": "Temporal definition to make sieve work properly",
+            "type": "Float"
+        },
         "feed.accuracy": {
             "description": "A float between 0 and 100 that represents how accurate the data in the feed is",
             "type": "Accuracy"
@@ -209,14 +241,30 @@
             "regex": "^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[0-9a-z]{12}$",
             "type": "LowercaseString"
         },
-        "notify": {
-            "description": "If mail will be sent out to affected or responsible contact.",
-            "type": "Boolean"
-        },
         "output": {
             "description": "Event data converted into foreign format, intended to be exported by output plugin.",
             "type": "JSON"
         },
+        "product.full_name": {
+            "description": "A human readable product name. If a machine-readable format isn't available, this field should be used. It can directly use the version identification strings presented by the product. If not given, a good enough value can usually be constructed by concatenating product.product and product.version, or by consulting external sources such as the CPE Product Dictionary. Example: openssh_/8.9",
+            "type": "String"
+        },
+        "product.name": {
+            "description": "Product name, recommended being as the product in the CPE format. Example: openssh",
+            "type": "LowercaseString"
+        },
+        "product.vendor": {
+            "description": "Vendor name, recommended being as vendor in the CPE format. Example: openbsd",
+            "type": "LowercaseString"
+        },
+        "product.version": {
+            "description": "Product version, recommended being as version in the CPE format. Example: 8.9",
+            "type": "LowercaseString"
+        },
+        "product.vulnerabilities": {
+            "description": "List of vulnerability IDs, separated by semicolons. It's recommended to use a CVE ID where available, and other easily retrievable IDs in other cases, e.g. Github Advisory Database ID. Each vulnerability should only be listed once, and multiple values should be used if there are several different vulnerabilities. However, it's not necessary for a source to list all possible vulnerabilities for a given piece of software. Example: cve-2023-38408;cve-2023-28531;cve-2008-3844;cve-2007-2768",
+            "type": "LowercaseString"
+        },
         "protocol.application": {
             "description": "e.g. vnc, ssh, sip, irc, http or smtp.",
             "length": 100,
@@ -233,29 +281,19 @@
             "description": "The original line of the event from encoded in base64.",
             "type": "Base64"
         },
-        "rtir_incident_id": {
-            "description": "Request Tracker Incident Response incident id.",
-            "type": "Integer"
-        },
-        "rtir_investigation_id": {
-            "description": "Request Tracker Incident Response investigation id.",
-            "type": "Integer"
-        },
-        "rtir_report_id": {
-            "description": "Request Tracker Incident Response incident report id.",
+        "rtir_id": {
+            "description": "Request Tracker Incident Response ticket id.",
             "type": "Integer"
         },
         "screenshot_url": {
             "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.",
             "type": "URL"
         },
-        "sent_at": {
-            "description": "Time when the report has been sent to the responsible recipient.",
-            "type": "DateTime"
-        },
-        "shareable_extra_info": {
-            "description": "Fields from extra which can be shared.",
-            "type": "JSON"
+        "severity": {
+            "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source. Allowed values: critical (highly critical vulnerabilities being actively exploited and pose a very high likelihood of compromise. For example, RCEs, sensitive data access), high (end of life systems, accessible internal systems that should not be exposed, risk of data leaks, malware drone and sinkhole events), medium (DDoS-amplifiers, unencrypted services requiring login, vulnerabilities requiring MITM to exploit, attacks need prior knowledge), low (deviation from best practice, little to no practical way to exploit, but setup is not ideal), info (informational only, no known risk), undefined (unknown or undetermined)",
+            "length": 10,
+            "regex": "^(critical|high|medium|low|info|undefined)$",
+            "type": "LowercaseString"
         },
         "source.abuse_contact": {
             "description": "Abuse contact for source address. A comma separated list.",
@@ -389,6 +427,10 @@
         }
     },
     "report": {
+        "rtir_report_id": {
+            "description": "Request Tracker Incident Response incident report id. CERT.at-specific",
+            "type": "Integer"
+        },
         "extra": {
             "description": "All anecdotal information of the report, which cannot be parsed into the data harmonization elements. E.g. subject of mails, etc. This is data is not automatically propagated to the events.",
             "type": "JSONDict"
@@ -422,8 +464,8 @@
             "description": "The original raw and unparsed data encoded in base64.",
             "type": "Base64"
         },
-        "rtir_report_id": {
-            "description": "Request Tracker Incident Response incident report id.",
+        "rtir_id": {
+            "description": "Request Tracker Incident Response ticket id.",
             "type": "Integer"
         },
         "time.observation": {
@@ -431,4 +473,4 @@
             "type": "DateTime"
         }
     }
-}
+}

intelmq_extensions-1.10.0/intelmq_extensions/tests/bots/collectors/modat/test_collector.py

@@ -0,0 +1,112 @@
+"""Tests for Modat Collector
+
+SPDX-FileCopyrightText: 2025 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import json
+import unittest
+from unittest import mock
+
+import intelmq.lib.message as message
+import requests
+from requests_mock import MockerCore
+
+from intelmq_extensions.bots.collectors.modat.collector import ModatCollectorBot
+
+from ....base import BotTestCase
+
+
+class TestModatCollectorBot(BotTestCase, unittest.TestCase):
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ModatCollectorBot
+        cls.sysconfig = {
+            "query": "test-query",
+            "type": "host",
+            "api_key": "secure-api-key",
+            "code": "feed-code",
+        }
+
+    def mock_http_session(self):
+        session = requests.Session()
+        session_mock = mock.patch(
+            "intelmq_extensions.bots.collectors.modat.collector.create_request_session",
+            return_value=session,
+        )
+        session_mock.start()
+        self.addCleanup(session_mock.stop)
+
+        self.requests = MockerCore(session=session)
+        self.requests.start()
+        self.addCleanup(self.requests.stop)
+
+    def setUp(self):
+        super().setUp()
+        self.mock_http_session()
+
+    def mock_request(
+        self,
+        path: str,
+        expected_query: str = "test-query",
+        expected_page: int = 1,
+        expected_page_size: int = 10,
+        **kwargs,
+    ):
+        def check_request(request):
+            if request.headers.get("Authorization", "") != "Bearer secure-api-key":
+                return False
+            if request.json()["query"] != expected_query:
+                return False
+            if request.json()["page"] != expected_page:
+                return False
+            if request.json()["page_size"] != expected_page_size:
+                return False
+            return True
+
+        self.requests.post(
+            f"https://api.magnify.modat.io/{path}",
+            status_code=200,
+            additional_matcher=check_request,
+            **kwargs,
+        )
+
+    def _create_report_dict(self, data, **kwargs):
+        report = message.Report(kwargs, harmonization=self.harmonization)
+        report.add("feed.name", "Test Bot")
+        report.add("feed.accuracy", 100.0)
+        report.add("feed.code", "feed-code")
+        report.add("raw", json.dumps(data))
+        return report.to_dict(with_type=True)
+
+    def test_query_hosts(self):
+        self.mock_request(
+            "host/search/v1",
+            json={
+                "page_nr": 1,
+                "total_pages": 1,
+                "total_records": 4,
+                "page": [{"ip": "ip1"}, {"ip": "ip2"}],
+            },
+        )
+        self.mock_request(
+            "host/search/v1",
+            json={
+                "page_nr": 2,
+                "total_pages": 1,
+                "total_records": 4,
+                "page": [{"ip": "ip3"}, {"ip": "ip4"}],
+            },
+            expected_page=2,
+        )
+
+        self.run_bot()
+
+        self.assertOutputQueueLen(2)
+        self.assertMessageEqual(
+            0, self._create_report_dict([{"ip": "ip1"}, {"ip": "ip2"}])
+        )
+        self.assertMessageEqual(
+            1, self._create_report_dict([{"ip": "ip3"}, {"ip": "ip4"}])
+        )
+        self.assertEqual(2, self.requests.call_count)