intelmq-extensions 1.8.1__tar.gz → 1.9.0__tar.gz

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/PKG-INFO

@@ -1,25 +1,25 @@
 Metadata-Version: 2.4
 Name: intelmq_extensions
-Version: 1.8.1
+Version: 1.9.0
 Summary: Additional bots for IntelMQ
 Author: CERT.at Data & Development Team
 License: AGPLv3
 Project-URL: Repository, https://github.com/certat/intelmq-extensions
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: rt<3.0.0,>=1.0.9
-Requires-Dist: netaddr>=0.7.14
-Requires-Dist: psycopg2-binary>=2.5.5
-Requires-Dist: mergedeep
-Requires-Dist: intelmq
 Requires-Dist: slixmpp
-Requires-Dist: tabulate>=0.7.5
-Requires-Dist: psycopg2-binary
-Requires-Dist: netaddr>=0.7.14
 Requires-Dist: python-termstyle>=0.1.10
+Requires-Dist: mergedeep
 Requires-Dist: importlib_metadata; python_version < "3.8"
+Requires-Dist: psycopg2-binary>=2.5.5
+Requires-Dist: psycopg2-binary
+Requires-Dist: netaddr>=0.7.14
+Requires-Dist: netaddr>=0.7.14
+Requires-Dist: intelmq
+Requires-Dist: tabulate>=0.7.5
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: tox>=4; extra == "dev"
@@ -33,14 +33,25 @@ Dynamic: requires-dist
 
 # IntelMQ Extensions
 
-This project collects customized bots used primary by CERT.at.
+[![Running tests](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml/badge.svg)](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
+
+This project collects customized bots and some helper scripts for
+[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
+
+It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
+as well as newer solutions.
 
 ## Usage
 
-Install the package on the machine. Then, it's enough to just declare the bot's module
-pointing to this package, e.g. `intelmq_extensions.bots.collectors.xmpp`
+Install the package on the machine or virtualenv, where you have the IntelMQ, using
+`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
+bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
+`intelmq.bots.experts.certat.vulnerability_lookup.expert`
 
+## Documentation
 
+There is a limited documentation available. Consult bot Python code to see information
+about the usage and available configuration.
 
 ## Running tests
 
@@ -61,3 +72,9 @@ This package comes with test runners configured using `tox`. To use them:
     tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
 
 ```
+
+---
+
+Part of the development was financed by the European Union.
+
+![CEF-Logo](https://github.com/certat/intelmq-extensions/blob/main/docs/cef_logo.png?raw=true)

intelmq_extensions-1.9.0/README.md

@@ -0,0 +1,47 @@
+# IntelMQ Extensions
+
+[![Running tests](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml/badge.svg)](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
+
+This project collects customized bots and some helper scripts for
+[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
+
+It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
+as well as newer solutions.
+
+## Usage
+
+Install the package on the machine or virtualenv, where you have the IntelMQ, using
+`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
+bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
+`intelmq.bots.experts.certat.vulnerability_lookup.expert`
+
+## Documentation
+
+There is a limited documentation available. Consult bot Python code to see information
+about the usage and available configuration.
+
+## Running tests
+
+This package comes with test runners configured using `tox`. To use them:
+
+```bash
+
+    tox -elint  # run code style checks
+    tox -epy310  # run simple unittests against Python 3.10
+
+    # For running all unittests, including connecting to external services / database
+    # use on of the following:
+    tox -efull  # assuming you run redis, postgres etc. on your own
+    tox -efull-with-docker  # this will use docker compose to provision services for tests;
+                            # please note it uses default ports
+
+    # You can pass arguments to the pytest, e.g. to run a specific test:
+    tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
+
+```
+
+---
+
+Part of the development was financed by the European Union.
+
+![CEF-Logo](https://github.com/certat/intelmq-extensions/blob/main/docs/cef_logo.png?raw=true)

intelmq_extensions-1.9.0/intelmq_extensions/bots/experts/replace_in_dict/expert.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+ReplaceInDict allow replacing pattern in any text field in a dict field(s)
+"""
+
+from intelmq.lib.bot import ExpertBot
+from intelmq.lib.exceptions import ConfigurationError, KeyNotExists
+
+
+class ReplaceInDictExpertBot(ExpertBot):
+    old_value: str = None
+    new_value: str = None
+    fields: str = None  # actually str | list on newer Python
+
+    def init(self):
+        if isinstance(self.fields, str):
+            self.fields = self.fields.split(",")
+        for field in self.fields:
+            definition = self.harmonization["event"][field]
+            if definition["type"] != "JSONDict":
+                raise ConfigurationError("Field is not a JSONDict", field)
+
+    def process(self):
+        event = self.receive_message()
+
+        for field in self.fields:
+            for name, value in event.finditems(f"{field}."):
+                if isinstance(value, str):
+                    try:
+                        event.change(
+                            name, value.replace(self.old_value, self.new_value)
+                        )
+                    except KeyNotExists:
+                        # Safeguard for an edge case if we would get default value
+                        # of an non-existing field
+                        pass
+
+        self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = ReplaceInDictExpertBot

intelmq_extensions-1.9.0/intelmq_extensions/tests/bots/experts/replace_in_dict/test_expert.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+"""
+Testing ReplaceInDictExpertBot.
+"""
+
+import copy
+import unittest
+
+from intelmq.lib.exceptions import ConfigurationError
+
+from intelmq_extensions.bots.experts.replace_in_dict.expert import (
+    ReplaceInDictExpertBot,
+)
+
+from ....base import BotTestCase
+
+
+class TestReplaceInDictExpertBot(BotTestCase, unittest.TestCase):
+    """
+    A TestCase for ReplaceInDictExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ReplaceInDictExpertBot
+        cls.sysconfig = {
+            "old_value": "\\u0000",
+            "new_value": "[nullbyte]",
+            "fields": "extra",
+        }
+        cls.default_input_message = {"__type": "Event"}
+
+    def test_event_no_changes(self):
+        message = {
+            "__type": "Event",
+            "time.observation": "2015-01-01T00:00:00+00:00",
+            "extra.payload": "foo",
+            "extra.name": "bar",
+            "extra.firmwarerev": 1,
+        }
+        self.input_message = copy.deepcopy(message)
+        self.run_bot()
+        self.assertMessageEqual(0, message)
+
+    def test_event_no_extra(self):
+        message = {
+            "__type": "Event",
+            "time.observation": "2015-01-01T00:00:00+00:00",
+            "feed.code": "foo",
+        }
+        self.input_message = copy.deepcopy(message)
+        self.run_bot()
+        self.assertMessageEqual(0, message)
+
+    def test_event_changes_one_dict(self):
+        message = {
+            "__type": "Event",
+            "time.observation": "2015-01-01T00:00:00+00:00",
+            "extra.payload": "foo\\u0000bar\\u0000",
+            "extra.name": "bar ok \\u0001 and not ok \\\\u0000",
+            "extra.firmwarerev": 1,
+            "feed.code": "foo",
+        }
+        self.input_message = copy.deepcopy(message)
+        self.run_bot()
+
+        message["extra.payload"] = "foo[nullbyte]bar[nullbyte]"
+        message["extra.name"] = "bar ok \\u0001 and not ok \\[nullbyte]"
+        self.assertMessageEqual(0, message)
+
+    def test_event_multiple_dict_fail_if_not_jsondict(self):
+        with self.assertRaises(ConfigurationError):
+            self.run_bot(
+                parameters={
+                    "fields": "extra,output",
+                }
+            )
+
+    def test_event_other_fields_not_modified(self):
+        message = {
+            "__type": "Event",
+            "time.observation": "2015-01-01T00:00:00+00:00",
+            "feed.code": "foo\\u0000",
+        }
+        self.input_message = copy.deepcopy(message)
+        self.run_bot()
+
+        self.assertMessageEqual(0, message)
+
+
+if __name__ == "__main__":
+    unittest.main()

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/intelmq_extensions.egg-info/PKG-INFO

@@ -1,25 +1,25 @@
 Metadata-Version: 2.4
 Name: intelmq_extensions
-Version: 1.8.1
+Version: 1.9.0
 Summary: Additional bots for IntelMQ
 Author: CERT.at Data & Development Team
 License: AGPLv3
 Project-URL: Repository, https://github.com/certat/intelmq-extensions
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: rt<3.0.0,>=1.0.9
-Requires-Dist: netaddr>=0.7.14
-Requires-Dist: psycopg2-binary>=2.5.5
-Requires-Dist: mergedeep
-Requires-Dist: intelmq
 Requires-Dist: slixmpp
-Requires-Dist: tabulate>=0.7.5
-Requires-Dist: psycopg2-binary
-Requires-Dist: netaddr>=0.7.14
 Requires-Dist: python-termstyle>=0.1.10
+Requires-Dist: mergedeep
 Requires-Dist: importlib_metadata; python_version < "3.8"
+Requires-Dist: psycopg2-binary>=2.5.5
+Requires-Dist: psycopg2-binary
+Requires-Dist: netaddr>=0.7.14
+Requires-Dist: netaddr>=0.7.14
+Requires-Dist: intelmq
+Requires-Dist: tabulate>=0.7.5
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: tox>=4; extra == "dev"
@@ -33,14 +33,25 @@ Dynamic: requires-dist
 
 # IntelMQ Extensions
 
-This project collects customized bots used primary by CERT.at.
+[![Running tests](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml/badge.svg)](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
+
+This project collects customized bots and some helper scripts for
+[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
+
+It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
+as well as newer solutions.
 
 ## Usage
 
-Install the package on the machine. Then, it's enough to just declare the bot's module
-pointing to this package, e.g. `intelmq_extensions.bots.collectors.xmpp`
+Install the package on the machine or virtualenv, where you have the IntelMQ, using
+`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
+bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
+`intelmq.bots.experts.certat.vulnerability_lookup.expert`
 
+## Documentation
 
+There is a limited documentation available. Consult bot Python code to see information
+about the usage and available configuration.
 
 ## Running tests
 
@@ -61,3 +72,9 @@ This package comes with test runners configured using `tox`. To use them:
     tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
 
 ```
+
+---
+
+Part of the development was financed by the European Union.
+
+![CEF-Logo](https://github.com/certat/intelmq-extensions/blob/main/docs/cef_logo.png?raw=true)

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/intelmq_extensions.egg-info/SOURCES.txt

@@ -27,6 +27,8 @@ intelmq_extensions/bots/experts/event_group_splitter/__init__.py
 intelmq_extensions/bots/experts/event_group_splitter/expert.py
 intelmq_extensions/bots/experts/event_splitter/__init__.py
 intelmq_extensions/bots/experts/event_splitter/expert.py
+intelmq_extensions/bots/experts/replace_in_dict/__init__.py
+intelmq_extensions/bots/experts/replace_in_dict/expert.py
 intelmq_extensions/bots/experts/squelcher/__init__.py
 intelmq_extensions/bots/experts/squelcher/expert.py
 intelmq_extensions/bots/experts/vulnerability_lookup/__init__.py
@@ -79,6 +81,8 @@ intelmq_extensions/tests/bots/experts/event_group_splitter/__init__.py
 intelmq_extensions/tests/bots/experts/event_group_splitter/test_expert.py
 intelmq_extensions/tests/bots/experts/event_splitter/__init__.py
 intelmq_extensions/tests/bots/experts/event_splitter/test_expert.py
+intelmq_extensions/tests/bots/experts/replace_in_dict/__init__.py
+intelmq_extensions/tests/bots/experts/replace_in_dict/test_expert.py
 intelmq_extensions/tests/bots/experts/squelcher/__init__.py
 intelmq_extensions/tests/bots/experts/squelcher/test_expert.py
 intelmq_extensions/tests/bots/experts/vulnerability_lookup/__init__.py

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/intelmq_extensions.egg-info/entry_points.txt

@@ -6,6 +6,7 @@ intelmq.bots.experts.certat.certat_contact_intern.expert = intelmq_extensions.bo
 intelmq.bots.experts.certat.copy_extra.expert = intelmq_extensions.bots.experts.copy_extra.expert:BOT.run
 intelmq.bots.experts.certat.event_group_splitter.expert = intelmq_extensions.bots.experts.event_group_splitter.expert:BOT.run
 intelmq.bots.experts.certat.event_splitter.expert = intelmq_extensions.bots.experts.event_splitter.expert:BOT.run
+intelmq.bots.experts.certat.replace_in_dict.expert = intelmq_extensions.bots.experts.replace_in_dict.expert:BOT.run
 intelmq.bots.experts.certat.squelcher.expert = intelmq_extensions.bots.experts.squelcher.expert:BOT.run
 intelmq.bots.experts.certat.vulnerability_lookup.expert = intelmq_extensions.bots.experts.vulnerability_lookup.expert:BOT.run
 intelmq.bots.outputs.certat.mattermost.output = intelmq_extensions.bots.outputs.mattermost.output:BOT.run
@@ -21,6 +22,7 @@ intelmq_extensions.bots.experts.certat_contact_intern.expert = intelmq_extension
 intelmq_extensions.bots.experts.copy_extra.expert = intelmq_extensions.bots.experts.copy_extra.expert:BOT.run
 intelmq_extensions.bots.experts.event_group_splitter.expert = intelmq_extensions.bots.experts.event_group_splitter.expert:BOT.run
 intelmq_extensions.bots.experts.event_splitter.expert = intelmq_extensions.bots.experts.event_splitter.expert:BOT.run
+intelmq_extensions.bots.experts.replace_in_dict.expert = intelmq_extensions.bots.experts.replace_in_dict.expert:BOT.run
 intelmq_extensions.bots.experts.squelcher.expert = intelmq_extensions.bots.experts.squelcher.expert:BOT.run
 intelmq_extensions.bots.experts.vulnerability_lookup.expert = intelmq_extensions.bots.experts.vulnerability_lookup.expert:BOT.run
 intelmq_extensions.bots.outputs.mattermost.output = intelmq_extensions.bots.outputs.mattermost.output:BOT.run

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "intelmq_extensions"
-version = "1.8.1"
+version = "1.9.0"
 description = "Additional bots for IntelMQ"
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 dynamic = ["entry-points", "dependencies", "scripts"]
 
 classifiers = ["Programming Language :: Python :: 3"]

intelmq_extensions-1.8.1/README.md

@@ -1,30 +0,0 @@
-# IntelMQ Extensions
-
-This project collects customized bots used primary by CERT.at.
-
-## Usage
-
-Install the package on the machine. Then, it's enough to just declare the bot's module
-pointing to this package, e.g. `intelmq_extensions.bots.collectors.xmpp`
-
-
-
-## Running tests
-
-This package comes with test runners configured using `tox`. To use them:
-
-```bash
-
-    tox -elint  # run code style checks
-    tox -epy310  # run simple unittests against Python 3.10
-
-    # For running all unittests, including connecting to external services / database
-    # use on of the following:
-    tox -efull  # assuming you run redis, postgres etc. on your own
-    tox -efull-with-docker  # this will use docker compose to provision services for tests;
-                            # please note it uses default ports
-
-    # You can pass arguments to the pytest, e.g. to run a specific test:
-    tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
-
-```

{intelmq_extensions-1.8.1 → intelmq_extensions-1.9.0}/intelmq_extensions.egg-info/requires.txt

@@ -1,13 +1,13 @@
 rt<3.0.0,>=1.0.9
-netaddr>=0.7.14
-psycopg2-binary>=2.5.5
-mergedeep
-intelmq
 slixmpp
-tabulate>=0.7.5
+python-termstyle>=0.1.10
+mergedeep
+psycopg2-binary>=2.5.5
 psycopg2-binary
 netaddr>=0.7.14
-python-termstyle>=0.1.10
+netaddr>=0.7.14
+intelmq
+tabulate>=0.7.5
 
 [:python_version < "3.8"]
 importlib_metadata