intelmq-extensions 1.8.1__py3-none-any.whl → 1.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intelmq_extensions/bots/experts/replace_in_dict/__init__.py +0 -0
- intelmq_extensions/bots/experts/replace_in_dict/expert.py +42 -0
- intelmq_extensions/tests/bots/experts/replace_in_dict/__init__.py +0 -0
- intelmq_extensions/tests/bots/experts/replace_in_dict/test_expert.py +92 -0
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/METADATA +28 -11
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/RECORD +10 -6
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/WHEEL +1 -1
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/entry_points.txt +2 -0
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/licenses/LICENSE +0 -0
- {intelmq_extensions-1.8.1.dist-info → intelmq_extensions-1.9.0.dist-info}/top_level.txt +0 -0
|
File without changes
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
ReplaceInDict allow replacing pattern in any text field in a dict field(s)
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from intelmq.lib.bot import ExpertBot
|
|
7
|
+
from intelmq.lib.exceptions import ConfigurationError, KeyNotExists
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ReplaceInDictExpertBot(ExpertBot):
|
|
11
|
+
old_value: str = None
|
|
12
|
+
new_value: str = None
|
|
13
|
+
fields: str = None # actually str | list on newer Python
|
|
14
|
+
|
|
15
|
+
def init(self):
|
|
16
|
+
if isinstance(self.fields, str):
|
|
17
|
+
self.fields = self.fields.split(",")
|
|
18
|
+
for field in self.fields:
|
|
19
|
+
definition = self.harmonization["event"][field]
|
|
20
|
+
if definition["type"] != "JSONDict":
|
|
21
|
+
raise ConfigurationError("Field is not a JSONDict", field)
|
|
22
|
+
|
|
23
|
+
def process(self):
|
|
24
|
+
event = self.receive_message()
|
|
25
|
+
|
|
26
|
+
for field in self.fields:
|
|
27
|
+
for name, value in event.finditems(f"{field}."):
|
|
28
|
+
if isinstance(value, str):
|
|
29
|
+
try:
|
|
30
|
+
event.change(
|
|
31
|
+
name, value.replace(self.old_value, self.new_value)
|
|
32
|
+
)
|
|
33
|
+
except KeyNotExists:
|
|
34
|
+
# Safeguard for an edge case if we would get default value
|
|
35
|
+
# of an non-existing field
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
self.send_message(event)
|
|
39
|
+
self.acknowledge_message()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
BOT = ReplaceInDictExpertBot
|
|
File without changes
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Testing ReplaceInDictExpertBot.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import copy
|
|
7
|
+
import unittest
|
|
8
|
+
|
|
9
|
+
from intelmq.lib.exceptions import ConfigurationError
|
|
10
|
+
|
|
11
|
+
from intelmq_extensions.bots.experts.replace_in_dict.expert import (
|
|
12
|
+
ReplaceInDictExpertBot,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ....base import BotTestCase
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TestReplaceInDictExpertBot(BotTestCase, unittest.TestCase):
|
|
19
|
+
"""
|
|
20
|
+
A TestCase for ReplaceInDictExpertBot.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def set_bot(cls):
|
|
25
|
+
cls.bot_reference = ReplaceInDictExpertBot
|
|
26
|
+
cls.sysconfig = {
|
|
27
|
+
"old_value": "\\u0000",
|
|
28
|
+
"new_value": "[nullbyte]",
|
|
29
|
+
"fields": "extra",
|
|
30
|
+
}
|
|
31
|
+
cls.default_input_message = {"__type": "Event"}
|
|
32
|
+
|
|
33
|
+
def test_event_no_changes(self):
|
|
34
|
+
message = {
|
|
35
|
+
"__type": "Event",
|
|
36
|
+
"time.observation": "2015-01-01T00:00:00+00:00",
|
|
37
|
+
"extra.payload": "foo",
|
|
38
|
+
"extra.name": "bar",
|
|
39
|
+
"extra.firmwarerev": 1,
|
|
40
|
+
}
|
|
41
|
+
self.input_message = copy.deepcopy(message)
|
|
42
|
+
self.run_bot()
|
|
43
|
+
self.assertMessageEqual(0, message)
|
|
44
|
+
|
|
45
|
+
def test_event_no_extra(self):
|
|
46
|
+
message = {
|
|
47
|
+
"__type": "Event",
|
|
48
|
+
"time.observation": "2015-01-01T00:00:00+00:00",
|
|
49
|
+
"feed.code": "foo",
|
|
50
|
+
}
|
|
51
|
+
self.input_message = copy.deepcopy(message)
|
|
52
|
+
self.run_bot()
|
|
53
|
+
self.assertMessageEqual(0, message)
|
|
54
|
+
|
|
55
|
+
def test_event_changes_one_dict(self):
|
|
56
|
+
message = {
|
|
57
|
+
"__type": "Event",
|
|
58
|
+
"time.observation": "2015-01-01T00:00:00+00:00",
|
|
59
|
+
"extra.payload": "foo\\u0000bar\\u0000",
|
|
60
|
+
"extra.name": "bar ok \\u0001 and not ok \\\\u0000",
|
|
61
|
+
"extra.firmwarerev": 1,
|
|
62
|
+
"feed.code": "foo",
|
|
63
|
+
}
|
|
64
|
+
self.input_message = copy.deepcopy(message)
|
|
65
|
+
self.run_bot()
|
|
66
|
+
|
|
67
|
+
message["extra.payload"] = "foo[nullbyte]bar[nullbyte]"
|
|
68
|
+
message["extra.name"] = "bar ok \\u0001 and not ok \\[nullbyte]"
|
|
69
|
+
self.assertMessageEqual(0, message)
|
|
70
|
+
|
|
71
|
+
def test_event_multiple_dict_fail_if_not_jsondict(self):
|
|
72
|
+
with self.assertRaises(ConfigurationError):
|
|
73
|
+
self.run_bot(
|
|
74
|
+
parameters={
|
|
75
|
+
"fields": "extra,output",
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
def test_event_other_fields_not_modified(self):
|
|
80
|
+
message = {
|
|
81
|
+
"__type": "Event",
|
|
82
|
+
"time.observation": "2015-01-01T00:00:00+00:00",
|
|
83
|
+
"feed.code": "foo\\u0000",
|
|
84
|
+
}
|
|
85
|
+
self.input_message = copy.deepcopy(message)
|
|
86
|
+
self.run_bot()
|
|
87
|
+
|
|
88
|
+
self.assertMessageEqual(0, message)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
if __name__ == "__main__":
|
|
92
|
+
unittest.main()
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: intelmq_extensions
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.0
|
|
4
4
|
Summary: Additional bots for IntelMQ
|
|
5
5
|
Author: CERT.at Data & Development Team
|
|
6
6
|
License: AGPLv3
|
|
7
7
|
Project-URL: Repository, https://github.com/certat/intelmq-extensions
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Requires-Python: >=3.
|
|
9
|
+
Requires-Python: >=3.9
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
|
-
Requires-Dist: rt<3.0.0,>=1.0.9
|
|
13
|
-
Requires-Dist: mergedeep
|
|
14
|
-
Requires-Dist: intelmq
|
|
15
|
-
Requires-Dist: tabulate>=0.7.5
|
|
16
|
-
Requires-Dist: psycopg2-binary
|
|
17
|
-
Requires-Dist: netaddr>=0.7.14
|
|
18
12
|
Requires-Dist: python-termstyle>=0.1.10
|
|
13
|
+
Requires-Dist: intelmq
|
|
19
14
|
Requires-Dist: importlib_metadata; python_version < "3.8"
|
|
15
|
+
Requires-Dist: netaddr>=0.7.14
|
|
16
|
+
Requires-Dist: rt<3.0.0,>=1.0.9
|
|
17
|
+
Requires-Dist: psycopg2-binary
|
|
18
|
+
Requires-Dist: tabulate>=0.7.5
|
|
19
|
+
Requires-Dist: mergedeep
|
|
20
20
|
Provides-Extra: dev
|
|
21
21
|
Requires-Dist: pytest; extra == "dev"
|
|
22
22
|
Requires-Dist: tox>=4; extra == "dev"
|
|
@@ -30,14 +30,25 @@ Dynamic: requires-dist
|
|
|
30
30
|
|
|
31
31
|
# IntelMQ Extensions
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
[](https://github.com/certat/intelmq-extensions/actions/workflows/ci.yml)
|
|
34
|
+
|
|
35
|
+
This project collects customized bots and some helper scripts for
|
|
36
|
+
[IntelMQ](https://github.com/certtools/intelmq) used primary by CERT.at.
|
|
37
|
+
|
|
38
|
+
It's a combination of customization previously available in [certat/intelmq](https://github.com/certat/intelmq)
|
|
39
|
+
as well as newer solutions.
|
|
34
40
|
|
|
35
41
|
## Usage
|
|
36
42
|
|
|
37
|
-
Install the package on the machine
|
|
38
|
-
|
|
43
|
+
Install the package on the machine or virtualenv, where you have the IntelMQ, using
|
|
44
|
+
`pip install intelmq-extensions`. Then, the bots will be available as any other IntelMQ
|
|
45
|
+
bot in the Manager as well to import using `intelmq.bots.*.certat` namespace, e.g.
|
|
46
|
+
`intelmq.bots.experts.certat.vulnerability_lookup.expert`
|
|
39
47
|
|
|
48
|
+
## Documentation
|
|
40
49
|
|
|
50
|
+
There is a limited documentation available. Consult bot Python code to see information
|
|
51
|
+
about the usage and available configuration.
|
|
41
52
|
|
|
42
53
|
## Running tests
|
|
43
54
|
|
|
@@ -58,3 +69,9 @@ This package comes with test runners configured using `tox`. To use them:
|
|
|
58
69
|
tox -efull-with-docker -- intelmq_extensions/tests/bots/experts/squelcher/test_expert.py::TestSquelcherExpertBot::test_address_match1
|
|
59
70
|
|
|
60
71
|
```
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
Part of the development was financed by the European Union.
|
|
76
|
+
|
|
77
|
+
|
|
@@ -17,6 +17,8 @@ intelmq_extensions/bots/experts/event_group_splitter/__init__.py,sha256=47DEQpj8
|
|
|
17
17
|
intelmq_extensions/bots/experts/event_group_splitter/expert.py,sha256=Jq9v1p94dwbiFKeaulf7pg5XA15hlb967Wwvjp-fZ3g,4095
|
|
18
18
|
intelmq_extensions/bots/experts/event_splitter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
19
|
intelmq_extensions/bots/experts/event_splitter/expert.py,sha256=DcAXwapXMmP-IP7sPwDBWJ9cMu9UCcKh3nJPSDJbGfU,1204
|
|
20
|
+
intelmq_extensions/bots/experts/replace_in_dict/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
intelmq_extensions/bots/experts/replace_in_dict/expert.py,sha256=GyevseD5n5E5cNO_6PZqnK8gZhILplPvLcwyS6iXNks,1375
|
|
20
22
|
intelmq_extensions/bots/experts/squelcher/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
23
|
intelmq_extensions/bots/experts/squelcher/expert.py,sha256=cvmA4N5MjC_QazcEmCgW_uffn0P-4jPIPMikjtOcEd0,11022
|
|
22
24
|
intelmq_extensions/bots/experts/vulnerability_lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -69,6 +71,8 @@ intelmq_extensions/tests/bots/experts/event_group_splitter/__init__.py,sha256=47
|
|
|
69
71
|
intelmq_extensions/tests/bots/experts/event_group_splitter/test_expert.py,sha256=FcuFOzOGctWQ0whG5e7t9_K2Vf7a_doDEl5WoWhDqq8,9124
|
|
70
72
|
intelmq_extensions/tests/bots/experts/event_splitter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
71
73
|
intelmq_extensions/tests/bots/experts/event_splitter/test_expert.py,sha256=KGjh86BacNiUcJGg8kxggHoJY3ORZPmPFmVPYJK0J6A,2752
|
|
74
|
+
intelmq_extensions/tests/bots/experts/replace_in_dict/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
75
|
+
intelmq_extensions/tests/bots/experts/replace_in_dict/test_expert.py,sha256=aGJ0zGjU4qEKP4HLIF4LnFyze7I__DMrMJz6ynf5bko,2630
|
|
72
76
|
intelmq_extensions/tests/bots/experts/squelcher/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
77
|
intelmq_extensions/tests/bots/experts/squelcher/test_expert.py,sha256=oHE3E9q2fMwPVavgZM4Bs8kQCGehpMFEv3biJQs_JJk,17731
|
|
74
78
|
intelmq_extensions/tests/bots/experts/vulnerability_lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -92,9 +96,9 @@ intelmq_extensions/tests/cli/test_intelmqcli.py,sha256=Izy0FHVkj44DCgp4Zm8f1vYTd
|
|
|
92
96
|
intelmq_extensions/tests/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
93
97
|
intelmq_extensions/tests/lib/base.py,sha256=iD4MuDq0Dh-CqFDgyXf4CfaJgT4AL7XAcNjPGxUlkPM,2560
|
|
94
98
|
intelmq_extensions/tests/lib/test_api_helpers.py,sha256=cO6KNfVjwpYgevyyxxokOAmk7VXo-8T8OCH8hQhf980,3807
|
|
95
|
-
intelmq_extensions-1.
|
|
96
|
-
intelmq_extensions-1.
|
|
97
|
-
intelmq_extensions-1.
|
|
98
|
-
intelmq_extensions-1.
|
|
99
|
-
intelmq_extensions-1.
|
|
100
|
-
intelmq_extensions-1.
|
|
99
|
+
intelmq_extensions-1.9.0.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
|
|
100
|
+
intelmq_extensions-1.9.0.dist-info/METADATA,sha256=LLNY1-QSCk8YJ-7WVaQtvqXNb7y_JK6Mb-Zgv6UD3ls,2841
|
|
101
|
+
intelmq_extensions-1.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
102
|
+
intelmq_extensions-1.9.0.dist-info/entry_points.txt,sha256=Kvg6WqHn9mGadCDWPFey0Bz6QYTT0JG7BuE0X_6MF80,3725
|
|
103
|
+
intelmq_extensions-1.9.0.dist-info/top_level.txt,sha256=YVqZnmAiBfQPNdJPAv64Afvc41p3B24Akx2v2-3BFyc,19
|
|
104
|
+
intelmq_extensions-1.9.0.dist-info/RECORD,,
|
|
@@ -6,6 +6,7 @@ intelmq.bots.experts.certat.certat_contact_intern.expert = intelmq_extensions.bo
|
|
|
6
6
|
intelmq.bots.experts.certat.copy_extra.expert = intelmq_extensions.bots.experts.copy_extra.expert:BOT.run
|
|
7
7
|
intelmq.bots.experts.certat.event_group_splitter.expert = intelmq_extensions.bots.experts.event_group_splitter.expert:BOT.run
|
|
8
8
|
intelmq.bots.experts.certat.event_splitter.expert = intelmq_extensions.bots.experts.event_splitter.expert:BOT.run
|
|
9
|
+
intelmq.bots.experts.certat.replace_in_dict.expert = intelmq_extensions.bots.experts.replace_in_dict.expert:BOT.run
|
|
9
10
|
intelmq.bots.experts.certat.squelcher.expert = intelmq_extensions.bots.experts.squelcher.expert:BOT.run
|
|
10
11
|
intelmq.bots.experts.certat.vulnerability_lookup.expert = intelmq_extensions.bots.experts.vulnerability_lookup.expert:BOT.run
|
|
11
12
|
intelmq.bots.outputs.certat.mattermost.output = intelmq_extensions.bots.outputs.mattermost.output:BOT.run
|
|
@@ -21,6 +22,7 @@ intelmq_extensions.bots.experts.certat_contact_intern.expert = intelmq_extension
|
|
|
21
22
|
intelmq_extensions.bots.experts.copy_extra.expert = intelmq_extensions.bots.experts.copy_extra.expert:BOT.run
|
|
22
23
|
intelmq_extensions.bots.experts.event_group_splitter.expert = intelmq_extensions.bots.experts.event_group_splitter.expert:BOT.run
|
|
23
24
|
intelmq_extensions.bots.experts.event_splitter.expert = intelmq_extensions.bots.experts.event_splitter.expert:BOT.run
|
|
25
|
+
intelmq_extensions.bots.experts.replace_in_dict.expert = intelmq_extensions.bots.experts.replace_in_dict.expert:BOT.run
|
|
24
26
|
intelmq_extensions.bots.experts.squelcher.expert = intelmq_extensions.bots.experts.squelcher.expert:BOT.run
|
|
25
27
|
intelmq_extensions.bots.experts.vulnerability_lookup.expert = intelmq_extensions.bots.experts.vulnerability_lookup.expert:BOT.run
|
|
26
28
|
intelmq_extensions.bots.outputs.mattermost.output = intelmq_extensions.bots.outputs.mattermost.output:BOT.run
|
|
File without changes
|
|
File without changes
|