detectmatelibrary 0.6.2__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectmatelibrary-0.6.2/src/detectmatelibrary.egg-info → detectmatelibrary-0.7.0}/PKG-INFO +13 -10
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/README.md +1 -1
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/pyproject.toml +23 -11
- detectmatelibrary-0.7.0/src/detectmatelibrary/alert_aggregation/__init__.py +29 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/alert_aggregation/alertbert.py +447 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/time_delta.py +1 -1
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/alert_aggregator.py +1 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/core.py +9 -3
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/ecvc_detector.py +18 -11
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/helper/select_component.py +1 -1
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/metadata.py +1 -1
- detectmatelibrary-0.7.0/src/detectmatelibrary/parsers/__init__.py +36 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +10 -15
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +20 -16
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +11 -9
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +17 -6
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +46 -31
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +63 -57
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +26 -23
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +10 -11
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +55 -50
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +3 -2
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +7 -11
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/_parser.py +6 -7
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/schemas/__init__.py +3 -1
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/schemas/_classes.py +97 -36
- detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/__init__.py +24 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/default.py +95 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/mongos.py +117 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/redis.py +90 -0
- {detectmatelibrary-0.6.2/src/detectmatelibrary/schemas → detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/protobuf}/_op.py +1 -1
- {detectmatelibrary-0.6.2/src/detectmatelibrary/schemas → detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/protobuf}/schemas_pb2.py +2 -2
- detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/datasets.py +364 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/models.py +1700 -0
- detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/preprocessing.py +325 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/_op.py +14 -4
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/deeplog.py +26 -12
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/logbert.py +31 -10
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/persistency_saver.py +0 -4
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0/src/detectmatelibrary.egg-info}/PKG-INFO +13 -10
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/SOURCES.txt +10 -3
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/requires.txt +10 -8
- detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/__init__.py +0 -12
- detectmatelibrary-0.6.2/src/detectmatelibrary/parsers/__init__.py +0 -23
- detectmatelibrary-0.6.2/src/detectmatelibrary/utils/data_normalizer.py +0 -31
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/LICENSE.md +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/setup.cfg +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_parser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/basic_concat.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/logid_count.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_alert_aggregation/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_persist.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_stability.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_variable_hooks.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/deeplearning_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/tracker_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/variable_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/_compile.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/_formats.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_basic_component.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_fed_component.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_train_buffer.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/parser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/constants.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/charset_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/deeplog_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/event_sequence_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/logbert_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_event_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_value_combo_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_value_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/random_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/rule_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/scvs_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/value_range_detector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/helper/from_to.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/autoparser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/drain.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/json_parser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/parser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/tree_matcher.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/pipeline.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/logging.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/create_workspace.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/aux.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/data_buffer.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/finetune.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/id_generator.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/key_extractor.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/basic_persistency.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/base.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/event_tracker.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/multi_tracker.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/single_tracker.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/rle_list.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/__init__.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/classification_methods.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_classifier.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_tracker.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/event_persistency.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/slow_persistency.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/sequence_encoding.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
- {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: detectmatelibrary
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: DetectMate Library for log processing components
|
|
5
|
-
Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
|
|
5
|
+
Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf, Lukas Karner
|
|
6
6
|
License-Expression: EUPL-1.2
|
|
7
7
|
Project-URL: Homepage, https://ait-detectmate.github.io/DetectMateLibrary/latest
|
|
8
8
|
Project-URL: Repository, https://github.com/ait-detectmate/DetectMateLibrary
|
|
@@ -21,6 +21,10 @@ Requires-Dist: pyarrow>=24.0.0
|
|
|
21
21
|
Requires-Dist: jax>=0.11.0
|
|
22
22
|
Requires-Dist: flax>=0.12.8
|
|
23
23
|
Requires-Dist: optax>=0.2.8
|
|
24
|
+
Requires-Dist: polars>=1.40.1
|
|
25
|
+
Requires-Dist: polars[rtcompat]>=1.38.1
|
|
26
|
+
Requires-Dist: pymongo>=4.19.0
|
|
27
|
+
Requires-Dist: redis[hiredis]>=8.1.0
|
|
24
28
|
Provides-Extra: llm
|
|
25
29
|
Requires-Dist: openai>=2.26.0; extra == "llm"
|
|
26
30
|
Requires-Dist: tenacity>=9.1.4; extra == "llm"
|
|
@@ -28,21 +32,20 @@ Requires-Dist: scipy>=1.17.1; extra == "llm"
|
|
|
28
32
|
Requires-Dist: scikit-learn>=1.8.0; extra == "llm"
|
|
29
33
|
Requires-Dist: tiktoken>=0.12.0; extra == "llm"
|
|
30
34
|
Requires-Dist: pandas>=2.3.2; extra == "llm"
|
|
31
|
-
Provides-Extra:
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
|
|
35
|
-
Requires-Dist:
|
|
35
|
+
Provides-Extra: alertbert
|
|
36
|
+
Requires-Dist: torch; extra == "alertbert"
|
|
37
|
+
Requires-Dist: tensordict; extra == "alertbert"
|
|
38
|
+
Requires-Dist: scikit-learn; extra == "alertbert"
|
|
39
|
+
Requires-Dist: scipy; extra == "alertbert"
|
|
36
40
|
Provides-Extra: full
|
|
37
41
|
Requires-Dist: detectmatelibrary[llm]; extra == "full"
|
|
38
|
-
Requires-Dist: detectmatelibrary[
|
|
39
|
-
Requires-Dist: detectmatelibrary[polars-rtcompat]; extra == "full"
|
|
42
|
+
Requires-Dist: detectmatelibrary[alertbert]; extra == "full"
|
|
40
43
|
Dynamic: license-file
|
|
41
44
|
|
|
42
45
|
|
|
43
46
|
# DetectMateLibrary
|
|
44
47
|
|
|
45
|
-

|
|
46
49
|
|
|
47
50
|
DetectMate is a flexible, component-based library for parsing and log anomaly detection.
|
|
48
51
|
|
|
@@ -4,7 +4,7 @@ description = "DetectMate Library for log processing components"
|
|
|
4
4
|
readme = {file = "README.md", content-type = "text/markdown"}
|
|
5
5
|
dynamic = ["version"]
|
|
6
6
|
authors = [{"name" = "André García Gómez"}, {"name" = "Viktor Beck"}, {"name" = "Thorina Boenke"},
|
|
7
|
-
{"name" = "Wolfgang Hotwagner"}, {"name" = "Anna Erdi"}, {"name" = "Ernst Leierzopf"}]
|
|
7
|
+
{"name" = "Wolfgang Hotwagner"}, {"name" = "Anna Erdi"}, {"name" = "Ernst Leierzopf"}, {"name" = "Lukas Karner"}]
|
|
8
8
|
license = "EUPL-1.2"
|
|
9
9
|
requires-python = ">=3.12"
|
|
10
10
|
dependencies = [
|
|
@@ -20,6 +20,10 @@ dependencies = [
|
|
|
20
20
|
"jax>=0.11.0",
|
|
21
21
|
"flax>=0.12.8",
|
|
22
22
|
"optax>=0.2.8",
|
|
23
|
+
"polars>=1.40.1",
|
|
24
|
+
"polars[rtcompat]>=1.38.1",
|
|
25
|
+
"pymongo>=4.19.0",
|
|
26
|
+
"redis[hiredis]>=8.1.0",
|
|
23
27
|
]
|
|
24
28
|
|
|
25
29
|
[tool.pytest.ini_options]
|
|
@@ -46,18 +50,16 @@ llm = [
|
|
|
46
50
|
"tiktoken>=0.12.0",
|
|
47
51
|
"pandas>=2.3.2",
|
|
48
52
|
]
|
|
49
|
-
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
"polars[rtcompat]>=1.38.1",
|
|
53
|
+
alertbert = [
|
|
54
|
+
"torch",
|
|
55
|
+
"tensordict",
|
|
56
|
+
"scikit-learn",
|
|
57
|
+
"scipy",
|
|
55
58
|
]
|
|
56
59
|
# install all optional extras at once: uv pip install detectmatelibrary[full]
|
|
57
60
|
full = [
|
|
58
61
|
"detectmatelibrary[llm]",
|
|
59
|
-
"detectmatelibrary[
|
|
60
|
-
"detectmatelibrary[polars-rtcompat]",
|
|
62
|
+
"detectmatelibrary[alertbert]",
|
|
61
63
|
]
|
|
62
64
|
|
|
63
65
|
[tool.setuptools.dynamic]
|
|
@@ -68,7 +70,7 @@ Homepage = "https://ait-detectmate.github.io/DetectMateLibrary/latest"
|
|
|
68
70
|
Repository = "https://github.com/ait-detectmate/DetectMateLibrary"
|
|
69
71
|
|
|
70
72
|
[build-system]
|
|
71
|
-
requires = ["setuptools>=
|
|
73
|
+
requires = ["setuptools>=84.1.0", "wheel", "packaging>=24.2"]
|
|
72
74
|
build-backend = "setuptools.build_meta"
|
|
73
75
|
|
|
74
76
|
[tool.setuptools.packages.find]
|
|
@@ -78,7 +80,7 @@ where = ["src"]
|
|
|
78
80
|
mate = "detectmatelibrary.tools.workspace.create_workspace:main"
|
|
79
81
|
|
|
80
82
|
[[tool.mypy.overrides]]
|
|
81
|
-
module = "detectmatelibrary.schemas.schemas_pb2"
|
|
83
|
+
module = "detectmatelibrary.schemas.protobuf.schemas_pb2"
|
|
82
84
|
ignore_errors = true
|
|
83
85
|
|
|
84
86
|
[tool.setuptools]
|
|
@@ -89,3 +91,13 @@ license-files = ["LICENSE.md"]
|
|
|
89
91
|
detectmatelibrary = [
|
|
90
92
|
"tools/workspace/templates/data/*.json"
|
|
91
93
|
]
|
|
94
|
+
|
|
95
|
+
[tool.uv.sources]
|
|
96
|
+
torch = [
|
|
97
|
+
{ index = "torch-cpu", extra = "alertbert" },
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
[[tool.uv.index]]
|
|
101
|
+
name = "torch-cpu"
|
|
102
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
103
|
+
explicit = true
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# mypy: ignore-errors
|
|
2
|
+
|
|
3
|
+
from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig
|
|
4
|
+
from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig
|
|
5
|
+
from .time_delta import TimeDeltaAggregation, TimeDeltaAggregationConfig
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"BasicConcatAggregation",
|
|
9
|
+
"BasicConcatAggregationConfig",
|
|
10
|
+
"TimeDeltaAggregation",
|
|
11
|
+
"TimeDeltaAggregationConfig",
|
|
12
|
+
"LogIDCountAggegation",
|
|
13
|
+
"LogIDCountAggegationConfig",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def __getattr__(name):
|
|
18
|
+
"""Load AlertBERT only when one of its exports is requested."""
|
|
19
|
+
if name in {"AlertBERTAggregation", "AlertBERTAggregationConfig"}:
|
|
20
|
+
from .alertbert import AlertBERTAggregation, AlertBERTAggregationConfig
|
|
21
|
+
|
|
22
|
+
# Cache both exports for subsequent attribute access.
|
|
23
|
+
globals().update(
|
|
24
|
+
AlertBERTAggregation=AlertBERTAggregation,
|
|
25
|
+
AlertBERTAggregationConfig=AlertBERTAggregationConfig,
|
|
26
|
+
)
|
|
27
|
+
return globals()[name]
|
|
28
|
+
|
|
29
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from datetime import datetime as dt
|
|
3
|
+
from typing import Any, Literal, cast
|
|
4
|
+
|
|
5
|
+
from pydantic import Field, NonNegativeFloat
|
|
6
|
+
|
|
7
|
+
from detectmatelibrary.base_alert_aggregation import (
|
|
8
|
+
DensityClusteringAggregation,
|
|
9
|
+
DensityClusteringAggregationConfig,
|
|
10
|
+
)
|
|
11
|
+
from detectmatelibrary.schemas import AggregateSchema, DetectorSchema
|
|
12
|
+
from detectmatelibrary.tools.logging import logger
|
|
13
|
+
from detectmatelibrary.utils.alertbert.datasets import AlertDataset, split_alert_dataset
|
|
14
|
+
from detectmatelibrary.utils.alertbert.models import (
|
|
15
|
+
AlertBERT,
|
|
16
|
+
MaskedLanguageModel,
|
|
17
|
+
)
|
|
18
|
+
from detectmatelibrary.utils.data_buffer import BufferMode
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AlertBERTAggregationConfig(DensityClusteringAggregationConfig):
|
|
22
|
+
"""Configuration of the AlertBERTAggregation class.
|
|
23
|
+
|
|
24
|
+
Besides the parameters of the alert aggregation this configuration
|
|
25
|
+
also determines the training of the masked-languge-model used to
|
|
26
|
+
obtain the alert embeddings.
|
|
27
|
+
"""
|
|
28
|
+
# general
|
|
29
|
+
method_type: str = Field(
|
|
30
|
+
default="AlertBERTAggregation",
|
|
31
|
+
description="The type of the aggregation method. Defaults to 'AlertBERT_aggregator'.",
|
|
32
|
+
)
|
|
33
|
+
_model_id: str = f"{method_type.default}_{dt.now().isoformat()}" # type: ignore[attr-defined]
|
|
34
|
+
model_id: str = Field(
|
|
35
|
+
default=_model_id,
|
|
36
|
+
description="""The unique identifier of the model.
|
|
37
|
+
Defaults to 'AlertBERT_aggregator_<current_datetime>'.""",
|
|
38
|
+
)
|
|
39
|
+
path: str = Field(
|
|
40
|
+
default="alertbert_models",
|
|
41
|
+
description="The path to the save location of the AlertBERT model. Defaults to 'alertbert_models'.",
|
|
42
|
+
)
|
|
43
|
+
train_model: bool = Field(
|
|
44
|
+
default=True,
|
|
45
|
+
description="""Whether to train the AlertBERT model.
|
|
46
|
+
If False, the model will be loaded from disk.
|
|
47
|
+
Defaults to False.""",
|
|
48
|
+
)
|
|
49
|
+
save_model: bool = Field(
|
|
50
|
+
default=True,
|
|
51
|
+
description="""Whether to save the trained AlertBERT model to disk.
|
|
52
|
+
Defaults to True.""",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# aggregation
|
|
56
|
+
delta: NonNegativeFloat = Field(
|
|
57
|
+
default=10.0,
|
|
58
|
+
description="The threshold for the clustering. Defaults to 10.0."
|
|
59
|
+
)
|
|
60
|
+
theta: NonNegativeFloat = Field(
|
|
61
|
+
default=20.0,
|
|
62
|
+
description="The parameter for the time-cosine-distance. Defaults to 20.0.",
|
|
63
|
+
)
|
|
64
|
+
dim_reduction: int = Field(
|
|
65
|
+
default=2,
|
|
66
|
+
description="""The dimension to which the alert embeddings will be reduced via PCA before clustering.
|
|
67
|
+
Defaults to 2."""
|
|
68
|
+
)
|
|
69
|
+
clustering_backend: Literal["detectmate", "scipy"] = Field(
|
|
70
|
+
default="detectmate",
|
|
71
|
+
description="""The backend to use for clustering.
|
|
72
|
+
The option "detectmate" uses the implementation inherited from the parent class,
|
|
73
|
+
the option "scipy" uses the implementation of the AlertBERT model.
|
|
74
|
+
Defaults to "detectmate".""",
|
|
75
|
+
)
|
|
76
|
+
layers: tuple[str] | tuple[str, str] = Field(
|
|
77
|
+
default=("embedding", "encoder"),
|
|
78
|
+
description="""The layers of the AlertBERT model to use for obtaining the alert embeddings.
|
|
79
|
+
Defaults to ("embedding", "encoder").""",
|
|
80
|
+
)
|
|
81
|
+
buffer_size: int = Field(
|
|
82
|
+
default=1000,
|
|
83
|
+
description="The size of the buffer. Defaults to 1000."
|
|
84
|
+
)
|
|
85
|
+
buffer_mode: BufferMode = Field(
|
|
86
|
+
default=BufferMode.BATCH,
|
|
87
|
+
description="The mode of the buffer. Defaults to 'BATCH'.",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# data
|
|
91
|
+
train_size: int = Field(
|
|
92
|
+
default=50000,
|
|
93
|
+
description="The number of alerts to use for training the AlertBERT model. Defaults to 50000."
|
|
94
|
+
)
|
|
95
|
+
val_size: int = Field(
|
|
96
|
+
default=10000,
|
|
97
|
+
description="""The number of alerts to use for validation during training of the AlertBERT model.
|
|
98
|
+
Defaults to 10000."""
|
|
99
|
+
)
|
|
100
|
+
skip_first: int = Field(
|
|
101
|
+
default=0,
|
|
102
|
+
description="""The number of alerts to skip before starting to collect alerts for training the
|
|
103
|
+
AlertBERT model.
|
|
104
|
+
Defaults to 0."""
|
|
105
|
+
)
|
|
106
|
+
min_freq: int = Field(
|
|
107
|
+
default=10,
|
|
108
|
+
description="""The minimum frequency of tokens to be included in the
|
|
109
|
+
vocabulary of the AlertBERT model.
|
|
110
|
+
Defaults to 10.""",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# model
|
|
114
|
+
features: tuple[str, ...] = Field(
|
|
115
|
+
default=("description",),
|
|
116
|
+
description="""The keys of fields of the alerts to use as features for training the AlertBERT model.
|
|
117
|
+
Fields within the 'alertDetails' field can be accessed by the syntax 'alertDetails:fieldName'.
|
|
118
|
+
Defaults to ("description",).""",
|
|
119
|
+
)
|
|
120
|
+
encoding: Literal["index", "relevantTimestamp"] = Field(
|
|
121
|
+
default="relevantTimestamp",
|
|
122
|
+
description="""The features of the alerts to use for rotary positional encoding.
|
|
123
|
+
Defaults to "relevantTimestamp".""",
|
|
124
|
+
)
|
|
125
|
+
n_heads: int = Field(
|
|
126
|
+
default=4,
|
|
127
|
+
description="The number of attention heads in the AlertBERT model. Defaults to 4."
|
|
128
|
+
)
|
|
129
|
+
dim_per_head: int = Field(
|
|
130
|
+
default=16,
|
|
131
|
+
description="The dimensionality of each attention head in the AlertBERT model. Defaults to 16.",
|
|
132
|
+
)
|
|
133
|
+
num_layers: int = Field(
|
|
134
|
+
default=1,
|
|
135
|
+
description="The number of layers in the model in the AlertBERT model. Defaults to 1."
|
|
136
|
+
)
|
|
137
|
+
feedforward_factor: int = Field(
|
|
138
|
+
default=4,
|
|
139
|
+
description="""The factor by which the feedforward dimension is larger than the model dimension
|
|
140
|
+
in the AlertBERT model. Defaults to 4.""",
|
|
141
|
+
)
|
|
142
|
+
rotary_max_exp: int = Field(
|
|
143
|
+
default=14,
|
|
144
|
+
description="""The maximum exponent for the rotary positional encoding in the AlertBERT model.
|
|
145
|
+
Defaults to 14.""",
|
|
146
|
+
)
|
|
147
|
+
rotary_cutoff: float = Field(
|
|
148
|
+
default=0.75,
|
|
149
|
+
description="""The cutoff for the rotary positional encoding in the AlertBERT model.
|
|
150
|
+
Defaults to 0.75.""",
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# training
|
|
154
|
+
context_size: int = Field(
|
|
155
|
+
default=1024,
|
|
156
|
+
description="The context size for the AlertBERT model. Defaults to 1024."
|
|
157
|
+
)
|
|
158
|
+
batch_size: int = Field(
|
|
159
|
+
default=8,
|
|
160
|
+
description="The batch size for training the AlertBERT model. Defaults to 8."
|
|
161
|
+
)
|
|
162
|
+
updates: tuple[int, int] = Field(
|
|
163
|
+
default=(240, 300),
|
|
164
|
+
description="The number of updates for training the AlertBERT model. Defaults to (240, 300).",
|
|
165
|
+
)
|
|
166
|
+
lr: float = Field(
|
|
167
|
+
default=5e-3,
|
|
168
|
+
description="The learning rate for training the AlertBERT model. Defaults to 5e-3.",
|
|
169
|
+
)
|
|
170
|
+
warm_up_steps: int = Field(
|
|
171
|
+
default=20,
|
|
172
|
+
description="The number of warm-up steps for training the AlertBERT model. Defaults to 100."
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def __init__(self, **data: Any) -> None:
|
|
176
|
+
"""Initialize the AlertBERTAggregationConfig class.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
**data (Any): The data to initialize the configuration with.
|
|
180
|
+
"""
|
|
181
|
+
super().__init__(**data)
|
|
182
|
+
if self.theta <= self.delta:
|
|
183
|
+
logger.warning(
|
|
184
|
+
f"The parameter theta ({self.theta}) should be larger than delta ({
|
|
185
|
+
self.delta
|
|
186
|
+
}). For theta <= delta, AlertBERT is equivalent to TimeDelta!"
|
|
187
|
+
)
|
|
188
|
+
if self.buffer_size > self.train_size + self.val_size:
|
|
189
|
+
logger.info(
|
|
190
|
+
f"The buffer size {
|
|
191
|
+
self.buffer_size
|
|
192
|
+
} is larger than the training and validation size ({
|
|
193
|
+
self.train_size + self.val_size
|
|
194
|
+
})."
|
|
195
|
+
)
|
|
196
|
+
if self.train_size < self.context_size * self.batch_size:
|
|
197
|
+
logger.error(
|
|
198
|
+
f"The training size {self.train_size} is smaller than context size ({
|
|
199
|
+
self.context_size
|
|
200
|
+
}) * batch size ({
|
|
201
|
+
self.batch_size
|
|
202
|
+
}). This will lead to empty batches during training!"
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
def _get_mlm_config(self) -> dict[str, Any]:
|
|
206
|
+
return self.model_dump(
|
|
207
|
+
exclude={
|
|
208
|
+
"buffer_mode",
|
|
209
|
+
"params",
|
|
210
|
+
"component_type",
|
|
211
|
+
"auto_config",
|
|
212
|
+
"start_id",
|
|
213
|
+
"data_use_training",
|
|
214
|
+
"data_use_configure",
|
|
215
|
+
"use_config_data_as_training",
|
|
216
|
+
"buffer_size",
|
|
217
|
+
"delta",
|
|
218
|
+
"theta",
|
|
219
|
+
"dim_reduciton",
|
|
220
|
+
"clustering_backend",
|
|
221
|
+
"layers",
|
|
222
|
+
}
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class AlertBERTAggregation(DensityClusteringAggregation):
|
|
227
|
+
"""Alert Aggregation based on alert timestamps and embeddings obtained from masked-language-models as
|
|
228
|
+
proposed by [AlertBERT: A noise-robust alert grouping framework for simultaneous cyber attacks]
|
|
229
|
+
(https://arxiv.org/abs/2602.06534).
|
|
230
|
+
The alerts will be assigned to the same group if their time-cosine-distance is below a
|
|
231
|
+
certain threshold theta.
|
|
232
|
+
The time-cosine-distance is the maximum of the time difference and the cosine similarity between
|
|
233
|
+
the embeddings of each alert.
|
|
234
|
+
"""
|
|
235
|
+
|
|
236
|
+
def __init__(
|
|
237
|
+
self,
|
|
238
|
+
name: str = "AlertBERTAggregation",
|
|
239
|
+
config: AlertBERTAggregationConfig = AlertBERTAggregationConfig(),
|
|
240
|
+
) -> None:
|
|
241
|
+
"""Initialize the AlertBERTAggregation class.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
config (AlertBERTAggregationConfig): The configuration of this method.
|
|
245
|
+
name (str): The name identifying this aggregation method.
|
|
246
|
+
"""
|
|
247
|
+
if isinstance(config, dict):
|
|
248
|
+
config = AlertBERTAggregationConfig.from_dict(config, name)
|
|
249
|
+
super().__init__(name=name, config=config)
|
|
250
|
+
self.config: AlertBERTAggregationConfig # ty: ignore[invalid-mutable-override]
|
|
251
|
+
self.description = "Alert aggregation by the AlertBERT method."
|
|
252
|
+
self._set_up_mlm()
|
|
253
|
+
if self.config.train_model:
|
|
254
|
+
self.collected_data: list[DetectorSchema] = []
|
|
255
|
+
self.timestamp_extraction_fn: Callable[[DetectorSchema], float] = lambda alert: (
|
|
256
|
+
float(alert["logTimestamps"][-1])
|
|
257
|
+
)
|
|
258
|
+
self.last_timestamp: float = 0.0
|
|
259
|
+
|
|
260
|
+
def aggregate_alerts(
|
|
261
|
+
self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
|
|
262
|
+
) -> bool:
|
|
263
|
+
"""Aggregate alerts into groups using the AlertBERT method. If the
|
|
264
|
+
model is set to be trained, it will first collect enough alerts to
|
|
265
|
+
train the model, and then proceed with the aggregation. If the model is
|
|
266
|
+
already trained, it will directly aggregate the alerts using the
|
|
267
|
+
specified clustering backend.
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
input_ (list[DetectorSchema] | DetectorSchema): The alerts to aggregate.
|
|
271
|
+
output_ (AggregateSchema): The output schema to populate.
|
|
272
|
+
|
|
273
|
+
Returns:
|
|
274
|
+
bool: True if the alerts were aggregated, False otherwise.
|
|
275
|
+
"""
|
|
276
|
+
if not isinstance(input_, list):
|
|
277
|
+
input_ = [input_]
|
|
278
|
+
if self.timestamp_extraction_fn(input_[0]) < self.last_timestamp:
|
|
279
|
+
raise ValueError("Alerts must be given in order of their timestamps!")
|
|
280
|
+
else:
|
|
281
|
+
self.last_timestamp = self.timestamp_extraction_fn(input_[-1])
|
|
282
|
+
|
|
283
|
+
if self.config.train_model:
|
|
284
|
+
self.collected_data.extend(input_)
|
|
285
|
+
if (
|
|
286
|
+
len(self.collected_data)
|
|
287
|
+
>= self.config.skip_first
|
|
288
|
+
+ self.config.train_size
|
|
289
|
+
+ self.config.val_size
|
|
290
|
+
):
|
|
291
|
+
logger.info("Data collection complete.")
|
|
292
|
+
self._train_mlm()
|
|
293
|
+
del self.collected_data
|
|
294
|
+
logger.info("Proceeding with alert aggregation ...")
|
|
295
|
+
self.last_timestamp = 0.0
|
|
296
|
+
return self.aggregate_alerts(input_, output_)
|
|
297
|
+
else:
|
|
298
|
+
return False
|
|
299
|
+
else:
|
|
300
|
+
if self.config.clustering_backend == "detectmate":
|
|
301
|
+
return super().aggregate_alerts(input_, output_)
|
|
302
|
+
elif self.config.clustering_backend == "scipy":
|
|
303
|
+
return self._aggregate_alerts(input_, output_)
|
|
304
|
+
else:
|
|
305
|
+
raise ValueError(
|
|
306
|
+
f"Clustering backend '{self.config.clustering_backend}' is not supported."
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
def _aggregate_alerts(
|
|
310
|
+
self, input_: list[DetectorSchema], output_: AggregateSchema
|
|
311
|
+
) -> bool:
|
|
312
|
+
"""Aggregate alerts into groups using the AlertBERT method with the
|
|
313
|
+
scipy backend.
|
|
314
|
+
|
|
315
|
+
Args:
|
|
316
|
+
input_ (list[DetectorSchema]): The alerts to aggregate.
|
|
317
|
+
output_ (AggregateSchema): The output schema to populate.
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
bool: True if the alerts were aggregated, False otherwise.
|
|
321
|
+
"""
|
|
322
|
+
self._add_new_alerts(input_)
|
|
323
|
+
labels = self.alertbert(self.relevant_alerts)
|
|
324
|
+
|
|
325
|
+
# keep track of which labels belong to which group_id's
|
|
326
|
+
label_to_group_id: dict[int, int] = {}
|
|
327
|
+
|
|
328
|
+
# first map labels to groups
|
|
329
|
+
# then assign new alerts to groups
|
|
330
|
+
for alert, label in zip(self.relevant_alerts, labels, strict=True):
|
|
331
|
+
# old alerts have group_id's already
|
|
332
|
+
if (
|
|
333
|
+
current_group_id := self.alert_id_to_group_id[alert["alertID"]]
|
|
334
|
+
) is not None:
|
|
335
|
+
# check which group_id the labels belong to and merge them if the labels say so
|
|
336
|
+
if label not in label_to_group_id:
|
|
337
|
+
label_to_group_id[label] = current_group_id
|
|
338
|
+
else:
|
|
339
|
+
other_group_id = label_to_group_id[label]
|
|
340
|
+
if other_group_id != current_group_id:
|
|
341
|
+
self._check_and_merge_groups({other_group_id, current_group_id})
|
|
342
|
+
# new alerts have no group_id's yet, assign them to the group of the label
|
|
343
|
+
else:
|
|
344
|
+
# a new label requires creating an new group
|
|
345
|
+
if label not in label_to_group_id:
|
|
346
|
+
group_id = self._new_group(alert["alertID"])
|
|
347
|
+
label_to_group_id[label] = group_id
|
|
348
|
+
# otherwise assign the new alert to the group of the label
|
|
349
|
+
else:
|
|
350
|
+
self.alert_id_to_group_id[alert["alertID"]] = label_to_group_id[
|
|
351
|
+
label
|
|
352
|
+
]
|
|
353
|
+
|
|
354
|
+
# write results to output
|
|
355
|
+
self._write_output(input_, output_)
|
|
356
|
+
return True
|
|
357
|
+
|
|
358
|
+
def metric(
|
|
359
|
+
self,
|
|
360
|
+
alerts: list[DetectorSchema],
|
|
361
|
+
return_last_rows: int = 0,
|
|
362
|
+
) -> list[list[NonNegativeFloat]]:
|
|
363
|
+
"""Compute the time-cosine-distance between the embeddings of the
|
|
364
|
+
alerts.
|
|
365
|
+
|
|
366
|
+
Args:
|
|
367
|
+
alerts (list[DetectorSchema]): The alerts for which to compute the metric.
|
|
368
|
+
return_last_rows (int): The number of last rows to return. Defaults to 0.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
list[list[NonNegativeFloat]]: The time-cosine-distances between the embeddings of the alerts.
|
|
372
|
+
"""
|
|
373
|
+
alert_emb = self.alertbert._get_embeddings(alerts)
|
|
374
|
+
return cast(
|
|
375
|
+
list[list[NonNegativeFloat]],
|
|
376
|
+
self.alertbert.time_cosine_metric(
|
|
377
|
+
alert_emb[-return_last_rows:], alert_emb
|
|
378
|
+
).tolist(),
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
def discard_alert_fn(self, alerts: list[DetectorSchema]) -> list[bool]:
|
|
382
|
+
"""Determine which alerts should be discarded based on their
|
|
383
|
+
timestamps.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
alerts (list[DetectorSchema]): The alerts to check for discarding.
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
list[bool]: A list of booleans indicating whether each alert should be discarded.
|
|
390
|
+
"""
|
|
391
|
+
cutoff = self.relevant_alerts[-1]["logTimestamps"][-1] - self.config.delta
|
|
392
|
+
return [alert["logTimestamps"][-1] < cutoff for alert in alerts]
|
|
393
|
+
|
|
394
|
+
def _set_up_mlm(self) -> None:
|
|
395
|
+
"""Set up the MaskedLanguageModel."""
|
|
396
|
+
self.mlm = MaskedLanguageModel(self.config)
|
|
397
|
+
if not self.config.train_model:
|
|
398
|
+
logger.info("Loading AlertBERT model from disk.")
|
|
399
|
+
self._set_up_alertbert()
|
|
400
|
+
else:
|
|
401
|
+
logger.debug("AlertBERT model will be trained ...")
|
|
402
|
+
|
|
403
|
+
def _set_up_alertbert(self) -> None:
|
|
404
|
+
"""Set up the AlertBERT model."""
|
|
405
|
+
self.alertbert = AlertBERT(
|
|
406
|
+
model=self.mlm,
|
|
407
|
+
config=self.config,
|
|
408
|
+
)
|
|
409
|
+
logger.info("AlertBERT model setup complete.")
|
|
410
|
+
|
|
411
|
+
def _train_mlm(
|
|
412
|
+
self,
|
|
413
|
+
use_gathered_alerts: bool = True,
|
|
414
|
+
train_data: AlertDataset | None = None,
|
|
415
|
+
val_data: AlertDataset | None = None,
|
|
416
|
+
) -> None:
|
|
417
|
+
"""Train the MaskedLanguageModel.
|
|
418
|
+
|
|
419
|
+
Args:
|
|
420
|
+
use_gathered_alerts (bool): Whether to use the gathered alerts for training. Defaults to True.
|
|
421
|
+
train_data (AlertDataset | None): The training dataset. Required if use_gathered_alerts is False.
|
|
422
|
+
val_data (AlertDataset | None): The validation dataset. Required if use_gathered_alerts is False.
|
|
423
|
+
Raises:
|
|
424
|
+
ValueError: If use_gathered_alerts is False and train_data or val_data is None.
|
|
425
|
+
"""
|
|
426
|
+
if use_gathered_alerts:
|
|
427
|
+
train_plus_val_size = self.config.train_size + self.config.val_size
|
|
428
|
+
train_split = self.config.train_size / train_plus_val_size
|
|
429
|
+
val_split = self.config.val_size / train_plus_val_size
|
|
430
|
+
train_data, val_data, _ = split_alert_dataset(
|
|
431
|
+
AlertDataset(
|
|
432
|
+
self.collected_data[self.config.skip_first:],
|
|
433
|
+
),
|
|
434
|
+
train_val_test_split=(train_split, val_split, 0.0),
|
|
435
|
+
)
|
|
436
|
+
else:
|
|
437
|
+
if train_data is None or val_data is None:
|
|
438
|
+
raise ValueError(
|
|
439
|
+
"train_data and val_data must be provided if use_gathered_alerts is False."
|
|
440
|
+
)
|
|
441
|
+
logger.info("Training AlertBERT model ...")
|
|
442
|
+
self.mlm.fit(train_data, val_data)
|
|
443
|
+
self.config.train_model = False
|
|
444
|
+
logger.info("AlertBERT model training complete.")
|
|
445
|
+
self._set_up_alertbert()
|
|
446
|
+
if self.config.train_model:
|
|
447
|
+
raise ValueError("The model should not be trained after training!")
|
|
@@ -38,7 +38,7 @@ class TimeDeltaAggregation(DensityClusteringAggregation):
|
|
|
38
38
|
|
|
39
39
|
def __init__(
|
|
40
40
|
self,
|
|
41
|
-
config: TimeDeltaAggregationConfig | dict[str, Any],
|
|
41
|
+
config: TimeDeltaAggregationConfig | dict[str, Any] = TimeDeltaAggregationConfig(),
|
|
42
42
|
name: str = "TimeDeltaAggregator",
|
|
43
43
|
) -> None:
|
|
44
44
|
"""Initialize the TimeDeltaAggregation class.
|
|
@@ -60,6 +60,12 @@ class CoreConfig(BasicConfig):
|
|
|
60
60
|
"removed after training reads it; a killed process leaves it behind."
|
|
61
61
|
),
|
|
62
62
|
)
|
|
63
|
+
connector_name: str | None = Field(
|
|
64
|
+
default=None, description="Add a connector to the output schema.",
|
|
65
|
+
)
|
|
66
|
+
connector_url: str | None = Field(
|
|
67
|
+
default=None, description="Set up the URL of the connector for the output schema.",
|
|
68
|
+
)
|
|
63
69
|
|
|
64
70
|
|
|
65
71
|
class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
@@ -67,7 +73,7 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
|
67
73
|
|
|
68
74
|
def __init__(
|
|
69
75
|
self,
|
|
70
|
-
name: str,
|
|
76
|
+
name: str = "Component",
|
|
71
77
|
type_: str = "Core",
|
|
72
78
|
config: CoreConfig | dict[str, Any] = CoreConfig(),
|
|
73
79
|
args_buffer: ArgsBuffer = ArgsBuffer(BufferMode.NO_BUF),
|
|
@@ -172,7 +178,7 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
|
172
178
|
logger.debug(f"<<{self.name}>> finalizing training")
|
|
173
179
|
self.post_train()
|
|
174
180
|
|
|
175
|
-
output_ = self.output_schema()
|
|
181
|
+
output_ = self.output_schema(connector=self.config.connector_name, url=self.config.connector_url)
|
|
176
182
|
logger.debug(f"<<{self.name}>> processing data")
|
|
177
183
|
return_schema = self.run(input_=data_buffered, output_=output_)
|
|
178
184
|
if not return_schema:
|
|
@@ -180,4 +186,4 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
|
180
186
|
return None
|
|
181
187
|
|
|
182
188
|
logger.debug(f"<<{self.name}>> processed:\n{output_}")
|
|
183
|
-
return SchemaPipeline.postprocess(output_, is_byte=is_byte)
|
|
189
|
+
return SchemaPipeline.postprocess(output_.store(), is_byte=is_byte)
|