detectmatelibrary 0.6.0__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- detectmatelibrary-0.6.2/PKG-INFO +74 -0
- detectmatelibrary-0.6.2/README.md +33 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/__init__.py +2 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +27 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/dummy_detector.py +5 -3
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/dummy_parser.py +1 -1
- detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/__init__.py +12 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/alert_aggregation/basic_concat.py +10 -5
- detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/logid_count.py +96 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/time_delta.py +111 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_alert_aggregation/__init__.py +9 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +429 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/__init__.py +30 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_persist.py +83 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_stability.py +77 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_variable_hooks.py +62 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/common → detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors}/deeplearning_detector.py +2 -2
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/tracker_detector.py +203 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/variable_detector.py +221 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/_compile.py +65 -4
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_basic_component.py +1 -4
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_fed_component.py +17 -6
- detectmatelibrary-0.6.2/src/detectmatelibrary/common/_core_op/_train_buffer.py +122 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/alert_aggregator.py +25 -15
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/core.py +62 -41
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/detector.py +11 -25
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/parser.py +5 -5
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +13 -19
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/charset_detector.py +2 -5
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/deeplog_detector.py +1 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/ecvc_detector.py +15 -28
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/event_sequence_detector.py +25 -46
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/logbert_detector.py +1 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_event_detector.py +15 -22
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_value_combo_detector.py +60 -29
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_value_detector.py +2 -5
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/random_detector.py +8 -8
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/rule_detector.py +3 -3
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/scvs_detector.py +9 -18
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/value_range_detector.py +2 -5
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/helper/from_to.py +3 -3
- detectmatelibrary-0.6.2/src/detectmatelibrary/helper/select_component.py +74 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/metadata.py +2 -2
- detectmatelibrary-0.6.2/src/detectmatelibrary/parsers/__init__.py +23 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/autoparser.py +2 -2
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/drain.py +2 -2
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/json_parser.py +20 -6
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +1 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/parser.py +1 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/_parser.py +1 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/tree_matcher.py +1 -1
- detectmatelibrary-0.6.2/src/detectmatelibrary/pipeline.py +174 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/__init__.py +4 -1
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/_classes.py +13 -6
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/_op.py +25 -9
- detectmatelibrary-0.6.2/src/detectmatelibrary/schemas/schemas_pb2.py +45 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/create_workspace.py +9 -7
- detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +36 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +1 -8
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +1 -6
- detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +194 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +12 -12
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +36 -36
- detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +50 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +46 -0
- detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +46 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/data_normalizer.py +3 -3
- detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/__init__.py +23 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/basic_persistency.py +53 -30
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures/trackers}/rle_list.py +1 -1
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/stability_classifier.py +1 -1
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/stability_tracker.py +4 -7
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/event_persistency.py +13 -11
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/persistency_saver.py +10 -28
- detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/slow_persistency.py +137 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/sequence_encoding.py +6 -7
- detectmatelibrary-0.6.2/src/detectmatelibrary.egg-info/PKG-INFO +74 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/SOURCES.txt +31 -19
- detectmatelibrary-0.6.0/PKG-INFO +0 -156
- detectmatelibrary-0.6.0/README.md +0 -115
- detectmatelibrary-0.6.0/src/detectmatelibrary/common/_other_op/_variable_hooks.py +0 -237
- detectmatelibrary-0.6.0/src/detectmatelibrary/common/persist.py +0 -32
- detectmatelibrary-0.6.0/src/detectmatelibrary/common/variable_detector.py +0 -150
- detectmatelibrary-0.6.0/src/detectmatelibrary/schemas/schemas_pb2.py +0 -45
- detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
- detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -37
- detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -32
- detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/__init__.py +0 -44
- detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/component_interfaces.py +0 -148
- detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/__init__.py +0 -4
- detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/chunked_event_dataframe.py +0 -122
- detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/event_dataframe.py +0 -57
- detectmatelibrary-0.6.0/src/detectmatelibrary.egg-info/PKG-INFO +0 -156
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/LICENSE.md +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/pyproject.toml +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/setup.cfg +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/_formats.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/constants.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/parsers → detectmatelibrary-0.6.2/src/detectmatelibrary/tools}/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/logging.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/tools → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace}/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates}/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/data}/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/data → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates}/__init__.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/aux.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/data_buffer.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/_op.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/deeplog.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/logbert.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/finetune.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/id_generator.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/key_extractor.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/base.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/event_tracker.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/multi_tracker.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/single_tracker.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/__init__.py +0 -0
- {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/classification_methods.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/requires.txt +0 -0
- {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: detectmatelibrary
|
|
3
|
+
Version: 0.6.2
|
|
4
|
+
Summary: DetectMate Library for log processing components
|
|
5
|
+
Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
|
|
6
|
+
License-Expression: EUPL-1.2
|
|
7
|
+
Project-URL: Homepage, https://ait-detectmate.github.io/DetectMateLibrary/latest
|
|
8
|
+
Project-URL: Repository, https://github.com/ait-detectmate/DetectMateLibrary
|
|
9
|
+
Requires-Python: >=3.12
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE.md
|
|
12
|
+
Requires-Dist: protobuf>=6.32.1
|
|
13
|
+
Requires-Dist: pydantic>=2.13.3
|
|
14
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
15
|
+
Requires-Dist: regex>=2025.11.3
|
|
16
|
+
Requires-Dist: numpy>=2.3.2
|
|
17
|
+
Requires-Dist: detectmateperformance>=0.1.6
|
|
18
|
+
Requires-Dist: msgpack>=1.0.0
|
|
19
|
+
Requires-Dist: fsspec>=2024.1.0
|
|
20
|
+
Requires-Dist: pyarrow>=24.0.0
|
|
21
|
+
Requires-Dist: jax>=0.11.0
|
|
22
|
+
Requires-Dist: flax>=0.12.8
|
|
23
|
+
Requires-Dist: optax>=0.2.8
|
|
24
|
+
Provides-Extra: llm
|
|
25
|
+
Requires-Dist: openai>=2.26.0; extra == "llm"
|
|
26
|
+
Requires-Dist: tenacity>=9.1.4; extra == "llm"
|
|
27
|
+
Requires-Dist: scipy>=1.17.1; extra == "llm"
|
|
28
|
+
Requires-Dist: scikit-learn>=1.8.0; extra == "llm"
|
|
29
|
+
Requires-Dist: tiktoken>=0.12.0; extra == "llm"
|
|
30
|
+
Requires-Dist: pandas>=2.3.2; extra == "llm"
|
|
31
|
+
Provides-Extra: dataframes
|
|
32
|
+
Requires-Dist: pandas>=2.3.2; extra == "dataframes"
|
|
33
|
+
Requires-Dist: polars>=1.40.1; extra == "dataframes"
|
|
34
|
+
Provides-Extra: polars-rtcompat
|
|
35
|
+
Requires-Dist: polars[rtcompat]>=1.38.1; extra == "polars-rtcompat"
|
|
36
|
+
Provides-Extra: full
|
|
37
|
+
Requires-Dist: detectmatelibrary[llm]; extra == "full"
|
|
38
|
+
Requires-Dist: detectmatelibrary[dataframes]; extra == "full"
|
|
39
|
+
Requires-Dist: detectmatelibrary[polars-rtcompat]; extra == "full"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# DetectMateLibrary
|
|
44
|
+
|
|
45
|
+

|
|
46
|
+
|
|
47
|
+
DetectMate is a flexible, component-based library for parsing and log anomaly detection.
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
Download us from `Pip`!!
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install detectmatelibrary
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Documentation
|
|
58
|
+
|
|
59
|
+
Main documentations:
|
|
60
|
+
|
|
61
|
+
- [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
|
|
62
|
+
- [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
|
|
63
|
+
|
|
64
|
+
## Contribution
|
|
65
|
+
|
|
66
|
+
We're happily taking patches and other contributions. Please see the following links for how to get started:
|
|
67
|
+
|
|
68
|
+
- [GitHub Workflow](https://ait-detectmate.github.io/DetectMateLibrary/latest/contribution/)
|
|
69
|
+
|
|
70
|
+
If you encounter any bugs, please create an issue on [Github](https://github.com/ait-detectmate/DetectMateLibrary/issues).
|
|
71
|
+
|
|
72
|
+
## License
|
|
73
|
+
|
|
74
|
+
[EUPL-1.2](https://github.com/ait-detectmate/DetectMateLibrary/blob/main/LICENSE.md)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
|
|
2
|
+
# DetectMateLibrary
|
|
3
|
+
|
|
4
|
+

|
|
5
|
+
|
|
6
|
+
DetectMate is a flexible, component-based library for parsing and log anomaly detection.
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
Download us from `Pip`!!
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install detectmatelibrary
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Documentation
|
|
17
|
+
|
|
18
|
+
Main documentations:
|
|
19
|
+
|
|
20
|
+
- [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
|
|
21
|
+
- [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
|
|
22
|
+
|
|
23
|
+
## Contribution
|
|
24
|
+
|
|
25
|
+
We're happily taking patches and other contributions. Please see the following links for how to get started:
|
|
26
|
+
|
|
27
|
+
- [GitHub Workflow](https://ait-detectmate.github.io/DetectMateLibrary/latest/contribution/)
|
|
28
|
+
|
|
29
|
+
If you encounter any bugs, please create an issue on [Github](https://github.com/ait-detectmate/DetectMateLibrary/issues).
|
|
30
|
+
|
|
31
|
+
## License
|
|
32
|
+
|
|
33
|
+
[EUPL-1.2](https://github.com/ait-detectmate/DetectMateLibrary/blob/main/LICENSE.md)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from . import common
|
|
2
|
+
from . import alert_aggregation
|
|
2
3
|
from . import detectors
|
|
3
4
|
from . import parsers
|
|
4
5
|
from . import schemas
|
|
@@ -10,6 +11,7 @@ __all__ = [
|
|
|
10
11
|
"common",
|
|
11
12
|
"detectors",
|
|
12
13
|
"parsers",
|
|
14
|
+
"alert_aggregation",
|
|
13
15
|
"schemas",
|
|
14
16
|
"utils",
|
|
15
17
|
"__authors__",
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from detectmatelibrary.common.alert_aggregator import (CoreAlertAggregatorConfig, CoreAlertAggregator)
|
|
2
|
+
from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DummyAlertAggregatorConfig(CoreAlertAggregatorConfig):
|
|
7
|
+
method_type: str = "dummy_alert_aggregator"
|
|
8
|
+
buffer_size: int = 3
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DummyAlertAggregator(CoreAlertAggregator):
|
|
12
|
+
def __init__(
|
|
13
|
+
self,
|
|
14
|
+
name: str = "DummyAlertAggregator",
|
|
15
|
+
config: DummyAlertAggregatorConfig | dict[str, Any] = DummyAlertAggregatorConfig(),
|
|
16
|
+
) -> None:
|
|
17
|
+
if isinstance(config, dict):
|
|
18
|
+
config = DummyAlertAggregatorConfig.from_dict(config, name)
|
|
19
|
+
buffer_size: int = config.buffer_size
|
|
20
|
+
super().__init__(name=name, buffer_size=buffer_size, config=config)
|
|
21
|
+
|
|
22
|
+
def aggregate_alerts(
|
|
23
|
+
self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
|
|
24
|
+
) -> bool:
|
|
25
|
+
output_["description"] = "Dummy alert aggregation"
|
|
26
|
+
output_["alertDetails"]["type"] = "Anomalies aggregated by DummyAlertAggregator"
|
|
27
|
+
return True
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
|
|
2
2
|
from detectmatelibrary.utils.data_buffer import BufferMode
|
|
3
3
|
from detectmatelibrary import schemas
|
|
4
|
-
from typing import List, Any
|
|
4
|
+
from typing import List, Any, Optional
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class DummyDetectorConfig(CoreDetectorConfig):
|
|
@@ -15,12 +15,14 @@ class DummyDetector(CoreDetector):
|
|
|
15
15
|
def __init__(
|
|
16
16
|
self,
|
|
17
17
|
name: str = "DummyDetector",
|
|
18
|
+
buffer_mode: BufferMode = BufferMode.NO_BUF,
|
|
19
|
+
buffer_size: Optional[int] = None,
|
|
18
20
|
config: DummyDetectorConfig | dict[str, Any] = DummyDetectorConfig()
|
|
19
21
|
) -> None:
|
|
20
22
|
|
|
21
23
|
if isinstance(config, dict):
|
|
22
24
|
config = DummyDetectorConfig.from_dict(config, name)
|
|
23
|
-
super().__init__(name=name, buffer_mode=
|
|
25
|
+
super().__init__(name=name, buffer_mode=buffer_mode, buffer_size=buffer_size, config=config)
|
|
24
26
|
self._call_count = 0
|
|
25
27
|
|
|
26
28
|
def detect(
|
|
@@ -36,5 +38,5 @@ class DummyDetector(CoreDetector):
|
|
|
36
38
|
result = pattern[self._call_count % len(pattern)]
|
|
37
39
|
if result:
|
|
38
40
|
output_["score"] = 1.0
|
|
39
|
-
output_["
|
|
41
|
+
output_["alertDetails"]["type"] = "Anomaly detected by DummyDetector"
|
|
40
42
|
return result
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig
|
|
2
|
+
from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig
|
|
3
|
+
from .time_delta import TimeDeltaAggregation, TimeDeltaAggregationConfig
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"BasicConcatAggregation",
|
|
7
|
+
"BasicConcatAggregationConfig",
|
|
8
|
+
"TimeDeltaAggregation",
|
|
9
|
+
"TimeDeltaAggregationConfig",
|
|
10
|
+
"LogIDCountAggegation",
|
|
11
|
+
"LogIDCountAggegationConfig",
|
|
12
|
+
]
|
|
@@ -4,24 +4,29 @@ from detectmatelibrary.common.alert_aggregator import (
|
|
|
4
4
|
|
|
5
5
|
from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
|
|
6
6
|
|
|
7
|
-
from typing import Any
|
|
7
|
+
from typing import Any
|
|
8
|
+
from pydantic import Field
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class BasicConcatAggregationConfig(CoreAlertAggregatorConfig):
|
|
11
|
-
method_type: str =
|
|
12
|
-
|
|
12
|
+
method_type: str = Field(
|
|
13
|
+
default="basic_concat_aggregator", description="Indicates what type of method it is."
|
|
14
|
+
)
|
|
15
|
+
buffer_size: int = Field(
|
|
16
|
+
default=3, description="Buffer size use in the aggregation."
|
|
17
|
+
)
|
|
13
18
|
|
|
14
19
|
|
|
15
20
|
class BasicConcatAggregation(CoreAlertAggregator):
|
|
16
21
|
def __init__(
|
|
17
22
|
self,
|
|
18
23
|
name: str = "BasicConcatAggregator",
|
|
19
|
-
config:
|
|
24
|
+
config: BasicConcatAggregationConfig | dict[str, Any] = BasicConcatAggregationConfig(),
|
|
20
25
|
) -> None:
|
|
21
26
|
if isinstance(config, dict):
|
|
22
27
|
config = BasicConcatAggregationConfig.from_dict(config, name)
|
|
23
28
|
|
|
24
|
-
buffer_size: int = config.buffer_size
|
|
29
|
+
buffer_size: int = config.buffer_size
|
|
25
30
|
super().__init__(name=name, buffer_size=buffer_size, config=config)
|
|
26
31
|
|
|
27
32
|
def aggregate_alerts(
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from detectmatelibrary.common.alert_aggregator import (
|
|
2
|
+
CoreAlertAggregatorConfig, CoreAlertAggregator
|
|
3
|
+
)
|
|
4
|
+
|
|
5
|
+
from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
|
|
6
|
+
from detectmatelibrary.utils.data_buffer import BufferMode
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
from functools import lru_cache
|
|
10
|
+
from pydantic import Field
|
|
11
|
+
import polars as pl
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Counter:
|
|
15
|
+
def __init__(self, max_count: int, overall_size: int) -> None:
|
|
16
|
+
self.max_count = max_count
|
|
17
|
+
|
|
18
|
+
self._df = pl.DataFrame()
|
|
19
|
+
self._df = self._df.insert_column(0, pl.repeat("", n=overall_size).alias("LogID"))
|
|
20
|
+
self._df = self._df.insert_column(1, pl.zeros(overall_size, dtype=pl.UInt8).alias("Count"))
|
|
21
|
+
self._df = self._df.insert_column(2, pl.zeros(overall_size, dtype=pl.Boolean).alias("WasAlert"))
|
|
22
|
+
self._df = self._df.with_row_index(name="Index")
|
|
23
|
+
|
|
24
|
+
self._cached_search = lru_cache(maxsize=overall_size)(self.__get_idx)
|
|
25
|
+
|
|
26
|
+
self.idx = 0
|
|
27
|
+
|
|
28
|
+
def __is_new_alert(self, idx: int) -> bool:
|
|
29
|
+
if not self._df[idx, "WasAlert"] and self._df[idx, "Count"] >= self.max_count:
|
|
30
|
+
self._df[idx, "WasAlert"] = True
|
|
31
|
+
return True
|
|
32
|
+
return False
|
|
33
|
+
|
|
34
|
+
def __get_idx(self, logId: str) -> int:
|
|
35
|
+
indices = self._df.filter(pl.col("LogID") == logId).select("Index").to_series()
|
|
36
|
+
|
|
37
|
+
if indices.shape[0] == 0:
|
|
38
|
+
new_row = pl.DataFrame(
|
|
39
|
+
data=[[self.idx, logId, 0, False]], schema=self._df.schema, orient="row"
|
|
40
|
+
)
|
|
41
|
+
self._df = self._df.update(new_row, on="Index")
|
|
42
|
+
idx = self.idx
|
|
43
|
+
self.idx = self.idx + 1 if self.idx < self._df.shape[0] - 1 else 0
|
|
44
|
+
else:
|
|
45
|
+
idx = indices[0]
|
|
46
|
+
|
|
47
|
+
return idx
|
|
48
|
+
|
|
49
|
+
def add_and_check(self, logId: str) -> bool:
|
|
50
|
+
idx = self._cached_search(logId)
|
|
51
|
+
self._df[idx, "Count"] += 1
|
|
52
|
+
return self.__is_new_alert(idx)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class LogIDCountAggegationConfig(CoreAlertAggregatorConfig):
|
|
56
|
+
method_type: str = Field(
|
|
57
|
+
default="logid_count_aggregator", description="Indicates what type of method it is."
|
|
58
|
+
)
|
|
59
|
+
max_count: int = Field(default=4, description="Max count of alerts in a specific logID.")
|
|
60
|
+
max_memory: int = Field(
|
|
61
|
+
default=5000, description="Max number of logIDs store in the count memory."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class LogIDCountAggegation(CoreAlertAggregator):
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
name: str = "LogIDCountAggregator",
|
|
69
|
+
config: LogIDCountAggegationConfig | dict[str, Any] = LogIDCountAggegationConfig(),
|
|
70
|
+
) -> None:
|
|
71
|
+
if isinstance(config, dict):
|
|
72
|
+
config = LogIDCountAggegationConfig.from_dict(config, name)
|
|
73
|
+
|
|
74
|
+
super().__init__(name=name, config=config, buffer_mode=BufferMode.NO_BUF, buffer_size=None)
|
|
75
|
+
self.config: LogIDCountAggegationConfig
|
|
76
|
+
|
|
77
|
+
self.counter = Counter(
|
|
78
|
+
max_count=self.config.max_count, overall_size=self.config.max_memory
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def aggregate_alerts(
|
|
82
|
+
self, input_: DetectorSchema, output_: AggregateSchema # type: ignore
|
|
83
|
+
) -> bool:
|
|
84
|
+
|
|
85
|
+
alert_logs = []
|
|
86
|
+
for logID in input_["logIDs"]:
|
|
87
|
+
if self.counter.add_and_check(logID):
|
|
88
|
+
alert_logs.append(logID)
|
|
89
|
+
|
|
90
|
+
if (aggreated_alerts := len(alert_logs) > 0):
|
|
91
|
+
output_["description"] = (
|
|
92
|
+
"LogID count how many times a log was count as alert."
|
|
93
|
+
f"Logs that pass the limit count: {", ".join(alert_logs)}"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
return aggreated_alerts
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from pydantic import Field, NonNegativeFloat
|
|
5
|
+
|
|
6
|
+
from detectmatelibrary.base_alert_aggregation.density_clustering import (
|
|
7
|
+
DensityClusteringAggregation,
|
|
8
|
+
DensityClusteringAggregationConfig,
|
|
9
|
+
)
|
|
10
|
+
from detectmatelibrary.schemas import AggregateSchema, DetectorSchema
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TimeDeltaAggregationConfig(DensityClusteringAggregationConfig):
|
|
14
|
+
"""Configuration of the TimeDeltaAggregation class.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
delta (NonNegativeFloat): The time threshold below which two alerts are allocated to the same group.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
method_type: str = Field(
|
|
21
|
+
default="time_delta_aggregator",
|
|
22
|
+
description="The kind of aggregation method.",
|
|
23
|
+
)
|
|
24
|
+
delta: NonNegativeFloat = Field(
|
|
25
|
+
default=10.0, description="The threshold for the clustering."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class TimeDeltaAggregation(DensityClusteringAggregation):
|
|
30
|
+
"""This class implements alert aggregation by the time-delta method introduced by
|
|
31
|
+
[Landauer et al.](https://doi.org/10.1145/3510581).
|
|
32
|
+
Two alerts will be assigned to the same group if the time difference
|
|
33
|
+
between them is below a certain threshold delta.
|
|
34
|
+
The time difference is computed based on the timestamps of the last logs associated
|
|
35
|
+
with each alert as these determine the earliest time at which the alert was detected.
|
|
36
|
+
Important: It is assumed that the alerts arrive in order of their timestamps!
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
config: TimeDeltaAggregationConfig | dict[str, Any],
|
|
42
|
+
name: str = "TimeDeltaAggregator",
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Initialize the TimeDeltaAggregation class.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
name (str): The name identifying this aggregation method.
|
|
48
|
+
config (TimeDeltaAggregationConfig): The configuration of this method.
|
|
49
|
+
"""
|
|
50
|
+
if isinstance(config, dict):
|
|
51
|
+
config = TimeDeltaAggregationConfig.from_dict(config, name)
|
|
52
|
+
|
|
53
|
+
super().__init__(name=name, config=config)
|
|
54
|
+
self.timestamp_extraction_fn: Callable[[DetectorSchema], float] = lambda alert: (
|
|
55
|
+
float(alert["logTimestamps"][-1])
|
|
56
|
+
)
|
|
57
|
+
self.description = "Alert aggregation by the time-delta method."
|
|
58
|
+
self.last_timestamp: float = 0.0
|
|
59
|
+
|
|
60
|
+
def aggregate_alerts(
|
|
61
|
+
self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
|
|
62
|
+
) -> bool:
|
|
63
|
+
"""Aggregate input alerts and write the results to the output.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
input_: The alert or alerts to aggregate.
|
|
67
|
+
output_: The schema to which aggregation results are written.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
``True`` after the aggregation results have been written.
|
|
71
|
+
``False`` if no new alerts were aggregated and the output schema was not modified.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
if not isinstance(input_, list):
|
|
75
|
+
input_ = [input_]
|
|
76
|
+
|
|
77
|
+
if self.timestamp_extraction_fn(input_[0]) < self.last_timestamp:
|
|
78
|
+
raise ValueError("Alerts must be aggregated in order of their timestamps!")
|
|
79
|
+
else:
|
|
80
|
+
self.last_timestamp = self.timestamp_extraction_fn(input_[-1])
|
|
81
|
+
|
|
82
|
+
return super().aggregate_alerts(input_, output_)
|
|
83
|
+
|
|
84
|
+
def single_metric(
|
|
85
|
+
self,
|
|
86
|
+
alert1: DetectorSchema,
|
|
87
|
+
alert2: DetectorSchema,
|
|
88
|
+
) -> NonNegativeFloat:
|
|
89
|
+
"""Compute the time difference between two alerts.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
alert1 (DetectorSchema): The first alert.
|
|
93
|
+
alert2 (DetectorSchema): The second alert.
|
|
94
|
+
Returns:
|
|
95
|
+
NonNegativeFloat: The time difference between the two alerts.
|
|
96
|
+
"""
|
|
97
|
+
return abs(
|
|
98
|
+
self.timestamp_extraction_fn(alert1) - self.timestamp_extraction_fn(alert2)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def discard_alert_fn(self, alerts: list[DetectorSchema]) -> list[bool]:
|
|
102
|
+
"""Discards all but the last alert in a list of alerts as this is the
|
|
103
|
+
only one relevant for the time-delta method.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
alerts (list[DetectorSchema]): The list of alerts to discard from.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
list[bool]: A list of booleans indicating which alerts to discard.
|
|
110
|
+
"""
|
|
111
|
+
return [i + 1 != len(alerts) for i in range(len(alerts))]
|