detectmatelibrary 0.7.0__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectmatelibrary-0.7.0/src/detectmatelibrary.egg-info → detectmatelibrary-0.7.2}/PKG-INFO +3 -3
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/README.md +1 -1
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/pyproject.toml +1 -1
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/__init__.py +4 -0
- detectmatelibrary-0.7.2/src/detectmatelibrary/base_detectors/count_vector_detector.py +113 -0
- detectmatelibrary-0.7.2/src/detectmatelibrary/base_detectors/deeplearning_detector.py +164 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/alert_aggregator.py +7 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/core.py +9 -3
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/detector.py +21 -20
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/parser.py +43 -17
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/deeplog_detector.py +1 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/ecvc_detector.py +29 -31
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/event_sequence_detector.py +98 -23
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/logbert_detector.py +1 -0
- detectmatelibrary-0.7.2/src/detectmatelibrary/detectors/scvs_detector.py +41 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/helper/from_to.py +155 -37
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/metadata.py +1 -1
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/__init__.py +28 -16
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/pipeline.py +17 -13
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/_op.py +33 -1
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/deeplog.py +30 -17
- detectmatelibrary-0.7.2/src/detectmatelibrary/utils/deep_learning/imodel.py +19 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/logbert.py +51 -26
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/sequence_encoding.py +6 -2
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2/src/detectmatelibrary.egg-info}/PKG-INFO +3 -3
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/SOURCES.txt +1 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/requires.txt +1 -1
- detectmatelibrary-0.7.0/src/detectmatelibrary/base_detectors/deeplearning_detector.py +0 -107
- detectmatelibrary-0.7.0/src/detectmatelibrary/detectors/scvs_detector.py +0 -71
- detectmatelibrary-0.7.0/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -19
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/LICENSE.md +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/setup.cfg +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_parser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/alertbert.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/basic_concat.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/logid_count.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/time_delta.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_alert_aggregation/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_persist.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_stability.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_variable_hooks.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/tracker_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/variable_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/_compile.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/_formats.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_basic_component.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_fed_component.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_train_buffer.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/constants.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/charset_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_event_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_value_combo_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_value_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/random_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/rule_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/value_range_detector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/helper/select_component.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/autoparser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/drain.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/json_parser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/parser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/_parser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/tree_matcher.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/_classes.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/default.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/mongos.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/redis.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/protobuf/_op.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/protobuf/schemas_pb2.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/logging.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/create_workspace.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/datasets.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/models.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/preprocessing.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/aux.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/data_buffer.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/finetune.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/id_generator.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/key_extractor.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/basic_persistency.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/base.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/event_tracker.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/multi_tracker.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/single_tracker.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/rle_list.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/__init__.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/classification_methods.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_classifier.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_tracker.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/event_persistency.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/persistency_saver.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/slow_persistency.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
- {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: detectmatelibrary
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: DetectMate Library for log processing components
|
|
5
5
|
Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf, Lukas Karner
|
|
6
6
|
License-Expression: EUPL-1.2
|
|
@@ -14,7 +14,7 @@ Requires-Dist: pydantic>=2.13.3
|
|
|
14
14
|
Requires-Dist: pyyaml>=6.0.3
|
|
15
15
|
Requires-Dist: regex>=2025.11.3
|
|
16
16
|
Requires-Dist: numpy>=2.3.2
|
|
17
|
-
Requires-Dist: detectmateperformance>=0.
|
|
17
|
+
Requires-Dist: detectmateperformance>=0.2.0
|
|
18
18
|
Requires-Dist: msgpack>=1.0.0
|
|
19
19
|
Requires-Dist: fsspec>=2024.1.0
|
|
20
20
|
Requires-Dist: pyarrow>=24.0.0
|
|
@@ -62,7 +62,7 @@ pip install detectmatelibrary
|
|
|
62
62
|
Main documentations:
|
|
63
63
|
|
|
64
64
|
- [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
|
|
65
|
-
- [Service Documentation](https://
|
|
65
|
+
- [Service Documentation](https://ait-detectmate.github.io/DetectMateService/latest/)
|
|
66
66
|
|
|
67
67
|
## Contribution
|
|
68
68
|
|
|
@@ -18,7 +18,7 @@ pip install detectmatelibrary
|
|
|
18
18
|
Main documentations:
|
|
19
19
|
|
|
20
20
|
- [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
|
|
21
|
-
- [Service Documentation](https://
|
|
21
|
+
- [Service Documentation](https://ait-detectmate.github.io/DetectMateService/latest/)
|
|
22
22
|
|
|
23
23
|
## Contribution
|
|
24
24
|
|
{detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/__init__.py
RENAMED
|
@@ -9,6 +9,7 @@ from detectmatelibrary.utils.persistency.data_structures.trackers.stability.stab
|
|
|
9
9
|
SingleStabilityTracker,
|
|
10
10
|
)
|
|
11
11
|
|
|
12
|
+
from .count_vector_detector import CountVectorDetector, CountVectorDetectorConfig, SessionWindows
|
|
12
13
|
from ._persist import PersistConfig
|
|
13
14
|
from .deeplearning_detector import DeepLearningDetector, DeepLearningDetectorConfig
|
|
14
15
|
from ._stability import StabilityAutoConfigParams
|
|
@@ -16,10 +17,13 @@ from .tracker_detector import TrackerDetector, TrackerDetectorConfig
|
|
|
16
17
|
from .variable_detector import VariableAutoConfigParams, VariableDetector, VariableDetectorConfig
|
|
17
18
|
|
|
18
19
|
__all__ = [
|
|
20
|
+
"CountVectorDetector",
|
|
21
|
+
"CountVectorDetectorConfig",
|
|
19
22
|
"DeepLearningDetector",
|
|
20
23
|
"DeepLearningDetectorConfig",
|
|
21
24
|
"EventStabilityTracker",
|
|
22
25
|
"PersistConfig",
|
|
26
|
+
"SessionWindows",
|
|
23
27
|
"SingleStabilityTracker",
|
|
24
28
|
"StabilityAutoConfigParams",
|
|
25
29
|
"TrackerDetector",
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
from typing import List, Sequence
|
|
2
|
+
from typing_extensions import override
|
|
3
|
+
|
|
4
|
+
from detectmatelibrary import schemas
|
|
5
|
+
from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode, DataBuffer
|
|
6
|
+
from detectmatelibrary.utils.sequence_encoding import (
|
|
7
|
+
count_events,
|
|
8
|
+
decode_count_vec,
|
|
9
|
+
encode_count_vec,
|
|
10
|
+
warn_on_window_size_mismatch,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
from pydantic import Field
|
|
14
|
+
|
|
15
|
+
from .tracker_detector import TrackerDetector, TrackerDetectorConfig
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SessionWindows:
|
|
19
|
+
"""The window ending at the latest event of each session: its last `size`
|
|
20
|
+
events, fewer at the session start."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, size: int) -> None:
|
|
23
|
+
self.size = size
|
|
24
|
+
self.windows: dict[str, tuple[int, ...]] = {}
|
|
25
|
+
|
|
26
|
+
def add(self, session: str, event: int) -> tuple[int, ...]:
|
|
27
|
+
"""Add ``event`` to ``session`` and return the session's window ending
|
|
28
|
+
at it."""
|
|
29
|
+
window = (self.windows.get(session, ()) + (event,))[-self.size:]
|
|
30
|
+
self.windows[session] = window
|
|
31
|
+
return window
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CountVectorDetectorConfig(TrackerDetectorConfig):
|
|
35
|
+
window_size: int = Field(
|
|
36
|
+
default=10,
|
|
37
|
+
description="Length of the event-ID window a count vector is built over.",
|
|
38
|
+
)
|
|
39
|
+
session_variable: str | None = Field(
|
|
40
|
+
default=None,
|
|
41
|
+
description=(
|
|
42
|
+
"Header variable that names the session of each event (HDFS: block_id). When set, "
|
|
43
|
+
"an event's window holds the last window_size events of its own session, fewer at "
|
|
44
|
+
"the session start, so windows never mix sessions and every event is checked."
|
|
45
|
+
),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class CountVectorDetector(TrackerDetector):
|
|
50
|
+
"""Base of the detectors that learn the count vectors of event-ID
|
|
51
|
+
windows."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, name: str, config: CountVectorDetectorConfig) -> None:
|
|
54
|
+
self.config: CountVectorDetectorConfig
|
|
55
|
+
|
|
56
|
+
by_session = config.session_variable is not None
|
|
57
|
+
super().__init__(
|
|
58
|
+
name=name,
|
|
59
|
+
config=config,
|
|
60
|
+
buffer_mode=BufferMode.NO_BUF if by_session else BufferMode.WINDOW,
|
|
61
|
+
buffer_size=None if by_session else config.window_size,
|
|
62
|
+
)
|
|
63
|
+
# With sessions, train() keeps its own windows: it also gets the configure records,
|
|
64
|
+
# replayed after the configure phase. detect() does not get them, so its windows are
|
|
65
|
+
# built where every record passes once, the buffer.
|
|
66
|
+
self.train_windows = SessionWindows(config.window_size)
|
|
67
|
+
self.detect_windows = SessionWindows(config.window_size)
|
|
68
|
+
self.window: tuple[int, ...] = ()
|
|
69
|
+
if by_session:
|
|
70
|
+
self.args_buffer = ArgsBuffer(BufferMode.NO_BUF, process_function=self._add_to_session)
|
|
71
|
+
self.data_buffer = DataBuffer(self.args_buffer)
|
|
72
|
+
|
|
73
|
+
def _session(self, input_: schemas.ParserSchema) -> str:
|
|
74
|
+
session = input_["headerVariables"].get(self.config.session_variable)
|
|
75
|
+
if session is None:
|
|
76
|
+
raise ValueError(f"<<{self.name}>> log has no header variable {self.config.session_variable!r}")
|
|
77
|
+
return str(session)
|
|
78
|
+
|
|
79
|
+
def _add_to_session(self, input_: schemas.ParserSchema) -> schemas.ParserSchema:
|
|
80
|
+
self.window = self.detect_windows.add(self._session(input_), input_["eventID"])
|
|
81
|
+
return input_
|
|
82
|
+
|
|
83
|
+
def window_events(self, input_: List[schemas.ParserSchema] | schemas.ParserSchema) -> Sequence[int]:
|
|
84
|
+
"""Event IDs of the window of ``input_``."""
|
|
85
|
+
if isinstance(input_, list):
|
|
86
|
+
return [in_["eventID"] for in_ in input_]
|
|
87
|
+
return self.window
|
|
88
|
+
|
|
89
|
+
@override
|
|
90
|
+
def _sync_from_state(self) -> None:
|
|
91
|
+
warn_on_window_size_mismatch(self.name, self.persistency.get_events_seen(), self.config.window_size)
|
|
92
|
+
|
|
93
|
+
def train(self, input_: List[schemas.ParserSchema] | schemas.ParserSchema) -> None:
|
|
94
|
+
if isinstance(input_, list):
|
|
95
|
+
events: Sequence[int] = [in_["eventID"] for in_ in input_]
|
|
96
|
+
last = input_[-1]
|
|
97
|
+
else:
|
|
98
|
+
events, last = self.train_windows.add(self._session(input_), input_["eventID"]), input_
|
|
99
|
+
self._ingest(
|
|
100
|
+
event_id=encode_count_vec(self.config.window_size, count_events(events)),
|
|
101
|
+
input_=last,
|
|
102
|
+
variables={}
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def post_train(self) -> None:
|
|
106
|
+
self.train_windows = SessionWindows(self.config.window_size)
|
|
107
|
+
|
|
108
|
+
def get_known_count_vecs(self) -> set[tuple[int, ...]]:
|
|
109
|
+
"""Return the count vectors learned during training."""
|
|
110
|
+
return {
|
|
111
|
+
decode_count_vec(str(encoded))[1]
|
|
112
|
+
for encoded in self.persistency.get_events_seen()
|
|
113
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
|
|
2
|
+
from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
|
|
3
|
+
|
|
4
|
+
from detectmatelibrary.utils.deep_learning.imodel import DeepModel
|
|
5
|
+
from detectmatelibrary.utils.deep_learning._op import PAD
|
|
6
|
+
from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode, DataBuffer
|
|
7
|
+
|
|
8
|
+
from detectmatelibrary import schemas
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
from pydantic import Field
|
|
12
|
+
import logging
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DeepLearningDetectorConfig(CoreDetectorConfig):
|
|
16
|
+
window_size: int = Field(
|
|
17
|
+
default=10,
|
|
18
|
+
description="Number of consecutive events used as one training/detection sequence.",
|
|
19
|
+
)
|
|
20
|
+
validation_per: float = Field(
|
|
21
|
+
default=0.2,
|
|
22
|
+
description="Fraction of data held out for validation during (fine)training.",
|
|
23
|
+
)
|
|
24
|
+
finetune_epochs: int = Field(
|
|
25
|
+
default=2,
|
|
26
|
+
description="Number of epochs used when finetuning during the configuration phase.",
|
|
27
|
+
)
|
|
28
|
+
session_variable: str | None = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description=(
|
|
31
|
+
"Header variable that names the session of each event (HDFS: block_id). When set, "
|
|
32
|
+
"an event's window holds the events before it in its own session, padded at the "
|
|
33
|
+
"session start, so windows never mix sessions and every event is checked."
|
|
34
|
+
),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
hyperparameters: dict[str, Any] = Field(
|
|
38
|
+
default={
|
|
39
|
+
"Model": {
|
|
40
|
+
|
|
41
|
+
},
|
|
42
|
+
"Train": {
|
|
43
|
+
|
|
44
|
+
},
|
|
45
|
+
"Finetune": [],
|
|
46
|
+
},
|
|
47
|
+
description=(
|
|
48
|
+
"Model, training and hyperparameter-search settings passed to "
|
|
49
|
+
"the underlying deep learning model."
|
|
50
|
+
),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def build_seq(input_: list[schemas.ParserSchema]) -> tuple[int, ...]:
|
|
55
|
+
return tuple([in_["eventID"] for in_ in input_])
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class SessionWindows:
|
|
59
|
+
"""The window ending at the latest event of each session, padded with PAD
|
|
60
|
+
at its start."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, size: int) -> None:
|
|
63
|
+
self.size = size
|
|
64
|
+
self.windows: dict[str, tuple[int, ...]] = {}
|
|
65
|
+
|
|
66
|
+
def add(self, session: str, event: int) -> tuple[int, ...]:
|
|
67
|
+
"""Add ``event`` to ``session`` and return the session's window ending
|
|
68
|
+
at it."""
|
|
69
|
+
window = self.windows.get(session, (PAD,) * self.size)[1:] + (event,)
|
|
70
|
+
self.windows[session] = window
|
|
71
|
+
return window
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class DeepLearningDetector(CoreDetector):
|
|
75
|
+
def __init__(
|
|
76
|
+
self,
|
|
77
|
+
model_cls: DeepModel,
|
|
78
|
+
name: str = "CoreDetector",
|
|
79
|
+
config: DeepLearningDetectorConfig | dict[str, Any] = DeepLearningDetectorConfig()
|
|
80
|
+
) -> None:
|
|
81
|
+
|
|
82
|
+
if isinstance(config, dict):
|
|
83
|
+
config = DeepLearningDetectorConfig.from_dict(config, name)
|
|
84
|
+
|
|
85
|
+
by_session = config.session_variable is not None
|
|
86
|
+
super().__init__(
|
|
87
|
+
name=name,
|
|
88
|
+
buffer_mode=BufferMode.NO_BUF if by_session else BufferMode.WINDOW,
|
|
89
|
+
buffer_size=None if by_session else config.window_size,
|
|
90
|
+
config=config
|
|
91
|
+
)
|
|
92
|
+
self.config: DeepLearningDetectorConfig
|
|
93
|
+
self.model: DeepModel = model_cls(config=self.config.hyperparameters) # type: ignore
|
|
94
|
+
|
|
95
|
+
self.train_seqs: list[tuple[int, ...]] = []
|
|
96
|
+
self.config_seqs: list[tuple[int, ...]] = []
|
|
97
|
+
self.stats: dict[str, float | int] = {}
|
|
98
|
+
self.top_k: int = 0
|
|
99
|
+
|
|
100
|
+
# With sessions, configure() and train() each keep their own windows: train() also
|
|
101
|
+
# gets the configure records, replayed after the configure phase. detect() does not
|
|
102
|
+
# get them, so its windows are built where every record passes once, the buffer.
|
|
103
|
+
self.config_windows = SessionWindows(self.config.window_size)
|
|
104
|
+
self.train_windows = SessionWindows(self.config.window_size)
|
|
105
|
+
self.detect_windows = SessionWindows(self.config.window_size)
|
|
106
|
+
self.window: tuple[int, ...] = ()
|
|
107
|
+
if by_session:
|
|
108
|
+
self.args_buffer = ArgsBuffer(BufferMode.NO_BUF, process_function=self._add_to_session)
|
|
109
|
+
self.data_buffer = DataBuffer(self.args_buffer)
|
|
110
|
+
|
|
111
|
+
def _session(self, input_: schemas.ParserSchema) -> str:
|
|
112
|
+
session = input_["headerVariables"].get(self.config.session_variable)
|
|
113
|
+
if session is None:
|
|
114
|
+
raise ValueError(f"<<{self.name}>> log has no header variable {self.config.session_variable!r}")
|
|
115
|
+
return str(session)
|
|
116
|
+
|
|
117
|
+
def _add_to_session(self, input_: schemas.ParserSchema) -> schemas.ParserSchema:
|
|
118
|
+
self.window = self.detect_windows.add(self._session(input_), input_["eventID"])
|
|
119
|
+
return input_
|
|
120
|
+
|
|
121
|
+
def _seq(
|
|
122
|
+
self, input_: list[schemas.ParserSchema] | schemas.ParserSchema, windows: SessionWindows
|
|
123
|
+
) -> tuple[int, ...]:
|
|
124
|
+
if self.config.session_variable is None:
|
|
125
|
+
return build_seq(input_) # type: ignore
|
|
126
|
+
return windows.add(self._session(input_), input_["eventID"]) # type: ignore
|
|
127
|
+
|
|
128
|
+
def train(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore
|
|
129
|
+
self.train_seqs.append(self._seq(input_, self.train_windows))
|
|
130
|
+
|
|
131
|
+
def configure(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore
|
|
132
|
+
self.config_seqs.append(self._seq(input_, self.config_windows))
|
|
133
|
+
|
|
134
|
+
def set_configuration(self) -> None:
|
|
135
|
+
self.model.finetune(
|
|
136
|
+
self.config_seqs, var_per=self.config.validation_per, epochs=self.config.finetune_epochs
|
|
137
|
+
)
|
|
138
|
+
self.config_seqs = []
|
|
139
|
+
self.config_windows = SessionWindows(self.config.window_size)
|
|
140
|
+
|
|
141
|
+
def post_train(self) -> None:
|
|
142
|
+
self.stats = self.model.train(self.train_seqs, var_per=self.config.validation_per)
|
|
143
|
+
self.train_seqs = []
|
|
144
|
+
self.train_windows = SessionWindows(self.config.window_size)
|
|
145
|
+
|
|
146
|
+
if "top_k" in self.stats:
|
|
147
|
+
self.top_k = int(self.stats["top_k"])
|
|
148
|
+
logging.info(self.model)
|
|
149
|
+
logging.info(f"Top k assigned {self.top_k}")
|
|
150
|
+
|
|
151
|
+
def detect(
|
|
152
|
+
self,
|
|
153
|
+
input_: list[schemas.ParserSchema], # type: ignore
|
|
154
|
+
output_: schemas.DetectorSchema,
|
|
155
|
+
) -> bool:
|
|
156
|
+
|
|
157
|
+
seq = build_seq(input_) if self.config.session_variable is None else self.window
|
|
158
|
+
alert = self.model.check_anomaly(seq, top_k=self.top_k)
|
|
159
|
+
if alert:
|
|
160
|
+
output_["score"] = 1.0
|
|
161
|
+
output_["description"] = f"{self.name} found an anomaly in the sequence"
|
|
162
|
+
return True
|
|
163
|
+
|
|
164
|
+
return False
|
{detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/alert_aggregator.py
RENAMED
|
@@ -63,6 +63,7 @@ class CoreAlertAggregator(CoreComponent):
|
|
|
63
63
|
input_schema=DetectorSchema,
|
|
64
64
|
output_schema=AggregateSchema,
|
|
65
65
|
)
|
|
66
|
+
self.config: CoreAlertAggregatorConfig
|
|
66
67
|
|
|
67
68
|
@override
|
|
68
69
|
def run(
|
|
@@ -108,3 +109,9 @@ class CoreAlertAggregator(CoreComponent):
|
|
|
108
109
|
@override
|
|
109
110
|
def post_train(self) -> None:
|
|
110
111
|
pass
|
|
112
|
+
|
|
113
|
+
@override
|
|
114
|
+
def from_binary(self, binary: bytes) -> "CoreAlertAggregator":
|
|
115
|
+
new = type(self)(name=self.name, config=self.config)
|
|
116
|
+
new.load_binary(binary)
|
|
117
|
+
return new
|
|
@@ -119,10 +119,10 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
|
119
119
|
path: str | None = None,
|
|
120
120
|
storage_options: dict[str, Any] | None = None,
|
|
121
121
|
) -> bytes | None:
|
|
122
|
-
"""Save this component's state.
|
|
122
|
+
"""Save this component's state.
|
|
123
123
|
|
|
124
|
-
Components with state
|
|
125
|
-
base_detectors.TrackerDetector).
|
|
124
|
+
The base has none, so returns None. Components with state
|
|
125
|
+
override this (see base_detectors.TrackerDetector).
|
|
126
126
|
"""
|
|
127
127
|
return None
|
|
128
128
|
|
|
@@ -149,10 +149,16 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
|
|
|
149
149
|
def get_window_size(self) -> int:
|
|
150
150
|
return self.data_buffer.get_window_size()
|
|
151
151
|
|
|
152
|
+
def _is_input_eligible(self, data: BaseSchema) -> bool:
|
|
153
|
+
"""Check if input should be filtered or not."""
|
|
154
|
+
return True
|
|
155
|
+
|
|
152
156
|
def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None:
|
|
153
157
|
is_byte, data = SchemaPipeline.preprocess(self.input_schema(), data)
|
|
154
158
|
logger.debug(f"<<{self.name}>> received:\n{data}")
|
|
155
159
|
|
|
160
|
+
if not self._is_input_eligible(data):
|
|
161
|
+
return None
|
|
156
162
|
if (data_buffered := self.data_buffer.add(data)) is None: # type: ignore
|
|
157
163
|
return None
|
|
158
164
|
|
{detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/detector.py
RENAMED
|
@@ -6,8 +6,8 @@ from detectmatelibrary.utils.aux import get_timestamp
|
|
|
6
6
|
|
|
7
7
|
from detectmatelibrary.schemas import ParserSchema, DetectorSchema
|
|
8
8
|
|
|
9
|
-
from typing_extensions import override
|
|
10
9
|
from typing import Dict, List, Optional, Any
|
|
10
|
+
from typing_extensions import override
|
|
11
11
|
from pydantic import Field
|
|
12
12
|
|
|
13
13
|
from detectmatelibrary.utils.time_format_handler import TimeFormatHandler
|
|
@@ -32,10 +32,14 @@ def _extract_logIDs(input_: List[ParserSchema] | ParserSchema) -> List[str]:
|
|
|
32
32
|
return [str(i["logID"]) for i in input_]
|
|
33
33
|
|
|
34
34
|
|
|
35
|
+
def define_detector_id(name: str, parser: Optional[str]) -> str:
|
|
36
|
+
return name if parser is None else f"{name}_from_{parser}"
|
|
37
|
+
|
|
38
|
+
|
|
35
39
|
class CoreDetectorConfig(CoreConfig):
|
|
36
40
|
component_type: str = Field(default="detectors", description="<$IGNORE$>")
|
|
37
41
|
method_type: str = Field(default="core_detector", description="<$IGNORE$>")
|
|
38
|
-
parser: str = Field(default=
|
|
42
|
+
parser: Optional[str] = Field(default=None, description="Name of the parser used.")
|
|
39
43
|
|
|
40
44
|
auto_config: bool = Field(
|
|
41
45
|
default=True,
|
|
@@ -76,6 +80,9 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
|
|
|
76
80
|
input_schema=ParserSchema,
|
|
77
81
|
output_schema=DetectorSchema,
|
|
78
82
|
)
|
|
83
|
+
self.config: CoreDetectorConfig
|
|
84
|
+
|
|
85
|
+
self._detector_id = define_detector_id(name, config.parser)
|
|
79
86
|
|
|
80
87
|
@override
|
|
81
88
|
def run(
|
|
@@ -84,7 +91,7 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
|
|
|
84
91
|
output_: DetectorSchema,
|
|
85
92
|
) -> bool:
|
|
86
93
|
|
|
87
|
-
output_["detectorID"] = self.
|
|
94
|
+
output_["detectorID"] = self._detector_id
|
|
88
95
|
output_["detectorType"] = self.config.method_type
|
|
89
96
|
output_["logIDs"] = _extract_logIDs(input_)
|
|
90
97
|
output_["logTimestamps"] = _extract_timestamp(input_)
|
|
@@ -96,31 +103,25 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
|
|
|
96
103
|
|
|
97
104
|
return anomaly_detected
|
|
98
105
|
|
|
106
|
+
@override
|
|
107
|
+
def _is_input_eligible(self, data: ParserSchema) -> bool: # type: ignore
|
|
108
|
+
return self.config.parser is None or self.config.parser == data["parserID"]
|
|
109
|
+
|
|
99
110
|
def detect(
|
|
100
|
-
self,
|
|
101
|
-
input_: List[ParserSchema] | ParserSchema,
|
|
102
|
-
output_: DetectorSchema,
|
|
111
|
+
self, input_: List[ParserSchema] | ParserSchema, output_: DetectorSchema,
|
|
103
112
|
) -> bool:
|
|
104
113
|
return True
|
|
105
114
|
|
|
106
115
|
@override
|
|
107
|
-
def train(
|
|
108
|
-
self,
|
|
109
|
-
input_: ParserSchema | list[ParserSchema],
|
|
110
|
-
) -> None:
|
|
116
|
+
def train(self, input_: ParserSchema | list[ParserSchema]) -> None:
|
|
111
117
|
pass
|
|
112
118
|
|
|
113
119
|
@override
|
|
114
|
-
def configure(
|
|
115
|
-
self,
|
|
116
|
-
input_: ParserSchema | list[ParserSchema],
|
|
117
|
-
) -> None:
|
|
120
|
+
def configure(self, input_: ParserSchema | list[ParserSchema]) -> None:
|
|
118
121
|
pass
|
|
119
122
|
|
|
120
123
|
@override
|
|
121
|
-
def
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def post_train(self) -> None:
|
|
126
|
-
pass
|
|
124
|
+
def from_binary(self, binary: bytes) -> "CoreDetector":
|
|
125
|
+
new = type(self)(name=self.name, config=self.config)
|
|
126
|
+
new.load_binary(binary)
|
|
127
|
+
return new
|
|
@@ -4,29 +4,39 @@ from detectmatelibrary.utils.time_format_handler import TimeFormatHandler
|
|
|
4
4
|
from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode
|
|
5
5
|
from detectmatelibrary.common.core import CoreComponent, CoreConfig
|
|
6
6
|
from detectmatelibrary.utils.aux import get_timestamp
|
|
7
|
-
from detectmatelibrary import schemas
|
|
7
|
+
from detectmatelibrary import schemas as sc
|
|
8
8
|
|
|
9
|
-
from typing import Any, cast
|
|
10
9
|
from pydantic import model_validator
|
|
11
|
-
import re
|
|
12
10
|
from pydantic import Field
|
|
13
11
|
|
|
12
|
+
from typing_extensions import override, Optional
|
|
13
|
+
from typing import Any
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def define_parser_id(name: str, log_source: Optional[str]) -> str:
|
|
18
|
+
return name if log_source is None else f"{name}_from_{log_source}"
|
|
19
|
+
|
|
14
20
|
|
|
15
21
|
class CoreParserConfig(CoreConfig):
|
|
16
22
|
component_type: str = Field(default="parsers", description="<$IGNORE$>")
|
|
17
23
|
method_type: str = Field(default="core_parser", description="<$IGNORE$>")
|
|
18
24
|
|
|
19
|
-
log_format: str
|
|
25
|
+
log_format: Optional[str] = Field(
|
|
20
26
|
default=None,
|
|
21
27
|
description=("fitting description yet to find"),
|
|
22
28
|
)
|
|
23
|
-
time_format: str
|
|
29
|
+
time_format: Optional[str] = Field(
|
|
24
30
|
default=None,
|
|
25
31
|
description=("fitting description yet to find"),
|
|
26
32
|
)
|
|
27
33
|
|
|
28
34
|
_regex: re.Pattern[str] | None = None
|
|
29
35
|
|
|
36
|
+
log_source: Optional[str] = Field(
|
|
37
|
+
default=None, description="Process only inputs of the same log source."
|
|
38
|
+
)
|
|
39
|
+
|
|
30
40
|
# generate regex from log_format
|
|
31
41
|
@model_validator(mode="after")
|
|
32
42
|
def generate_regex(self) -> "CoreParserConfig":
|
|
@@ -40,7 +50,7 @@ class CoreParserConfig(CoreConfig):
|
|
|
40
50
|
return self
|
|
41
51
|
|
|
42
52
|
|
|
43
|
-
class CoreParser(CoreComponent):
|
|
53
|
+
class CoreParser(CoreComponent[sc.LogSchema, sc.ParserSchema]):
|
|
44
54
|
def __init__(
|
|
45
55
|
self,
|
|
46
56
|
name: str = "CoreParser",
|
|
@@ -54,21 +64,24 @@ class CoreParser(CoreComponent):
|
|
|
54
64
|
type_=config.method_type,
|
|
55
65
|
config=config,
|
|
56
66
|
args_buffer=ArgsBuffer(mode=BufferMode.NO_BUF, size=None),
|
|
57
|
-
input_schema=
|
|
58
|
-
output_schema=
|
|
67
|
+
input_schema=sc.LogSchema,
|
|
68
|
+
output_schema=sc.ParserSchema,
|
|
59
69
|
)
|
|
60
70
|
self.time_format_handler = TimeFormatHandler()
|
|
71
|
+
self.config: CoreParserConfig
|
|
72
|
+
|
|
73
|
+
self.__parser_id = define_parser_id(self.name, config.log_source)
|
|
61
74
|
|
|
62
|
-
|
|
63
|
-
|
|
75
|
+
@override
|
|
76
|
+
def run(self, input_: sc.LogSchema, output_: sc.ParserSchema) -> bool: # type: ignore
|
|
64
77
|
var, content = get_format_variables(
|
|
65
|
-
config._regex,
|
|
78
|
+
self.config._regex,
|
|
66
79
|
log=input_["log"],
|
|
67
|
-
time_format=config.time_format,
|
|
80
|
+
time_format=self.config.time_format,
|
|
68
81
|
time_format_handler=self.time_format_handler,
|
|
69
82
|
)
|
|
70
83
|
|
|
71
|
-
output_["parserID"] = self.
|
|
84
|
+
output_["parserID"] = self.__parser_id
|
|
72
85
|
output_["parsedLogID"] = str(self.id_generator())
|
|
73
86
|
output_["parserType"] = self.config.method_type
|
|
74
87
|
output_["logID"] = input_["logID"]
|
|
@@ -82,10 +95,23 @@ class CoreParser(CoreComponent):
|
|
|
82
95
|
|
|
83
96
|
return True if use_schema is None else use_schema
|
|
84
97
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
98
|
+
@override
|
|
99
|
+
def _is_input_eligible(self, data: sc.LogSchema) -> bool: # type: ignore
|
|
100
|
+
return self.config.log_source is None or self.config.log_source == data["logSource"]
|
|
101
|
+
|
|
102
|
+
def parse(self, input_: sc.LogSchema, output_: sc.ParserSchema) -> bool | None:
|
|
88
103
|
return True
|
|
89
104
|
|
|
90
|
-
|
|
105
|
+
@override
|
|
106
|
+
def train(self, input_: sc.LogSchema) -> None: # type: ignore
|
|
91
107
|
pass
|
|
108
|
+
|
|
109
|
+
@override
|
|
110
|
+
def configure(self, input_: sc.LogSchema,) -> None: # type: ignore
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
@override
|
|
114
|
+
def from_binary(self, binary: bytes) -> "CoreParser":
|
|
115
|
+
new = type(self)(name=self.name, config=self.config)
|
|
116
|
+
new.load_binary(binary)
|
|
117
|
+
return new
|