detectmatelibrary 0.7.0__tar.gz → 0.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. {detectmatelibrary-0.7.0/src/detectmatelibrary.egg-info → detectmatelibrary-0.7.2}/PKG-INFO +3 -3
  2. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/README.md +1 -1
  3. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/pyproject.toml +1 -1
  4. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/__init__.py +4 -0
  5. detectmatelibrary-0.7.2/src/detectmatelibrary/base_detectors/count_vector_detector.py +113 -0
  6. detectmatelibrary-0.7.2/src/detectmatelibrary/base_detectors/deeplearning_detector.py +164 -0
  7. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/alert_aggregator.py +7 -0
  8. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/core.py +9 -3
  9. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/detector.py +21 -20
  10. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/parser.py +43 -17
  11. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/deeplog_detector.py +1 -0
  12. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/ecvc_detector.py +29 -31
  13. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/event_sequence_detector.py +98 -23
  14. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/logbert_detector.py +1 -0
  15. detectmatelibrary-0.7.2/src/detectmatelibrary/detectors/scvs_detector.py +41 -0
  16. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/helper/from_to.py +155 -37
  17. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/metadata.py +1 -1
  18. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/__init__.py +28 -16
  19. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/pipeline.py +17 -13
  20. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/_op.py +33 -1
  21. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/deeplog.py +30 -17
  22. detectmatelibrary-0.7.2/src/detectmatelibrary/utils/deep_learning/imodel.py +19 -0
  23. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/deep_learning/logbert.py +51 -26
  24. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/sequence_encoding.py +6 -2
  25. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2/src/detectmatelibrary.egg-info}/PKG-INFO +3 -3
  26. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/SOURCES.txt +1 -0
  27. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/requires.txt +1 -1
  28. detectmatelibrary-0.7.0/src/detectmatelibrary/base_detectors/deeplearning_detector.py +0 -107
  29. detectmatelibrary-0.7.0/src/detectmatelibrary/detectors/scvs_detector.py +0 -71
  30. detectmatelibrary-0.7.0/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -19
  31. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/LICENSE.md +0 -0
  32. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/setup.cfg +0 -0
  33. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/__init__.py +0 -0
  34. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/__init__.py +0 -0
  35. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +0 -0
  36. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_detector.py +0 -0
  37. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/_testutils/dummy_parser.py +0 -0
  38. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/__init__.py +0 -0
  39. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/alertbert.py +0 -0
  40. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/basic_concat.py +0 -0
  41. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/logid_count.py +0 -0
  42. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/alert_aggregation/time_delta.py +0 -0
  43. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_alert_aggregation/__init__.py +0 -0
  44. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +0 -0
  45. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_persist.py +0 -0
  46. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_stability.py +0 -0
  47. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/_variable_hooks.py +0 -0
  48. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/tracker_detector.py +0 -0
  49. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/base_detectors/variable_detector.py +0 -0
  50. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/__init__.py +0 -0
  51. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/__init__.py +0 -0
  52. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/_compile.py +0 -0
  53. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_config/_formats.py +0 -0
  54. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_basic_component.py +0 -0
  55. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_fed_component.py +0 -0
  56. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
  57. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
  58. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/common/_core_op/_train_buffer.py +0 -0
  59. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/constants.py +0 -0
  60. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/__init__.py +0 -0
  61. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +0 -0
  62. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/charset_detector.py +0 -0
  63. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_event_detector.py +0 -0
  64. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_value_combo_detector.py +0 -0
  65. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/new_value_detector.py +0 -0
  66. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/random_detector.py +0 -0
  67. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/rule_detector.py +0 -0
  68. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/detectors/value_range_detector.py +0 -0
  69. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/helper/select_component.py +0 -0
  70. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/autoparser.py +0 -0
  71. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/drain.py +0 -0
  72. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/json_parser.py +0 -0
  73. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
  74. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
  75. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +0 -0
  76. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +0 -0
  77. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +0 -0
  78. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +0 -0
  79. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +0 -0
  80. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +0 -0
  81. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +0 -0
  82. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +0 -0
  83. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +0 -0
  84. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +0 -0
  85. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/logbatcher/parser.py +0 -0
  86. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
  87. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +0 -0
  88. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/template_matcher/_parser.py +0 -0
  89. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/parsers/tree_matcher.py +0 -0
  90. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/__init__.py +0 -0
  91. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/_classes.py +0 -0
  92. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/__init__.py +0 -0
  93. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/default.py +0 -0
  94. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/mongos.py +0 -0
  95. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/connectors/redis.py +0 -0
  96. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/protobuf/_op.py +0 -0
  97. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/schemas/protobuf/schemas_pb2.py +0 -0
  98. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/__init__.py +0 -0
  99. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/logging.py +0 -0
  100. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/__init__.py +0 -0
  101. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/create_workspace.py +0 -0
  102. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +0 -0
  103. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +0 -0
  104. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +0 -0
  105. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/__init__.py +0 -0
  106. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/__init__.py +0 -0
  107. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +0 -0
  108. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +0 -0
  109. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +0 -0
  110. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
  111. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +0 -0
  112. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -0
  113. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -0
  114. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
  115. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/datasets.py +0 -0
  116. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/models.py +0 -0
  117. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/alertbert/preprocessing.py +0 -0
  118. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/aux.py +0 -0
  119. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/data_buffer.py +0 -0
  120. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/finetune.py +0 -0
  121. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/id_generator.py +0 -0
  122. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/key_extractor.py +0 -0
  123. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
  124. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/__init__.py +0 -0
  125. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/basic_persistency.py +0 -0
  126. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/base.py +0 -0
  127. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/__init__.py +0 -0
  128. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/__init__.py +0 -0
  129. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/event_tracker.py +0 -0
  130. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/multi_tracker.py +0 -0
  131. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/single_tracker.py +0 -0
  132. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/rle_list.py +0 -0
  133. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/__init__.py +0 -0
  134. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/classification_methods.py +0 -0
  135. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_classifier.py +0 -0
  136. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_tracker.py +0 -0
  137. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/event_persistency.py +0 -0
  138. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/persistency_saver.py +0 -0
  139. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/persistency/slow_persistency.py +0 -0
  140. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
  141. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
  142. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
  143. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
  144. {detectmatelibrary-0.7.0 → detectmatelibrary-0.7.2}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectmatelibrary
3
- Version: 0.7.0
3
+ Version: 0.7.2
4
4
  Summary: DetectMate Library for log processing components
5
5
  Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf, Lukas Karner
6
6
  License-Expression: EUPL-1.2
@@ -14,7 +14,7 @@ Requires-Dist: pydantic>=2.13.3
14
14
  Requires-Dist: pyyaml>=6.0.3
15
15
  Requires-Dist: regex>=2025.11.3
16
16
  Requires-Dist: numpy>=2.3.2
17
- Requires-Dist: detectmateperformance>=0.1.6
17
+ Requires-Dist: detectmateperformance>=0.2.0
18
18
  Requires-Dist: msgpack>=1.0.0
19
19
  Requires-Dist: fsspec>=2024.1.0
20
20
  Requires-Dist: pyarrow>=24.0.0
@@ -62,7 +62,7 @@ pip install detectmatelibrary
62
62
  Main documentations:
63
63
 
64
64
  - [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
65
- - [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
65
+ - [Service Documentation](https://ait-detectmate.github.io/DetectMateService/latest/)
66
66
 
67
67
  ## Contribution
68
68
 
@@ -18,7 +18,7 @@ pip install detectmatelibrary
18
18
  Main documentations:
19
19
 
20
20
  - [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
21
- - [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
21
+ - [Service Documentation](https://ait-detectmate.github.io/DetectMateService/latest/)
22
22
 
23
23
  ## Contribution
24
24
 
@@ -13,7 +13,7 @@ dependencies = [
13
13
  "pyyaml>=6.0.3",
14
14
  "regex>=2025.11.3",
15
15
  "numpy>=2.3.2",
16
- "detectmateperformance>=0.1.6",
16
+ "detectmateperformance>=0.2.0",
17
17
  "msgpack>=1.0.0",
18
18
  "fsspec>=2024.1.0",
19
19
  "pyarrow>=24.0.0",
@@ -9,6 +9,7 @@ from detectmatelibrary.utils.persistency.data_structures.trackers.stability.stab
9
9
  SingleStabilityTracker,
10
10
  )
11
11
 
12
+ from .count_vector_detector import CountVectorDetector, CountVectorDetectorConfig, SessionWindows
12
13
  from ._persist import PersistConfig
13
14
  from .deeplearning_detector import DeepLearningDetector, DeepLearningDetectorConfig
14
15
  from ._stability import StabilityAutoConfigParams
@@ -16,10 +17,13 @@ from .tracker_detector import TrackerDetector, TrackerDetectorConfig
16
17
  from .variable_detector import VariableAutoConfigParams, VariableDetector, VariableDetectorConfig
17
18
 
18
19
  __all__ = [
20
+ "CountVectorDetector",
21
+ "CountVectorDetectorConfig",
19
22
  "DeepLearningDetector",
20
23
  "DeepLearningDetectorConfig",
21
24
  "EventStabilityTracker",
22
25
  "PersistConfig",
26
+ "SessionWindows",
23
27
  "SingleStabilityTracker",
24
28
  "StabilityAutoConfigParams",
25
29
  "TrackerDetector",
@@ -0,0 +1,113 @@
1
+ from typing import List, Sequence
2
+ from typing_extensions import override
3
+
4
+ from detectmatelibrary import schemas
5
+ from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode, DataBuffer
6
+ from detectmatelibrary.utils.sequence_encoding import (
7
+ count_events,
8
+ decode_count_vec,
9
+ encode_count_vec,
10
+ warn_on_window_size_mismatch,
11
+ )
12
+
13
+ from pydantic import Field
14
+
15
+ from .tracker_detector import TrackerDetector, TrackerDetectorConfig
16
+
17
+
18
+ class SessionWindows:
19
+ """The window ending at the latest event of each session: its last `size`
20
+ events, fewer at the session start."""
21
+
22
+ def __init__(self, size: int) -> None:
23
+ self.size = size
24
+ self.windows: dict[str, tuple[int, ...]] = {}
25
+
26
+ def add(self, session: str, event: int) -> tuple[int, ...]:
27
+ """Add ``event`` to ``session`` and return the session's window ending
28
+ at it."""
29
+ window = (self.windows.get(session, ()) + (event,))[-self.size:]
30
+ self.windows[session] = window
31
+ return window
32
+
33
+
34
+ class CountVectorDetectorConfig(TrackerDetectorConfig):
35
+ window_size: int = Field(
36
+ default=10,
37
+ description="Length of the event-ID window a count vector is built over.",
38
+ )
39
+ session_variable: str | None = Field(
40
+ default=None,
41
+ description=(
42
+ "Header variable that names the session of each event (HDFS: block_id). When set, "
43
+ "an event's window holds the last window_size events of its own session, fewer at "
44
+ "the session start, so windows never mix sessions and every event is checked."
45
+ ),
46
+ )
47
+
48
+
49
+ class CountVectorDetector(TrackerDetector):
50
+ """Base of the detectors that learn the count vectors of event-ID
51
+ windows."""
52
+
53
+ def __init__(self, name: str, config: CountVectorDetectorConfig) -> None:
54
+ self.config: CountVectorDetectorConfig
55
+
56
+ by_session = config.session_variable is not None
57
+ super().__init__(
58
+ name=name,
59
+ config=config,
60
+ buffer_mode=BufferMode.NO_BUF if by_session else BufferMode.WINDOW,
61
+ buffer_size=None if by_session else config.window_size,
62
+ )
63
+ # With sessions, train() keeps its own windows: it also gets the configure records,
64
+ # replayed after the configure phase. detect() does not get them, so its windows are
65
+ # built where every record passes once, the buffer.
66
+ self.train_windows = SessionWindows(config.window_size)
67
+ self.detect_windows = SessionWindows(config.window_size)
68
+ self.window: tuple[int, ...] = ()
69
+ if by_session:
70
+ self.args_buffer = ArgsBuffer(BufferMode.NO_BUF, process_function=self._add_to_session)
71
+ self.data_buffer = DataBuffer(self.args_buffer)
72
+
73
+ def _session(self, input_: schemas.ParserSchema) -> str:
74
+ session = input_["headerVariables"].get(self.config.session_variable)
75
+ if session is None:
76
+ raise ValueError(f"<<{self.name}>> log has no header variable {self.config.session_variable!r}")
77
+ return str(session)
78
+
79
+ def _add_to_session(self, input_: schemas.ParserSchema) -> schemas.ParserSchema:
80
+ self.window = self.detect_windows.add(self._session(input_), input_["eventID"])
81
+ return input_
82
+
83
+ def window_events(self, input_: List[schemas.ParserSchema] | schemas.ParserSchema) -> Sequence[int]:
84
+ """Event IDs of the window of ``input_``."""
85
+ if isinstance(input_, list):
86
+ return [in_["eventID"] for in_ in input_]
87
+ return self.window
88
+
89
+ @override
90
+ def _sync_from_state(self) -> None:
91
+ warn_on_window_size_mismatch(self.name, self.persistency.get_events_seen(), self.config.window_size)
92
+
93
+ def train(self, input_: List[schemas.ParserSchema] | schemas.ParserSchema) -> None:
94
+ if isinstance(input_, list):
95
+ events: Sequence[int] = [in_["eventID"] for in_ in input_]
96
+ last = input_[-1]
97
+ else:
98
+ events, last = self.train_windows.add(self._session(input_), input_["eventID"]), input_
99
+ self._ingest(
100
+ event_id=encode_count_vec(self.config.window_size, count_events(events)),
101
+ input_=last,
102
+ variables={}
103
+ )
104
+
105
+ def post_train(self) -> None:
106
+ self.train_windows = SessionWindows(self.config.window_size)
107
+
108
+ def get_known_count_vecs(self) -> set[tuple[int, ...]]:
109
+ """Return the count vectors learned during training."""
110
+ return {
111
+ decode_count_vec(str(encoded))[1]
112
+ for encoded in self.persistency.get_events_seen()
113
+ }
@@ -0,0 +1,164 @@
1
+
2
+ from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
3
+
4
+ from detectmatelibrary.utils.deep_learning.imodel import DeepModel
5
+ from detectmatelibrary.utils.deep_learning._op import PAD
6
+ from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode, DataBuffer
7
+
8
+ from detectmatelibrary import schemas
9
+
10
+ from typing import Any
11
+ from pydantic import Field
12
+ import logging
13
+
14
+
15
+ class DeepLearningDetectorConfig(CoreDetectorConfig):
16
+ window_size: int = Field(
17
+ default=10,
18
+ description="Number of consecutive events used as one training/detection sequence.",
19
+ )
20
+ validation_per: float = Field(
21
+ default=0.2,
22
+ description="Fraction of data held out for validation during (fine)training.",
23
+ )
24
+ finetune_epochs: int = Field(
25
+ default=2,
26
+ description="Number of epochs used when finetuning during the configuration phase.",
27
+ )
28
+ session_variable: str | None = Field(
29
+ default=None,
30
+ description=(
31
+ "Header variable that names the session of each event (HDFS: block_id). When set, "
32
+ "an event's window holds the events before it in its own session, padded at the "
33
+ "session start, so windows never mix sessions and every event is checked."
34
+ ),
35
+ )
36
+
37
+ hyperparameters: dict[str, Any] = Field(
38
+ default={
39
+ "Model": {
40
+
41
+ },
42
+ "Train": {
43
+
44
+ },
45
+ "Finetune": [],
46
+ },
47
+ description=(
48
+ "Model, training and hyperparameter-search settings passed to "
49
+ "the underlying deep learning model."
50
+ ),
51
+ )
52
+
53
+
54
+ def build_seq(input_: list[schemas.ParserSchema]) -> tuple[int, ...]:
55
+ return tuple([in_["eventID"] for in_ in input_])
56
+
57
+
58
+ class SessionWindows:
59
+ """The window ending at the latest event of each session, padded with PAD
60
+ at its start."""
61
+
62
+ def __init__(self, size: int) -> None:
63
+ self.size = size
64
+ self.windows: dict[str, tuple[int, ...]] = {}
65
+
66
+ def add(self, session: str, event: int) -> tuple[int, ...]:
67
+ """Add ``event`` to ``session`` and return the session's window ending
68
+ at it."""
69
+ window = self.windows.get(session, (PAD,) * self.size)[1:] + (event,)
70
+ self.windows[session] = window
71
+ return window
72
+
73
+
74
+ class DeepLearningDetector(CoreDetector):
75
+ def __init__(
76
+ self,
77
+ model_cls: DeepModel,
78
+ name: str = "CoreDetector",
79
+ config: DeepLearningDetectorConfig | dict[str, Any] = DeepLearningDetectorConfig()
80
+ ) -> None:
81
+
82
+ if isinstance(config, dict):
83
+ config = DeepLearningDetectorConfig.from_dict(config, name)
84
+
85
+ by_session = config.session_variable is not None
86
+ super().__init__(
87
+ name=name,
88
+ buffer_mode=BufferMode.NO_BUF if by_session else BufferMode.WINDOW,
89
+ buffer_size=None if by_session else config.window_size,
90
+ config=config
91
+ )
92
+ self.config: DeepLearningDetectorConfig
93
+ self.model: DeepModel = model_cls(config=self.config.hyperparameters) # type: ignore
94
+
95
+ self.train_seqs: list[tuple[int, ...]] = []
96
+ self.config_seqs: list[tuple[int, ...]] = []
97
+ self.stats: dict[str, float | int] = {}
98
+ self.top_k: int = 0
99
+
100
+ # With sessions, configure() and train() each keep their own windows: train() also
101
+ # gets the configure records, replayed after the configure phase. detect() does not
102
+ # get them, so its windows are built where every record passes once, the buffer.
103
+ self.config_windows = SessionWindows(self.config.window_size)
104
+ self.train_windows = SessionWindows(self.config.window_size)
105
+ self.detect_windows = SessionWindows(self.config.window_size)
106
+ self.window: tuple[int, ...] = ()
107
+ if by_session:
108
+ self.args_buffer = ArgsBuffer(BufferMode.NO_BUF, process_function=self._add_to_session)
109
+ self.data_buffer = DataBuffer(self.args_buffer)
110
+
111
+ def _session(self, input_: schemas.ParserSchema) -> str:
112
+ session = input_["headerVariables"].get(self.config.session_variable)
113
+ if session is None:
114
+ raise ValueError(f"<<{self.name}>> log has no header variable {self.config.session_variable!r}")
115
+ return str(session)
116
+
117
+ def _add_to_session(self, input_: schemas.ParserSchema) -> schemas.ParserSchema:
118
+ self.window = self.detect_windows.add(self._session(input_), input_["eventID"])
119
+ return input_
120
+
121
+ def _seq(
122
+ self, input_: list[schemas.ParserSchema] | schemas.ParserSchema, windows: SessionWindows
123
+ ) -> tuple[int, ...]:
124
+ if self.config.session_variable is None:
125
+ return build_seq(input_) # type: ignore
126
+ return windows.add(self._session(input_), input_["eventID"]) # type: ignore
127
+
128
+ def train(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore
129
+ self.train_seqs.append(self._seq(input_, self.train_windows))
130
+
131
+ def configure(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore
132
+ self.config_seqs.append(self._seq(input_, self.config_windows))
133
+
134
+ def set_configuration(self) -> None:
135
+ self.model.finetune(
136
+ self.config_seqs, var_per=self.config.validation_per, epochs=self.config.finetune_epochs
137
+ )
138
+ self.config_seqs = []
139
+ self.config_windows = SessionWindows(self.config.window_size)
140
+
141
+ def post_train(self) -> None:
142
+ self.stats = self.model.train(self.train_seqs, var_per=self.config.validation_per)
143
+ self.train_seqs = []
144
+ self.train_windows = SessionWindows(self.config.window_size)
145
+
146
+ if "top_k" in self.stats:
147
+ self.top_k = int(self.stats["top_k"])
148
+ logging.info(self.model)
149
+ logging.info(f"Top k assigned {self.top_k}")
150
+
151
+ def detect(
152
+ self,
153
+ input_: list[schemas.ParserSchema], # type: ignore
154
+ output_: schemas.DetectorSchema,
155
+ ) -> bool:
156
+
157
+ seq = build_seq(input_) if self.config.session_variable is None else self.window
158
+ alert = self.model.check_anomaly(seq, top_k=self.top_k)
159
+ if alert:
160
+ output_["score"] = 1.0
161
+ output_["description"] = f"{self.name} found an anomaly in the sequence"
162
+ return True
163
+
164
+ return False
@@ -63,6 +63,7 @@ class CoreAlertAggregator(CoreComponent):
63
63
  input_schema=DetectorSchema,
64
64
  output_schema=AggregateSchema,
65
65
  )
66
+ self.config: CoreAlertAggregatorConfig
66
67
 
67
68
  @override
68
69
  def run(
@@ -108,3 +109,9 @@ class CoreAlertAggregator(CoreComponent):
108
109
  @override
109
110
  def post_train(self) -> None:
110
111
  pass
112
+
113
+ @override
114
+ def from_binary(self, binary: bytes) -> "CoreAlertAggregator":
115
+ new = type(self)(name=self.name, config=self.config)
116
+ new.load_binary(binary)
117
+ return new
@@ -119,10 +119,10 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
119
119
  path: str | None = None,
120
120
  storage_options: dict[str, Any] | None = None,
121
121
  ) -> bytes | None:
122
- """Save this component's state. The base has none, so returns None.
122
+ """Save this component's state.
123
123
 
124
- Components with state override this (see
125
- base_detectors.TrackerDetector).
124
+ The base has none, so returns None. Components with state
125
+ override this (see base_detectors.TrackerDetector).
126
126
  """
127
127
  return None
128
128
 
@@ -149,10 +149,16 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
149
149
  def get_window_size(self) -> int:
150
150
  return self.data_buffer.get_window_size()
151
151
 
152
+ def _is_input_eligible(self, data: BaseSchema) -> bool:
153
+ """Check if input should be filtered or not."""
154
+ return True
155
+
152
156
  def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None:
153
157
  is_byte, data = SchemaPipeline.preprocess(self.input_schema(), data)
154
158
  logger.debug(f"<<{self.name}>> received:\n{data}")
155
159
 
160
+ if not self._is_input_eligible(data):
161
+ return None
156
162
  if (data_buffered := self.data_buffer.add(data)) is None: # type: ignore
157
163
  return None
158
164
 
@@ -6,8 +6,8 @@ from detectmatelibrary.utils.aux import get_timestamp
6
6
 
7
7
  from detectmatelibrary.schemas import ParserSchema, DetectorSchema
8
8
 
9
- from typing_extensions import override
10
9
  from typing import Dict, List, Optional, Any
10
+ from typing_extensions import override
11
11
  from pydantic import Field
12
12
 
13
13
  from detectmatelibrary.utils.time_format_handler import TimeFormatHandler
@@ -32,10 +32,14 @@ def _extract_logIDs(input_: List[ParserSchema] | ParserSchema) -> List[str]:
32
32
  return [str(i["logID"]) for i in input_]
33
33
 
34
34
 
35
+ def define_detector_id(name: str, parser: Optional[str]) -> str:
36
+ return name if parser is None else f"{name}_from_{parser}"
37
+
38
+
35
39
  class CoreDetectorConfig(CoreConfig):
36
40
  component_type: str = Field(default="detectors", description="<$IGNORE$>")
37
41
  method_type: str = Field(default="core_detector", description="<$IGNORE$>")
38
- parser: str = Field(default="PARSER", description="Name of the parser used.")
42
+ parser: Optional[str] = Field(default=None, description="Name of the parser used.")
39
43
 
40
44
  auto_config: bool = Field(
41
45
  default=True,
@@ -76,6 +80,9 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
76
80
  input_schema=ParserSchema,
77
81
  output_schema=DetectorSchema,
78
82
  )
83
+ self.config: CoreDetectorConfig
84
+
85
+ self._detector_id = define_detector_id(name, config.parser)
79
86
 
80
87
  @override
81
88
  def run(
@@ -84,7 +91,7 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
84
91
  output_: DetectorSchema,
85
92
  ) -> bool:
86
93
 
87
- output_["detectorID"] = self.name
94
+ output_["detectorID"] = self._detector_id
88
95
  output_["detectorType"] = self.config.method_type
89
96
  output_["logIDs"] = _extract_logIDs(input_)
90
97
  output_["logTimestamps"] = _extract_timestamp(input_)
@@ -96,31 +103,25 @@ class CoreDetector(CoreComponent[ParserSchema, DetectorSchema]):
96
103
 
97
104
  return anomaly_detected
98
105
 
106
+ @override
107
+ def _is_input_eligible(self, data: ParserSchema) -> bool: # type: ignore
108
+ return self.config.parser is None or self.config.parser == data["parserID"]
109
+
99
110
  def detect(
100
- self,
101
- input_: List[ParserSchema] | ParserSchema,
102
- output_: DetectorSchema,
111
+ self, input_: List[ParserSchema] | ParserSchema, output_: DetectorSchema,
103
112
  ) -> bool:
104
113
  return True
105
114
 
106
115
  @override
107
- def train(
108
- self,
109
- input_: ParserSchema | list[ParserSchema],
110
- ) -> None:
116
+ def train(self, input_: ParserSchema | list[ParserSchema]) -> None:
111
117
  pass
112
118
 
113
119
  @override
114
- def configure(
115
- self,
116
- input_: ParserSchema | list[ParserSchema],
117
- ) -> None:
120
+ def configure(self, input_: ParserSchema | list[ParserSchema]) -> None:
118
121
  pass
119
122
 
120
123
  @override
121
- def set_configuration(self) -> None:
122
- pass
123
-
124
- @override
125
- def post_train(self) -> None:
126
- pass
124
+ def from_binary(self, binary: bytes) -> "CoreDetector":
125
+ new = type(self)(name=self.name, config=self.config)
126
+ new.load_binary(binary)
127
+ return new
@@ -4,29 +4,39 @@ from detectmatelibrary.utils.time_format_handler import TimeFormatHandler
4
4
  from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode
5
5
  from detectmatelibrary.common.core import CoreComponent, CoreConfig
6
6
  from detectmatelibrary.utils.aux import get_timestamp
7
- from detectmatelibrary import schemas
7
+ from detectmatelibrary import schemas as sc
8
8
 
9
- from typing import Any, cast
10
9
  from pydantic import model_validator
11
- import re
12
10
  from pydantic import Field
13
11
 
12
+ from typing_extensions import override, Optional
13
+ from typing import Any
14
+ import re
15
+
16
+
17
+ def define_parser_id(name: str, log_source: Optional[str]) -> str:
18
+ return name if log_source is None else f"{name}_from_{log_source}"
19
+
14
20
 
15
21
  class CoreParserConfig(CoreConfig):
16
22
  component_type: str = Field(default="parsers", description="<$IGNORE$>")
17
23
  method_type: str = Field(default="core_parser", description="<$IGNORE$>")
18
24
 
19
- log_format: str | None = Field(
25
+ log_format: Optional[str] = Field(
20
26
  default=None,
21
27
  description=("fitting description yet to find"),
22
28
  )
23
- time_format: str | None = Field(
29
+ time_format: Optional[str] = Field(
24
30
  default=None,
25
31
  description=("fitting description yet to find"),
26
32
  )
27
33
 
28
34
  _regex: re.Pattern[str] | None = None
29
35
 
36
+ log_source: Optional[str] = Field(
37
+ default=None, description="Process only inputs of the same log source."
38
+ )
39
+
30
40
  # generate regex from log_format
31
41
  @model_validator(mode="after")
32
42
  def generate_regex(self) -> "CoreParserConfig":
@@ -40,7 +50,7 @@ class CoreParserConfig(CoreConfig):
40
50
  return self
41
51
 
42
52
 
43
- class CoreParser(CoreComponent):
53
+ class CoreParser(CoreComponent[sc.LogSchema, sc.ParserSchema]):
44
54
  def __init__(
45
55
  self,
46
56
  name: str = "CoreParser",
@@ -54,21 +64,24 @@ class CoreParser(CoreComponent):
54
64
  type_=config.method_type,
55
65
  config=config,
56
66
  args_buffer=ArgsBuffer(mode=BufferMode.NO_BUF, size=None),
57
- input_schema=schemas.LogSchema,
58
- output_schema=schemas.ParserSchema,
67
+ input_schema=sc.LogSchema,
68
+ output_schema=sc.ParserSchema,
59
69
  )
60
70
  self.time_format_handler = TimeFormatHandler()
71
+ self.config: CoreParserConfig
72
+
73
+ self.__parser_id = define_parser_id(self.name, config.log_source)
61
74
 
62
- def run(self, input_: schemas.LogSchema, output_: schemas.ParserSchema) -> bool: # type: ignore
63
- config = cast(CoreParserConfig, self.config)
75
+ @override
76
+ def run(self, input_: sc.LogSchema, output_: sc.ParserSchema) -> bool: # type: ignore
64
77
  var, content = get_format_variables(
65
- config._regex,
78
+ self.config._regex,
66
79
  log=input_["log"],
67
- time_format=config.time_format,
80
+ time_format=self.config.time_format,
68
81
  time_format_handler=self.time_format_handler,
69
82
  )
70
83
 
71
- output_["parserID"] = self.name
84
+ output_["parserID"] = self.__parser_id
72
85
  output_["parsedLogID"] = str(self.id_generator())
73
86
  output_["parserType"] = self.config.method_type
74
87
  output_["logID"] = input_["logID"]
@@ -82,10 +95,23 @@ class CoreParser(CoreComponent):
82
95
 
83
96
  return True if use_schema is None else use_schema
84
97
 
85
- def parse(
86
- self, input_: schemas.LogSchema, output_: schemas.ParserSchema
87
- ) -> bool | None:
98
+ @override
99
+ def _is_input_eligible(self, data: sc.LogSchema) -> bool: # type: ignore
100
+ return self.config.log_source is None or self.config.log_source == data["logSource"]
101
+
102
+ def parse(self, input_: sc.LogSchema, output_: sc.ParserSchema) -> bool | None:
88
103
  return True
89
104
 
90
- def train(self, input_: schemas.LogSchema) -> None: # type: ignore
105
+ @override
106
+ def train(self, input_: sc.LogSchema) -> None: # type: ignore
91
107
  pass
108
+
109
+ @override
110
+ def configure(self, input_: sc.LogSchema,) -> None: # type: ignore
111
+ pass
112
+
113
+ @override
114
+ def from_binary(self, binary: bytes) -> "CoreParser":
115
+ new = type(self)(name=self.name, config=self.config)
116
+ new.load_binary(binary)
117
+ return new
@@ -26,6 +26,7 @@ class DeeplogDetectorConfig(DeepLearningDetectorConfig):
26
26
  "learning_rate": 0.01,
27
27
  "epochs": 10,
28
28
  "patience": 3,
29
+ "alert_rate": 0.001,
29
30
  },
30
31
  "Finetune": [
31
32
  ["Model", "hidden_dim", [128, 256, 512]],