detectmatelibrary 0.6.0__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. detectmatelibrary-0.6.2/PKG-INFO +74 -0
  2. detectmatelibrary-0.6.2/README.md +33 -0
  3. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/__init__.py +2 -0
  4. detectmatelibrary-0.6.2/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +27 -0
  5. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/dummy_detector.py +5 -3
  6. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/dummy_parser.py +1 -1
  7. detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/__init__.py +12 -0
  8. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/alert_aggregation/basic_concat.py +10 -5
  9. detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/logid_count.py +96 -0
  10. detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/time_delta.py +111 -0
  11. detectmatelibrary-0.6.2/src/detectmatelibrary/base_alert_aggregation/__init__.py +9 -0
  12. detectmatelibrary-0.6.2/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +429 -0
  13. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/__init__.py +30 -0
  14. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_persist.py +83 -0
  15. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_stability.py +77 -0
  16. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/_variable_hooks.py +62 -0
  17. {detectmatelibrary-0.6.0/src/detectmatelibrary/common → detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors}/deeplearning_detector.py +2 -2
  18. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/tracker_detector.py +203 -0
  19. detectmatelibrary-0.6.2/src/detectmatelibrary/base_detectors/variable_detector.py +221 -0
  20. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/_compile.py +65 -4
  21. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_basic_component.py +1 -4
  22. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_fed_component.py +17 -6
  23. detectmatelibrary-0.6.2/src/detectmatelibrary/common/_core_op/_train_buffer.py +122 -0
  24. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/alert_aggregator.py +25 -15
  25. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/core.py +62 -41
  26. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/detector.py +11 -25
  27. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/parser.py +5 -5
  28. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +13 -19
  29. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/charset_detector.py +2 -5
  30. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/deeplog_detector.py +1 -1
  31. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/ecvc_detector.py +15 -28
  32. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/event_sequence_detector.py +25 -46
  33. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/logbert_detector.py +1 -1
  34. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_event_detector.py +15 -22
  35. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_value_combo_detector.py +60 -29
  36. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/new_value_detector.py +2 -5
  37. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/random_detector.py +8 -8
  38. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/rule_detector.py +3 -3
  39. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/scvs_detector.py +9 -18
  40. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/value_range_detector.py +2 -5
  41. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/helper/from_to.py +3 -3
  42. detectmatelibrary-0.6.2/src/detectmatelibrary/helper/select_component.py +74 -0
  43. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/metadata.py +2 -2
  44. detectmatelibrary-0.6.2/src/detectmatelibrary/parsers/__init__.py +23 -0
  45. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/autoparser.py +2 -2
  46. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/drain.py +2 -2
  47. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/json_parser.py +20 -6
  48. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +1 -1
  49. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/parser.py +1 -1
  50. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/_parser.py +1 -1
  51. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/tree_matcher.py +1 -1
  52. detectmatelibrary-0.6.2/src/detectmatelibrary/pipeline.py +174 -0
  53. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/__init__.py +4 -1
  54. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/_classes.py +13 -6
  55. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/schemas/_op.py +25 -9
  56. detectmatelibrary-0.6.2/src/detectmatelibrary/schemas/schemas_pb2.py +45 -0
  57. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/create_workspace.py +9 -7
  58. detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +36 -0
  59. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +1 -8
  60. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +1 -6
  61. detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +194 -0
  62. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +12 -12
  63. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +36 -36
  64. detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +50 -0
  65. detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +46 -0
  66. detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +46 -0
  67. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/data_normalizer.py +3 -3
  68. detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/__init__.py +23 -0
  69. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/basic_persistency.py +53 -30
  70. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures/trackers}/rle_list.py +1 -1
  71. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/stability_classifier.py +1 -1
  72. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/stability_tracker.py +4 -7
  73. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/event_persistency.py +13 -11
  74. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/persistency/persistency_saver.py +10 -28
  75. detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/slow_persistency.py +137 -0
  76. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/sequence_encoding.py +6 -7
  77. detectmatelibrary-0.6.2/src/detectmatelibrary.egg-info/PKG-INFO +74 -0
  78. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/SOURCES.txt +31 -19
  79. detectmatelibrary-0.6.0/PKG-INFO +0 -156
  80. detectmatelibrary-0.6.0/README.md +0 -115
  81. detectmatelibrary-0.6.0/src/detectmatelibrary/common/_other_op/_variable_hooks.py +0 -237
  82. detectmatelibrary-0.6.0/src/detectmatelibrary/common/persist.py +0 -32
  83. detectmatelibrary-0.6.0/src/detectmatelibrary/common/variable_detector.py +0 -150
  84. detectmatelibrary-0.6.0/src/detectmatelibrary/schemas/schemas_pb2.py +0 -45
  85. detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
  86. detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -37
  87. detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -32
  88. detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/__init__.py +0 -44
  89. detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/component_interfaces.py +0 -148
  90. detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/__init__.py +0 -4
  91. detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/chunked_event_dataframe.py +0 -122
  92. detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures/dataframes/event_dataframe.py +0 -57
  93. detectmatelibrary-0.6.0/src/detectmatelibrary.egg-info/PKG-INFO +0 -156
  94. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/LICENSE.md +0 -0
  95. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/pyproject.toml +0 -0
  96. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/setup.cfg +0 -0
  97. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/_testutils/__init__.py +0 -0
  98. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/__init__.py +0 -0
  99. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/__init__.py +0 -0
  100. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_config/_formats.py +0 -0
  101. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
  102. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
  103. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/constants.py +0 -0
  104. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/detectors/__init__.py +0 -0
  105. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
  106. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
  107. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +0 -0
  108. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +0 -0
  109. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +0 -0
  110. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +0 -0
  111. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +0 -0
  112. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +0 -0
  113. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +0 -0
  114. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +0 -0
  115. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +0 -0
  116. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
  117. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +0 -0
  118. {detectmatelibrary-0.6.0/src/detectmatelibrary/parsers → detectmatelibrary-0.6.2/src/detectmatelibrary/tools}/__init__.py +0 -0
  119. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/logging.py +0 -0
  120. {detectmatelibrary-0.6.0/src/detectmatelibrary/tools → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace}/__init__.py +0 -0
  121. {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates}/__init__.py +0 -0
  122. {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/data}/__init__.py +0 -0
  123. {detectmatelibrary-0.6.0/src/detectmatelibrary/tools/workspace/templates/data → detectmatelibrary-0.6.2/src/detectmatelibrary/tools/workspace/templates/test_templates}/__init__.py +0 -0
  124. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
  125. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/aux.py +0 -0
  126. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/data_buffer.py +0 -0
  127. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/_op.py +0 -0
  128. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/deeplog.py +0 -0
  129. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -0
  130. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/deep_learning/logbert.py +0 -0
  131. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/finetune.py +0 -0
  132. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/id_generator.py +0 -0
  133. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/key_extractor.py +0 -0
  134. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
  135. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/base.py +0 -0
  136. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/__init__.py +0 -0
  137. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/__init__.py +0 -0
  138. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/event_tracker.py +0 -0
  139. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/multi_tracker.py +0 -0
  140. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/base/single_tracker.py +0 -0
  141. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/__init__.py +0 -0
  142. {detectmatelibrary-0.6.0/src/detectmatelibrary/utils/persistency/event_data_structures → detectmatelibrary-0.6.2/src/detectmatelibrary/utils/persistency/data_structures}/trackers/stability/classification_methods.py +0 -0
  143. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
  144. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
  145. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
  146. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
  147. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/requires.txt +0 -0
  148. {detectmatelibrary-0.6.0 → detectmatelibrary-0.6.2}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
@@ -0,0 +1,74 @@
1
+ Metadata-Version: 2.4
2
+ Name: detectmatelibrary
3
+ Version: 0.6.2
4
+ Summary: DetectMate Library for log processing components
5
+ Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
6
+ License-Expression: EUPL-1.2
7
+ Project-URL: Homepage, https://ait-detectmate.github.io/DetectMateLibrary/latest
8
+ Project-URL: Repository, https://github.com/ait-detectmate/DetectMateLibrary
9
+ Requires-Python: >=3.12
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE.md
12
+ Requires-Dist: protobuf>=6.32.1
13
+ Requires-Dist: pydantic>=2.13.3
14
+ Requires-Dist: pyyaml>=6.0.3
15
+ Requires-Dist: regex>=2025.11.3
16
+ Requires-Dist: numpy>=2.3.2
17
+ Requires-Dist: detectmateperformance>=0.1.6
18
+ Requires-Dist: msgpack>=1.0.0
19
+ Requires-Dist: fsspec>=2024.1.0
20
+ Requires-Dist: pyarrow>=24.0.0
21
+ Requires-Dist: jax>=0.11.0
22
+ Requires-Dist: flax>=0.12.8
23
+ Requires-Dist: optax>=0.2.8
24
+ Provides-Extra: llm
25
+ Requires-Dist: openai>=2.26.0; extra == "llm"
26
+ Requires-Dist: tenacity>=9.1.4; extra == "llm"
27
+ Requires-Dist: scipy>=1.17.1; extra == "llm"
28
+ Requires-Dist: scikit-learn>=1.8.0; extra == "llm"
29
+ Requires-Dist: tiktoken>=0.12.0; extra == "llm"
30
+ Requires-Dist: pandas>=2.3.2; extra == "llm"
31
+ Provides-Extra: dataframes
32
+ Requires-Dist: pandas>=2.3.2; extra == "dataframes"
33
+ Requires-Dist: polars>=1.40.1; extra == "dataframes"
34
+ Provides-Extra: polars-rtcompat
35
+ Requires-Dist: polars[rtcompat]>=1.38.1; extra == "polars-rtcompat"
36
+ Provides-Extra: full
37
+ Requires-Dist: detectmatelibrary[llm]; extra == "full"
38
+ Requires-Dist: detectmatelibrary[dataframes]; extra == "full"
39
+ Requires-Dist: detectmatelibrary[polars-rtcompat]; extra == "full"
40
+ Dynamic: license-file
41
+
42
+
43
+ # DetectMateLibrary
44
+
45
+ ![UML](docs/img/logo.png)
46
+
47
+ DetectMate is a flexible, component-based library for parsing and log anomaly detection.
48
+
49
+ ## Installation
50
+
51
+ Download us from `Pip`!!
52
+
53
+ ```bash
54
+ pip install detectmatelibrary
55
+ ```
56
+
57
+ ## Documentation
58
+
59
+ Main documentations:
60
+
61
+ - [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
62
+ - [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
63
+
64
+ ## Contribution
65
+
66
+ We're happily taking patches and other contributions. Please see the following links for how to get started:
67
+
68
+ - [GitHub Workflow](https://ait-detectmate.github.io/DetectMateLibrary/latest/contribution/)
69
+
70
+ If you encounter any bugs, please create an issue on [Github](https://github.com/ait-detectmate/DetectMateLibrary/issues).
71
+
72
+ ## License
73
+
74
+ [EUPL-1.2](https://github.com/ait-detectmate/DetectMateLibrary/blob/main/LICENSE.md)
@@ -0,0 +1,33 @@
1
+
2
+ # DetectMateLibrary
3
+
4
+ ![UML](docs/img/logo.png)
5
+
6
+ DetectMate is a flexible, component-based library for parsing and log anomaly detection.
7
+
8
+ ## Installation
9
+
10
+ Download us from `Pip`!!
11
+
12
+ ```bash
13
+ pip install detectmatelibrary
14
+ ```
15
+
16
+ ## Documentation
17
+
18
+ Main documentations:
19
+
20
+ - [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo)
21
+ - [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main)
22
+
23
+ ## Contribution
24
+
25
+ We're happily taking patches and other contributions. Please see the following links for how to get started:
26
+
27
+ - [GitHub Workflow](https://ait-detectmate.github.io/DetectMateLibrary/latest/contribution/)
28
+
29
+ If you encounter any bugs, please create an issue on [Github](https://github.com/ait-detectmate/DetectMateLibrary/issues).
30
+
31
+ ## License
32
+
33
+ [EUPL-1.2](https://github.com/ait-detectmate/DetectMateLibrary/blob/main/LICENSE.md)
@@ -1,4 +1,5 @@
1
1
  from . import common
2
+ from . import alert_aggregation
2
3
  from . import detectors
3
4
  from . import parsers
4
5
  from . import schemas
@@ -10,6 +11,7 @@ __all__ = [
10
11
  "common",
11
12
  "detectors",
12
13
  "parsers",
14
+ "alert_aggregation",
13
15
  "schemas",
14
16
  "utils",
15
17
  "__authors__",
@@ -0,0 +1,27 @@
1
+ from detectmatelibrary.common.alert_aggregator import (CoreAlertAggregatorConfig, CoreAlertAggregator)
2
+ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
3
+ from typing import Any
4
+
5
+
6
+ class DummyAlertAggregatorConfig(CoreAlertAggregatorConfig):
7
+ method_type: str = "dummy_alert_aggregator"
8
+ buffer_size: int = 3
9
+
10
+
11
+ class DummyAlertAggregator(CoreAlertAggregator):
12
+ def __init__(
13
+ self,
14
+ name: str = "DummyAlertAggregator",
15
+ config: DummyAlertAggregatorConfig | dict[str, Any] = DummyAlertAggregatorConfig(),
16
+ ) -> None:
17
+ if isinstance(config, dict):
18
+ config = DummyAlertAggregatorConfig.from_dict(config, name)
19
+ buffer_size: int = config.buffer_size
20
+ super().__init__(name=name, buffer_size=buffer_size, config=config)
21
+
22
+ def aggregate_alerts(
23
+ self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
24
+ ) -> bool:
25
+ output_["description"] = "Dummy alert aggregation"
26
+ output_["alertDetails"]["type"] = "Anomalies aggregated by DummyAlertAggregator"
27
+ return True
@@ -1,7 +1,7 @@
1
1
  from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
2
2
  from detectmatelibrary.utils.data_buffer import BufferMode
3
3
  from detectmatelibrary import schemas
4
- from typing import List, Any
4
+ from typing import List, Any, Optional
5
5
 
6
6
 
7
7
  class DummyDetectorConfig(CoreDetectorConfig):
@@ -15,12 +15,14 @@ class DummyDetector(CoreDetector):
15
15
  def __init__(
16
16
  self,
17
17
  name: str = "DummyDetector",
18
+ buffer_mode: BufferMode = BufferMode.NO_BUF,
19
+ buffer_size: Optional[int] = None,
18
20
  config: DummyDetectorConfig | dict[str, Any] = DummyDetectorConfig()
19
21
  ) -> None:
20
22
 
21
23
  if isinstance(config, dict):
22
24
  config = DummyDetectorConfig.from_dict(config, name)
23
- super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
25
+ super().__init__(name=name, buffer_mode=buffer_mode, buffer_size=buffer_size, config=config)
24
26
  self._call_count = 0
25
27
 
26
28
  def detect(
@@ -36,5 +38,5 @@ class DummyDetector(CoreDetector):
36
38
  result = pattern[self._call_count % len(pattern)]
37
39
  if result:
38
40
  output_["score"] = 1.0
39
- output_["alertsObtain"]["type"] = "Anomaly detected by DummyDetector"
41
+ output_["alertDetails"]["type"] = "Anomaly detected by DummyDetector"
40
42
  return result
@@ -28,6 +28,6 @@ class DummyParser(CoreParser):
28
28
  output_: schemas.ParserSchema
29
29
  ) -> None:
30
30
 
31
- output_["EventID"] = 2
31
+ output_["eventID"] = 2
32
32
  output_["variables"].extend(["dummy_variable"])
33
33
  output_["template"] = "This is a dummy template"
@@ -0,0 +1,12 @@
1
+ from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig
2
+ from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig
3
+ from .time_delta import TimeDeltaAggregation, TimeDeltaAggregationConfig
4
+
5
+ __all__ = [
6
+ "BasicConcatAggregation",
7
+ "BasicConcatAggregationConfig",
8
+ "TimeDeltaAggregation",
9
+ "TimeDeltaAggregationConfig",
10
+ "LogIDCountAggegation",
11
+ "LogIDCountAggegationConfig",
12
+ ]
@@ -4,24 +4,29 @@ from detectmatelibrary.common.alert_aggregator import (
4
4
 
5
5
  from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
6
6
 
7
- from typing import Any, Optional
7
+ from typing import Any
8
+ from pydantic import Field
8
9
 
9
10
 
10
11
  class BasicConcatAggregationConfig(CoreAlertAggregatorConfig):
11
- method_type: str = "basic_concat_aggregator"
12
- buffer_size: int = 3
12
+ method_type: str = Field(
13
+ default="basic_concat_aggregator", description="Indicates what type of method it is."
14
+ )
15
+ buffer_size: int = Field(
16
+ default=3, description="Buffer size use in the aggregation."
17
+ )
13
18
 
14
19
 
15
20
  class BasicConcatAggregation(CoreAlertAggregator):
16
21
  def __init__(
17
22
  self,
18
23
  name: str = "BasicConcatAggregator",
19
- config: Optional[BasicConcatAggregationConfig | dict[str, Any]] = BasicConcatAggregationConfig(),
24
+ config: BasicConcatAggregationConfig | dict[str, Any] = BasicConcatAggregationConfig(),
20
25
  ) -> None:
21
26
  if isinstance(config, dict):
22
27
  config = BasicConcatAggregationConfig.from_dict(config, name)
23
28
 
24
- buffer_size: int = config.buffer_size # type: ignore
29
+ buffer_size: int = config.buffer_size
25
30
  super().__init__(name=name, buffer_size=buffer_size, config=config)
26
31
 
27
32
  def aggregate_alerts(
@@ -0,0 +1,96 @@
1
+ from detectmatelibrary.common.alert_aggregator import (
2
+ CoreAlertAggregatorConfig, CoreAlertAggregator
3
+ )
4
+
5
+ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
6
+ from detectmatelibrary.utils.data_buffer import BufferMode
7
+
8
+ from typing import Any
9
+ from functools import lru_cache
10
+ from pydantic import Field
11
+ import polars as pl
12
+
13
+
14
+ class Counter:
15
+ def __init__(self, max_count: int, overall_size: int) -> None:
16
+ self.max_count = max_count
17
+
18
+ self._df = pl.DataFrame()
19
+ self._df = self._df.insert_column(0, pl.repeat("", n=overall_size).alias("LogID"))
20
+ self._df = self._df.insert_column(1, pl.zeros(overall_size, dtype=pl.UInt8).alias("Count"))
21
+ self._df = self._df.insert_column(2, pl.zeros(overall_size, dtype=pl.Boolean).alias("WasAlert"))
22
+ self._df = self._df.with_row_index(name="Index")
23
+
24
+ self._cached_search = lru_cache(maxsize=overall_size)(self.__get_idx)
25
+
26
+ self.idx = 0
27
+
28
+ def __is_new_alert(self, idx: int) -> bool:
29
+ if not self._df[idx, "WasAlert"] and self._df[idx, "Count"] >= self.max_count:
30
+ self._df[idx, "WasAlert"] = True
31
+ return True
32
+ return False
33
+
34
+ def __get_idx(self, logId: str) -> int:
35
+ indices = self._df.filter(pl.col("LogID") == logId).select("Index").to_series()
36
+
37
+ if indices.shape[0] == 0:
38
+ new_row = pl.DataFrame(
39
+ data=[[self.idx, logId, 0, False]], schema=self._df.schema, orient="row"
40
+ )
41
+ self._df = self._df.update(new_row, on="Index")
42
+ idx = self.idx
43
+ self.idx = self.idx + 1 if self.idx < self._df.shape[0] - 1 else 0
44
+ else:
45
+ idx = indices[0]
46
+
47
+ return idx
48
+
49
+ def add_and_check(self, logId: str) -> bool:
50
+ idx = self._cached_search(logId)
51
+ self._df[idx, "Count"] += 1
52
+ return self.__is_new_alert(idx)
53
+
54
+
55
+ class LogIDCountAggegationConfig(CoreAlertAggregatorConfig):
56
+ method_type: str = Field(
57
+ default="logid_count_aggregator", description="Indicates what type of method it is."
58
+ )
59
+ max_count: int = Field(default=4, description="Max count of alerts in a specific logID.")
60
+ max_memory: int = Field(
61
+ default=5000, description="Max number of logIDs store in the count memory."
62
+ )
63
+
64
+
65
+ class LogIDCountAggegation(CoreAlertAggregator):
66
+ def __init__(
67
+ self,
68
+ name: str = "LogIDCountAggregator",
69
+ config: LogIDCountAggegationConfig | dict[str, Any] = LogIDCountAggegationConfig(),
70
+ ) -> None:
71
+ if isinstance(config, dict):
72
+ config = LogIDCountAggegationConfig.from_dict(config, name)
73
+
74
+ super().__init__(name=name, config=config, buffer_mode=BufferMode.NO_BUF, buffer_size=None)
75
+ self.config: LogIDCountAggegationConfig
76
+
77
+ self.counter = Counter(
78
+ max_count=self.config.max_count, overall_size=self.config.max_memory
79
+ )
80
+
81
+ def aggregate_alerts(
82
+ self, input_: DetectorSchema, output_: AggregateSchema # type: ignore
83
+ ) -> bool:
84
+
85
+ alert_logs = []
86
+ for logID in input_["logIDs"]:
87
+ if self.counter.add_and_check(logID):
88
+ alert_logs.append(logID)
89
+
90
+ if (aggreated_alerts := len(alert_logs) > 0):
91
+ output_["description"] = (
92
+ "LogID count how many times a log was count as alert."
93
+ f"Logs that pass the limit count: {", ".join(alert_logs)}"
94
+ )
95
+
96
+ return aggreated_alerts
@@ -0,0 +1,111 @@
1
+ from collections.abc import Callable
2
+ from typing import Any
3
+
4
+ from pydantic import Field, NonNegativeFloat
5
+
6
+ from detectmatelibrary.base_alert_aggregation.density_clustering import (
7
+ DensityClusteringAggregation,
8
+ DensityClusteringAggregationConfig,
9
+ )
10
+ from detectmatelibrary.schemas import AggregateSchema, DetectorSchema
11
+
12
+
13
+ class TimeDeltaAggregationConfig(DensityClusteringAggregationConfig):
14
+ """Configuration of the TimeDeltaAggregation class.
15
+
16
+ Args:
17
+ delta (NonNegativeFloat): The time threshold below which two alerts are allocated to the same group.
18
+ """
19
+
20
+ method_type: str = Field(
21
+ default="time_delta_aggregator",
22
+ description="The kind of aggregation method.",
23
+ )
24
+ delta: NonNegativeFloat = Field(
25
+ default=10.0, description="The threshold for the clustering."
26
+ )
27
+
28
+
29
+ class TimeDeltaAggregation(DensityClusteringAggregation):
30
+ """This class implements alert aggregation by the time-delta method introduced by
31
+ [Landauer et al.](https://doi.org/10.1145/3510581).
32
+ Two alerts will be assigned to the same group if the time difference
33
+ between them is below a certain threshold delta.
34
+ The time difference is computed based on the timestamps of the last logs associated
35
+ with each alert as these determine the earliest time at which the alert was detected.
36
+ Important: It is assumed that the alerts arrive in order of their timestamps!
37
+ """
38
+
39
+ def __init__(
40
+ self,
41
+ config: TimeDeltaAggregationConfig | dict[str, Any],
42
+ name: str = "TimeDeltaAggregator",
43
+ ) -> None:
44
+ """Initialize the TimeDeltaAggregation class.
45
+
46
+ Args:
47
+ name (str): The name identifying this aggregation method.
48
+ config (TimeDeltaAggregationConfig): The configuration of this method.
49
+ """
50
+ if isinstance(config, dict):
51
+ config = TimeDeltaAggregationConfig.from_dict(config, name)
52
+
53
+ super().__init__(name=name, config=config)
54
+ self.timestamp_extraction_fn: Callable[[DetectorSchema], float] = lambda alert: (
55
+ float(alert["logTimestamps"][-1])
56
+ )
57
+ self.description = "Alert aggregation by the time-delta method."
58
+ self.last_timestamp: float = 0.0
59
+
60
+ def aggregate_alerts(
61
+ self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
62
+ ) -> bool:
63
+ """Aggregate input alerts and write the results to the output.
64
+
65
+ Args:
66
+ input_: The alert or alerts to aggregate.
67
+ output_: The schema to which aggregation results are written.
68
+
69
+ Returns:
70
+ ``True`` after the aggregation results have been written.
71
+ ``False`` if no new alerts were aggregated and the output schema was not modified.
72
+ """
73
+
74
+ if not isinstance(input_, list):
75
+ input_ = [input_]
76
+
77
+ if self.timestamp_extraction_fn(input_[0]) < self.last_timestamp:
78
+ raise ValueError("Alerts must be aggregated in order of their timestamps!")
79
+ else:
80
+ self.last_timestamp = self.timestamp_extraction_fn(input_[-1])
81
+
82
+ return super().aggregate_alerts(input_, output_)
83
+
84
+ def single_metric(
85
+ self,
86
+ alert1: DetectorSchema,
87
+ alert2: DetectorSchema,
88
+ ) -> NonNegativeFloat:
89
+ """Compute the time difference between two alerts.
90
+
91
+ Args:
92
+ alert1 (DetectorSchema): The first alert.
93
+ alert2 (DetectorSchema): The second alert.
94
+ Returns:
95
+ NonNegativeFloat: The time difference between the two alerts.
96
+ """
97
+ return abs(
98
+ self.timestamp_extraction_fn(alert1) - self.timestamp_extraction_fn(alert2)
99
+ )
100
+
101
+ def discard_alert_fn(self, alerts: list[DetectorSchema]) -> list[bool]:
102
+ """Discards all but the last alert in a list of alerts as this is the
103
+ only one relevant for the time-delta method.
104
+
105
+ Args:
106
+ alerts (list[DetectorSchema]): The list of alerts to discard from.
107
+
108
+ Returns:
109
+ list[bool]: A list of booleans indicating which alerts to discard.
110
+ """
111
+ return [i + 1 != len(alerts) for i in range(len(alerts))]
@@ -0,0 +1,9 @@
1
+ from .density_clustering import (
2
+ DensityClusteringAggregation,
3
+ DensityClusteringAggregationConfig,
4
+ )
5
+
6
+ __all__ = [
7
+ "DensityClusteringAggregation",
8
+ "DensityClusteringAggregationConfig",
9
+ ]