detectmatelibrary 0.6.2__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {detectmatelibrary-0.6.2/src/detectmatelibrary.egg-info → detectmatelibrary-0.7.0}/PKG-INFO +13 -10
  2. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/README.md +1 -1
  3. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/pyproject.toml +23 -11
  4. detectmatelibrary-0.7.0/src/detectmatelibrary/alert_aggregation/__init__.py +29 -0
  5. detectmatelibrary-0.7.0/src/detectmatelibrary/alert_aggregation/alertbert.py +447 -0
  6. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/time_delta.py +1 -1
  7. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/alert_aggregator.py +1 -0
  8. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/core.py +9 -3
  9. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/ecvc_detector.py +18 -11
  10. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/helper/select_component.py +1 -1
  11. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/metadata.py +1 -1
  12. detectmatelibrary-0.7.0/src/detectmatelibrary/parsers/__init__.py +36 -0
  13. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/additional_cluster.py +10 -15
  14. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/cluster.py +20 -16
  15. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/matching.py +11 -9
  16. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parser.py +17 -6
  17. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_base.py +46 -31
  18. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/parsing_cache.py +63 -57
  19. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/postprocess.py +26 -23
  20. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/sample.py +10 -11
  21. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/util.py +55 -50
  22. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/vars.py +3 -2
  23. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/_matcher_op.py +7 -11
  24. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/_parser.py +6 -7
  25. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/schemas/__init__.py +3 -1
  26. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/schemas/_classes.py +97 -36
  27. detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/__init__.py +24 -0
  28. detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/default.py +95 -0
  29. detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/mongos.py +117 -0
  30. detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/connectors/redis.py +90 -0
  31. {detectmatelibrary-0.6.2/src/detectmatelibrary/schemas → detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/protobuf}/_op.py +1 -1
  32. {detectmatelibrary-0.6.2/src/detectmatelibrary/schemas → detectmatelibrary-0.7.0/src/detectmatelibrary/schemas/protobuf}/schemas_pb2.py +2 -2
  33. detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/datasets.py +364 -0
  34. detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/models.py +1700 -0
  35. detectmatelibrary-0.7.0/src/detectmatelibrary/utils/alertbert/preprocessing.py +325 -0
  36. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/_op.py +14 -4
  37. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/deeplog.py +26 -12
  38. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/logbert.py +31 -10
  39. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/persistency_saver.py +0 -4
  40. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0/src/detectmatelibrary.egg-info}/PKG-INFO +13 -10
  41. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/SOURCES.txt +10 -3
  42. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/requires.txt +10 -8
  43. detectmatelibrary-0.6.2/src/detectmatelibrary/alert_aggregation/__init__.py +0 -12
  44. detectmatelibrary-0.6.2/src/detectmatelibrary/parsers/__init__.py +0 -23
  45. detectmatelibrary-0.6.2/src/detectmatelibrary/utils/data_normalizer.py +0 -31
  46. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/LICENSE.md +0 -0
  47. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/setup.cfg +0 -0
  48. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/__init__.py +0 -0
  49. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/__init__.py +0 -0
  50. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_alert_aggregator.py +0 -0
  51. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_detector.py +0 -0
  52. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/_testutils/dummy_parser.py +0 -0
  53. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/basic_concat.py +0 -0
  54. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/alert_aggregation/logid_count.py +0 -0
  55. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_alert_aggregation/__init__.py +0 -0
  56. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_alert_aggregation/density_clustering.py +0 -0
  57. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/__init__.py +0 -0
  58. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_persist.py +0 -0
  59. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_stability.py +0 -0
  60. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/_variable_hooks.py +0 -0
  61. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/deeplearning_detector.py +0 -0
  62. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/tracker_detector.py +0 -0
  63. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/base_detectors/variable_detector.py +0 -0
  64. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/__init__.py +0 -0
  65. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/__init__.py +0 -0
  66. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/_compile.py +0 -0
  67. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_config/_formats.py +0 -0
  68. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_basic_component.py +0 -0
  69. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_fed_component.py +0 -0
  70. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_fit_logic.py +0 -0
  71. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_schema_pipeline.py +0 -0
  72. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/_core_op/_train_buffer.py +0 -0
  73. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/detector.py +0 -0
  74. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/common/parser.py +0 -0
  75. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/constants.py +0 -0
  76. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/__init__.py +0 -0
  77. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/bigram_frequency_detector.py +0 -0
  78. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/charset_detector.py +0 -0
  79. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/deeplog_detector.py +0 -0
  80. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/event_sequence_detector.py +0 -0
  81. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/logbert_detector.py +0 -0
  82. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_event_detector.py +0 -0
  83. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_value_combo_detector.py +0 -0
  84. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/new_value_detector.py +0 -0
  85. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/random_detector.py +0 -0
  86. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/rule_detector.py +0 -0
  87. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/scvs_detector.py +0 -0
  88. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/detectors/value_range_detector.py +0 -0
  89. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/helper/from_to.py +0 -0
  90. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/autoparser.py +0 -0
  91. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/drain.py +0 -0
  92. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/json_parser.py +0 -0
  93. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/__init__.py +0 -0
  94. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/engine/__init__.py +0 -0
  95. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/logbatcher/parser.py +0 -0
  96. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/template_matcher/__init__.py +0 -0
  97. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/parsers/tree_matcher.py +0 -0
  98. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/pipeline.py +0 -0
  99. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/__init__.py +0 -0
  100. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/logging.py +0 -0
  101. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/__init__.py +0 -0
  102. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/create_workspace.py +0 -0
  103. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomAlertAggregator.py +0 -0
  104. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomDetector.py +0 -0
  105. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/CustomParser.py +0 -0
  106. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/__init__.py +0 -0
  107. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/__init__.py +0 -0
  108. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/alerts.json +0 -0
  109. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/logs.json +0 -0
  110. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/data/parsed_log.json +0 -0
  111. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/__init__.py +0 -0
  112. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +0 -0
  113. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomDetector.py +0 -0
  114. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomParser.py +0 -0
  115. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/tools/workspace/utils.py +0 -0
  116. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/aux.py +0 -0
  117. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/data_buffer.py +0 -0
  118. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/deep_learning/imodel.py +0 -0
  119. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/finetune.py +0 -0
  120. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/id_generator.py +0 -0
  121. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/key_extractor.py +0 -0
  122. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/log_format_utils.py +0 -0
  123. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/__init__.py +0 -0
  124. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/basic_persistency.py +0 -0
  125. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/base.py +0 -0
  126. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/__init__.py +0 -0
  127. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/__init__.py +0 -0
  128. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/event_tracker.py +0 -0
  129. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/multi_tracker.py +0 -0
  130. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/base/single_tracker.py +0 -0
  131. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/rle_list.py +0 -0
  132. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/__init__.py +0 -0
  133. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/classification_methods.py +0 -0
  134. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_classifier.py +0 -0
  135. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/data_structures/trackers/stability/stability_tracker.py +0 -0
  136. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/event_persistency.py +0 -0
  137. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/persistency/slow_persistency.py +0 -0
  138. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/preview_helpers.py +0 -0
  139. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/sequence_encoding.py +0 -0
  140. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary/utils/time_format_handler.py +0 -0
  141. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/dependency_links.txt +0 -0
  142. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/entry_points.txt +0 -0
  143. {detectmatelibrary-0.6.2 → detectmatelibrary-0.7.0}/src/detectmatelibrary.egg-info/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectmatelibrary
3
- Version: 0.6.2
3
+ Version: 0.7.0
4
4
  Summary: DetectMate Library for log processing components
5
- Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
5
+ Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf, Lukas Karner
6
6
  License-Expression: EUPL-1.2
7
7
  Project-URL: Homepage, https://ait-detectmate.github.io/DetectMateLibrary/latest
8
8
  Project-URL: Repository, https://github.com/ait-detectmate/DetectMateLibrary
@@ -21,6 +21,10 @@ Requires-Dist: pyarrow>=24.0.0
21
21
  Requires-Dist: jax>=0.11.0
22
22
  Requires-Dist: flax>=0.12.8
23
23
  Requires-Dist: optax>=0.2.8
24
+ Requires-Dist: polars>=1.40.1
25
+ Requires-Dist: polars[rtcompat]>=1.38.1
26
+ Requires-Dist: pymongo>=4.19.0
27
+ Requires-Dist: redis[hiredis]>=8.1.0
24
28
  Provides-Extra: llm
25
29
  Requires-Dist: openai>=2.26.0; extra == "llm"
26
30
  Requires-Dist: tenacity>=9.1.4; extra == "llm"
@@ -28,21 +32,20 @@ Requires-Dist: scipy>=1.17.1; extra == "llm"
28
32
  Requires-Dist: scikit-learn>=1.8.0; extra == "llm"
29
33
  Requires-Dist: tiktoken>=0.12.0; extra == "llm"
30
34
  Requires-Dist: pandas>=2.3.2; extra == "llm"
31
- Provides-Extra: dataframes
32
- Requires-Dist: pandas>=2.3.2; extra == "dataframes"
33
- Requires-Dist: polars>=1.40.1; extra == "dataframes"
34
- Provides-Extra: polars-rtcompat
35
- Requires-Dist: polars[rtcompat]>=1.38.1; extra == "polars-rtcompat"
35
+ Provides-Extra: alertbert
36
+ Requires-Dist: torch; extra == "alertbert"
37
+ Requires-Dist: tensordict; extra == "alertbert"
38
+ Requires-Dist: scikit-learn; extra == "alertbert"
39
+ Requires-Dist: scipy; extra == "alertbert"
36
40
  Provides-Extra: full
37
41
  Requires-Dist: detectmatelibrary[llm]; extra == "full"
38
- Requires-Dist: detectmatelibrary[dataframes]; extra == "full"
39
- Requires-Dist: detectmatelibrary[polars-rtcompat]; extra == "full"
42
+ Requires-Dist: detectmatelibrary[alertbert]; extra == "full"
40
43
  Dynamic: license-file
41
44
 
42
45
 
43
46
  # DetectMateLibrary
44
47
 
45
- ![UML](docs/img/logo.png)
48
+ ![UML](docs/img/logo.svg)
46
49
 
47
50
  DetectMate is a flexible, component-based library for parsing and log anomaly detection.
48
51
 
@@ -1,7 +1,7 @@
1
1
 
2
2
  # DetectMateLibrary
3
3
 
4
- ![UML](docs/img/logo.png)
4
+ ![UML](docs/img/logo.svg)
5
5
 
6
6
  DetectMate is a flexible, component-based library for parsing and log anomaly detection.
7
7
 
@@ -4,7 +4,7 @@ description = "DetectMate Library for log processing components"
4
4
  readme = {file = "README.md", content-type = "text/markdown"}
5
5
  dynamic = ["version"]
6
6
  authors = [{"name" = "André García Gómez"}, {"name" = "Viktor Beck"}, {"name" = "Thorina Boenke"},
7
- {"name" = "Wolfgang Hotwagner"}, {"name" = "Anna Erdi"}, {"name" = "Ernst Leierzopf"}]
7
+ {"name" = "Wolfgang Hotwagner"}, {"name" = "Anna Erdi"}, {"name" = "Ernst Leierzopf"}, {"name" = "Lukas Karner"}]
8
8
  license = "EUPL-1.2"
9
9
  requires-python = ">=3.12"
10
10
  dependencies = [
@@ -20,6 +20,10 @@ dependencies = [
20
20
  "jax>=0.11.0",
21
21
  "flax>=0.12.8",
22
22
  "optax>=0.2.8",
23
+ "polars>=1.40.1",
24
+ "polars[rtcompat]>=1.38.1",
25
+ "pymongo>=4.19.0",
26
+ "redis[hiredis]>=8.1.0",
23
27
  ]
24
28
 
25
29
  [tool.pytest.ini_options]
@@ -46,18 +50,16 @@ llm = [
46
50
  "tiktoken>=0.12.0",
47
51
  "pandas>=2.3.2",
48
52
  ]
49
- dataframes = [
50
- "pandas>=2.3.2",
51
- "polars>=1.40.1",
52
- ]
53
- polars-rtcompat = [
54
- "polars[rtcompat]>=1.38.1",
53
+ alertbert = [
54
+ "torch",
55
+ "tensordict",
56
+ "scikit-learn",
57
+ "scipy",
55
58
  ]
56
59
  # install all optional extras at once: uv pip install detectmatelibrary[full]
57
60
  full = [
58
61
  "detectmatelibrary[llm]",
59
- "detectmatelibrary[dataframes]",
60
- "detectmatelibrary[polars-rtcompat]",
62
+ "detectmatelibrary[alertbert]",
61
63
  ]
62
64
 
63
65
  [tool.setuptools.dynamic]
@@ -68,7 +70,7 @@ Homepage = "https://ait-detectmate.github.io/DetectMateLibrary/latest"
68
70
  Repository = "https://github.com/ait-detectmate/DetectMateLibrary"
69
71
 
70
72
  [build-system]
71
- requires = ["setuptools>=82.0.1", "wheel"]
73
+ requires = ["setuptools>=84.1.0", "wheel", "packaging>=24.2"]
72
74
  build-backend = "setuptools.build_meta"
73
75
 
74
76
  [tool.setuptools.packages.find]
@@ -78,7 +80,7 @@ where = ["src"]
78
80
  mate = "detectmatelibrary.tools.workspace.create_workspace:main"
79
81
 
80
82
  [[tool.mypy.overrides]]
81
- module = "detectmatelibrary.schemas.schemas_pb2"
83
+ module = "detectmatelibrary.schemas.protobuf.schemas_pb2"
82
84
  ignore_errors = true
83
85
 
84
86
  [tool.setuptools]
@@ -89,3 +91,13 @@ license-files = ["LICENSE.md"]
89
91
  detectmatelibrary = [
90
92
  "tools/workspace/templates/data/*.json"
91
93
  ]
94
+
95
+ [tool.uv.sources]
96
+ torch = [
97
+ { index = "torch-cpu", extra = "alertbert" },
98
+ ]
99
+
100
+ [[tool.uv.index]]
101
+ name = "torch-cpu"
102
+ url = "https://download.pytorch.org/whl/cpu"
103
+ explicit = true
@@ -0,0 +1,29 @@
1
+ # mypy: ignore-errors
2
+
3
+ from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig
4
+ from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig
5
+ from .time_delta import TimeDeltaAggregation, TimeDeltaAggregationConfig
6
+
7
+ __all__ = [
8
+ "BasicConcatAggregation",
9
+ "BasicConcatAggregationConfig",
10
+ "TimeDeltaAggregation",
11
+ "TimeDeltaAggregationConfig",
12
+ "LogIDCountAggegation",
13
+ "LogIDCountAggegationConfig",
14
+ ]
15
+
16
+
17
+ def __getattr__(name):
18
+ """Load AlertBERT only when one of its exports is requested."""
19
+ if name in {"AlertBERTAggregation", "AlertBERTAggregationConfig"}:
20
+ from .alertbert import AlertBERTAggregation, AlertBERTAggregationConfig
21
+
22
+ # Cache both exports for subsequent attribute access.
23
+ globals().update(
24
+ AlertBERTAggregation=AlertBERTAggregation,
25
+ AlertBERTAggregationConfig=AlertBERTAggregationConfig,
26
+ )
27
+ return globals()[name]
28
+
29
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,447 @@
1
+ from collections.abc import Callable
2
+ from datetime import datetime as dt
3
+ from typing import Any, Literal, cast
4
+
5
+ from pydantic import Field, NonNegativeFloat
6
+
7
+ from detectmatelibrary.base_alert_aggregation import (
8
+ DensityClusteringAggregation,
9
+ DensityClusteringAggregationConfig,
10
+ )
11
+ from detectmatelibrary.schemas import AggregateSchema, DetectorSchema
12
+ from detectmatelibrary.tools.logging import logger
13
+ from detectmatelibrary.utils.alertbert.datasets import AlertDataset, split_alert_dataset
14
+ from detectmatelibrary.utils.alertbert.models import (
15
+ AlertBERT,
16
+ MaskedLanguageModel,
17
+ )
18
+ from detectmatelibrary.utils.data_buffer import BufferMode
19
+
20
+
21
+ class AlertBERTAggregationConfig(DensityClusteringAggregationConfig):
22
+ """Configuration of the AlertBERTAggregation class.
23
+
24
+ Besides the parameters of the alert aggregation this configuration
25
+ also determines the training of the masked-languge-model used to
26
+ obtain the alert embeddings.
27
+ """
28
+ # general
29
+ method_type: str = Field(
30
+ default="AlertBERTAggregation",
31
+ description="The type of the aggregation method. Defaults to 'AlertBERT_aggregator'.",
32
+ )
33
+ _model_id: str = f"{method_type.default}_{dt.now().isoformat()}" # type: ignore[attr-defined]
34
+ model_id: str = Field(
35
+ default=_model_id,
36
+ description="""The unique identifier of the model.
37
+ Defaults to 'AlertBERT_aggregator_<current_datetime>'.""",
38
+ )
39
+ path: str = Field(
40
+ default="alertbert_models",
41
+ description="The path to the save location of the AlertBERT model. Defaults to 'alertbert_models'.",
42
+ )
43
+ train_model: bool = Field(
44
+ default=True,
45
+ description="""Whether to train the AlertBERT model.
46
+ If False, the model will be loaded from disk.
47
+ Defaults to False.""",
48
+ )
49
+ save_model: bool = Field(
50
+ default=True,
51
+ description="""Whether to save the trained AlertBERT model to disk.
52
+ Defaults to True.""",
53
+ )
54
+
55
+ # aggregation
56
+ delta: NonNegativeFloat = Field(
57
+ default=10.0,
58
+ description="The threshold for the clustering. Defaults to 10.0."
59
+ )
60
+ theta: NonNegativeFloat = Field(
61
+ default=20.0,
62
+ description="The parameter for the time-cosine-distance. Defaults to 20.0.",
63
+ )
64
+ dim_reduction: int = Field(
65
+ default=2,
66
+ description="""The dimension to which the alert embeddings will be reduced via PCA before clustering.
67
+ Defaults to 2."""
68
+ )
69
+ clustering_backend: Literal["detectmate", "scipy"] = Field(
70
+ default="detectmate",
71
+ description="""The backend to use for clustering.
72
+ The option "detectmate" uses the implementation inherited from the parent class,
73
+ the option "scipy" uses the implementation of the AlertBERT model.
74
+ Defaults to "detectmate".""",
75
+ )
76
+ layers: tuple[str] | tuple[str, str] = Field(
77
+ default=("embedding", "encoder"),
78
+ description="""The layers of the AlertBERT model to use for obtaining the alert embeddings.
79
+ Defaults to ("embedding", "encoder").""",
80
+ )
81
+ buffer_size: int = Field(
82
+ default=1000,
83
+ description="The size of the buffer. Defaults to 1000."
84
+ )
85
+ buffer_mode: BufferMode = Field(
86
+ default=BufferMode.BATCH,
87
+ description="The mode of the buffer. Defaults to 'BATCH'.",
88
+ )
89
+
90
+ # data
91
+ train_size: int = Field(
92
+ default=50000,
93
+ description="The number of alerts to use for training the AlertBERT model. Defaults to 50000."
94
+ )
95
+ val_size: int = Field(
96
+ default=10000,
97
+ description="""The number of alerts to use for validation during training of the AlertBERT model.
98
+ Defaults to 10000."""
99
+ )
100
+ skip_first: int = Field(
101
+ default=0,
102
+ description="""The number of alerts to skip before starting to collect alerts for training the
103
+ AlertBERT model.
104
+ Defaults to 0."""
105
+ )
106
+ min_freq: int = Field(
107
+ default=10,
108
+ description="""The minimum frequency of tokens to be included in the
109
+ vocabulary of the AlertBERT model.
110
+ Defaults to 10.""",
111
+ )
112
+
113
+ # model
114
+ features: tuple[str, ...] = Field(
115
+ default=("description",),
116
+ description="""The keys of fields of the alerts to use as features for training the AlertBERT model.
117
+ Fields within the 'alertDetails' field can be accessed by the syntax 'alertDetails:fieldName'.
118
+ Defaults to ("description",).""",
119
+ )
120
+ encoding: Literal["index", "relevantTimestamp"] = Field(
121
+ default="relevantTimestamp",
122
+ description="""The features of the alerts to use for rotary positional encoding.
123
+ Defaults to "relevantTimestamp".""",
124
+ )
125
+ n_heads: int = Field(
126
+ default=4,
127
+ description="The number of attention heads in the AlertBERT model. Defaults to 4."
128
+ )
129
+ dim_per_head: int = Field(
130
+ default=16,
131
+ description="The dimensionality of each attention head in the AlertBERT model. Defaults to 16.",
132
+ )
133
+ num_layers: int = Field(
134
+ default=1,
135
+ description="The number of layers in the model in the AlertBERT model. Defaults to 1."
136
+ )
137
+ feedforward_factor: int = Field(
138
+ default=4,
139
+ description="""The factor by which the feedforward dimension is larger than the model dimension
140
+ in the AlertBERT model. Defaults to 4.""",
141
+ )
142
+ rotary_max_exp: int = Field(
143
+ default=14,
144
+ description="""The maximum exponent for the rotary positional encoding in the AlertBERT model.
145
+ Defaults to 14.""",
146
+ )
147
+ rotary_cutoff: float = Field(
148
+ default=0.75,
149
+ description="""The cutoff for the rotary positional encoding in the AlertBERT model.
150
+ Defaults to 0.75.""",
151
+ )
152
+
153
+ # training
154
+ context_size: int = Field(
155
+ default=1024,
156
+ description="The context size for the AlertBERT model. Defaults to 1024."
157
+ )
158
+ batch_size: int = Field(
159
+ default=8,
160
+ description="The batch size for training the AlertBERT model. Defaults to 8."
161
+ )
162
+ updates: tuple[int, int] = Field(
163
+ default=(240, 300),
164
+ description="The number of updates for training the AlertBERT model. Defaults to (240, 300).",
165
+ )
166
+ lr: float = Field(
167
+ default=5e-3,
168
+ description="The learning rate for training the AlertBERT model. Defaults to 5e-3.",
169
+ )
170
+ warm_up_steps: int = Field(
171
+ default=20,
172
+ description="The number of warm-up steps for training the AlertBERT model. Defaults to 100."
173
+ )
174
+
175
+ def __init__(self, **data: Any) -> None:
176
+ """Initialize the AlertBERTAggregationConfig class.
177
+
178
+ Args:
179
+ **data (Any): The data to initialize the configuration with.
180
+ """
181
+ super().__init__(**data)
182
+ if self.theta <= self.delta:
183
+ logger.warning(
184
+ f"The parameter theta ({self.theta}) should be larger than delta ({
185
+ self.delta
186
+ }). For theta <= delta, AlertBERT is equivalent to TimeDelta!"
187
+ )
188
+ if self.buffer_size > self.train_size + self.val_size:
189
+ logger.info(
190
+ f"The buffer size {
191
+ self.buffer_size
192
+ } is larger than the training and validation size ({
193
+ self.train_size + self.val_size
194
+ })."
195
+ )
196
+ if self.train_size < self.context_size * self.batch_size:
197
+ logger.error(
198
+ f"The training size {self.train_size} is smaller than context size ({
199
+ self.context_size
200
+ }) * batch size ({
201
+ self.batch_size
202
+ }). This will lead to empty batches during training!"
203
+ )
204
+
205
+ def _get_mlm_config(self) -> dict[str, Any]:
206
+ return self.model_dump(
207
+ exclude={
208
+ "buffer_mode",
209
+ "params",
210
+ "component_type",
211
+ "auto_config",
212
+ "start_id",
213
+ "data_use_training",
214
+ "data_use_configure",
215
+ "use_config_data_as_training",
216
+ "buffer_size",
217
+ "delta",
218
+ "theta",
219
+ "dim_reduciton",
220
+ "clustering_backend",
221
+ "layers",
222
+ }
223
+ )
224
+
225
+
226
+ class AlertBERTAggregation(DensityClusteringAggregation):
227
+ """Alert Aggregation based on alert timestamps and embeddings obtained from masked-language-models as
228
+ proposed by [AlertBERT: A noise-robust alert grouping framework for simultaneous cyber attacks]
229
+ (https://arxiv.org/abs/2602.06534).
230
+ The alerts will be assigned to the same group if their time-cosine-distance is below a
231
+ certain threshold theta.
232
+ The time-cosine-distance is the maximum of the time difference and the cosine similarity between
233
+ the embeddings of each alert.
234
+ """
235
+
236
+ def __init__(
237
+ self,
238
+ name: str = "AlertBERTAggregation",
239
+ config: AlertBERTAggregationConfig = AlertBERTAggregationConfig(),
240
+ ) -> None:
241
+ """Initialize the AlertBERTAggregation class.
242
+
243
+ Args:
244
+ config (AlertBERTAggregationConfig): The configuration of this method.
245
+ name (str): The name identifying this aggregation method.
246
+ """
247
+ if isinstance(config, dict):
248
+ config = AlertBERTAggregationConfig.from_dict(config, name)
249
+ super().__init__(name=name, config=config)
250
+ self.config: AlertBERTAggregationConfig # ty: ignore[invalid-mutable-override]
251
+ self.description = "Alert aggregation by the AlertBERT method."
252
+ self._set_up_mlm()
253
+ if self.config.train_model:
254
+ self.collected_data: list[DetectorSchema] = []
255
+ self.timestamp_extraction_fn: Callable[[DetectorSchema], float] = lambda alert: (
256
+ float(alert["logTimestamps"][-1])
257
+ )
258
+ self.last_timestamp: float = 0.0
259
+
260
+ def aggregate_alerts(
261
+ self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema
262
+ ) -> bool:
263
+ """Aggregate alerts into groups using the AlertBERT method. If the
264
+ model is set to be trained, it will first collect enough alerts to
265
+ train the model, and then proceed with the aggregation. If the model is
266
+ already trained, it will directly aggregate the alerts using the
267
+ specified clustering backend.
268
+
269
+ Args:
270
+ input_ (list[DetectorSchema] | DetectorSchema): The alerts to aggregate.
271
+ output_ (AggregateSchema): The output schema to populate.
272
+
273
+ Returns:
274
+ bool: True if the alerts were aggregated, False otherwise.
275
+ """
276
+ if not isinstance(input_, list):
277
+ input_ = [input_]
278
+ if self.timestamp_extraction_fn(input_[0]) < self.last_timestamp:
279
+ raise ValueError("Alerts must be given in order of their timestamps!")
280
+ else:
281
+ self.last_timestamp = self.timestamp_extraction_fn(input_[-1])
282
+
283
+ if self.config.train_model:
284
+ self.collected_data.extend(input_)
285
+ if (
286
+ len(self.collected_data)
287
+ >= self.config.skip_first
288
+ + self.config.train_size
289
+ + self.config.val_size
290
+ ):
291
+ logger.info("Data collection complete.")
292
+ self._train_mlm()
293
+ del self.collected_data
294
+ logger.info("Proceeding with alert aggregation ...")
295
+ self.last_timestamp = 0.0
296
+ return self.aggregate_alerts(input_, output_)
297
+ else:
298
+ return False
299
+ else:
300
+ if self.config.clustering_backend == "detectmate":
301
+ return super().aggregate_alerts(input_, output_)
302
+ elif self.config.clustering_backend == "scipy":
303
+ return self._aggregate_alerts(input_, output_)
304
+ else:
305
+ raise ValueError(
306
+ f"Clustering backend '{self.config.clustering_backend}' is not supported."
307
+ )
308
+
309
+ def _aggregate_alerts(
310
+ self, input_: list[DetectorSchema], output_: AggregateSchema
311
+ ) -> bool:
312
+ """Aggregate alerts into groups using the AlertBERT method with the
313
+ scipy backend.
314
+
315
+ Args:
316
+ input_ (list[DetectorSchema]): The alerts to aggregate.
317
+ output_ (AggregateSchema): The output schema to populate.
318
+
319
+ Returns:
320
+ bool: True if the alerts were aggregated, False otherwise.
321
+ """
322
+ self._add_new_alerts(input_)
323
+ labels = self.alertbert(self.relevant_alerts)
324
+
325
+ # keep track of which labels belong to which group_id's
326
+ label_to_group_id: dict[int, int] = {}
327
+
328
+ # first map labels to groups
329
+ # then assign new alerts to groups
330
+ for alert, label in zip(self.relevant_alerts, labels, strict=True):
331
+ # old alerts have group_id's already
332
+ if (
333
+ current_group_id := self.alert_id_to_group_id[alert["alertID"]]
334
+ ) is not None:
335
+ # check which group_id the labels belong to and merge them if the labels say so
336
+ if label not in label_to_group_id:
337
+ label_to_group_id[label] = current_group_id
338
+ else:
339
+ other_group_id = label_to_group_id[label]
340
+ if other_group_id != current_group_id:
341
+ self._check_and_merge_groups({other_group_id, current_group_id})
342
+ # new alerts have no group_id's yet, assign them to the group of the label
343
+ else:
344
+ # a new label requires creating an new group
345
+ if label not in label_to_group_id:
346
+ group_id = self._new_group(alert["alertID"])
347
+ label_to_group_id[label] = group_id
348
+ # otherwise assign the new alert to the group of the label
349
+ else:
350
+ self.alert_id_to_group_id[alert["alertID"]] = label_to_group_id[
351
+ label
352
+ ]
353
+
354
+ # write results to output
355
+ self._write_output(input_, output_)
356
+ return True
357
+
358
+ def metric(
359
+ self,
360
+ alerts: list[DetectorSchema],
361
+ return_last_rows: int = 0,
362
+ ) -> list[list[NonNegativeFloat]]:
363
+ """Compute the time-cosine-distance between the embeddings of the
364
+ alerts.
365
+
366
+ Args:
367
+ alerts (list[DetectorSchema]): The alerts for which to compute the metric.
368
+ return_last_rows (int): The number of last rows to return. Defaults to 0.
369
+
370
+ Returns:
371
+ list[list[NonNegativeFloat]]: The time-cosine-distances between the embeddings of the alerts.
372
+ """
373
+ alert_emb = self.alertbert._get_embeddings(alerts)
374
+ return cast(
375
+ list[list[NonNegativeFloat]],
376
+ self.alertbert.time_cosine_metric(
377
+ alert_emb[-return_last_rows:], alert_emb
378
+ ).tolist(),
379
+ )
380
+
381
+ def discard_alert_fn(self, alerts: list[DetectorSchema]) -> list[bool]:
382
+ """Determine which alerts should be discarded based on their
383
+ timestamps.
384
+
385
+ Args:
386
+ alerts (list[DetectorSchema]): The alerts to check for discarding.
387
+
388
+ Returns:
389
+ list[bool]: A list of booleans indicating whether each alert should be discarded.
390
+ """
391
+ cutoff = self.relevant_alerts[-1]["logTimestamps"][-1] - self.config.delta
392
+ return [alert["logTimestamps"][-1] < cutoff for alert in alerts]
393
+
394
+ def _set_up_mlm(self) -> None:
395
+ """Set up the MaskedLanguageModel."""
396
+ self.mlm = MaskedLanguageModel(self.config)
397
+ if not self.config.train_model:
398
+ logger.info("Loading AlertBERT model from disk.")
399
+ self._set_up_alertbert()
400
+ else:
401
+ logger.debug("AlertBERT model will be trained ...")
402
+
403
+ def _set_up_alertbert(self) -> None:
404
+ """Set up the AlertBERT model."""
405
+ self.alertbert = AlertBERT(
406
+ model=self.mlm,
407
+ config=self.config,
408
+ )
409
+ logger.info("AlertBERT model setup complete.")
410
+
411
+ def _train_mlm(
412
+ self,
413
+ use_gathered_alerts: bool = True,
414
+ train_data: AlertDataset | None = None,
415
+ val_data: AlertDataset | None = None,
416
+ ) -> None:
417
+ """Train the MaskedLanguageModel.
418
+
419
+ Args:
420
+ use_gathered_alerts (bool): Whether to use the gathered alerts for training. Defaults to True.
421
+ train_data (AlertDataset | None): The training dataset. Required if use_gathered_alerts is False.
422
+ val_data (AlertDataset | None): The validation dataset. Required if use_gathered_alerts is False.
423
+ Raises:
424
+ ValueError: If use_gathered_alerts is False and train_data or val_data is None.
425
+ """
426
+ if use_gathered_alerts:
427
+ train_plus_val_size = self.config.train_size + self.config.val_size
428
+ train_split = self.config.train_size / train_plus_val_size
429
+ val_split = self.config.val_size / train_plus_val_size
430
+ train_data, val_data, _ = split_alert_dataset(
431
+ AlertDataset(
432
+ self.collected_data[self.config.skip_first:],
433
+ ),
434
+ train_val_test_split=(train_split, val_split, 0.0),
435
+ )
436
+ else:
437
+ if train_data is None or val_data is None:
438
+ raise ValueError(
439
+ "train_data and val_data must be provided if use_gathered_alerts is False."
440
+ )
441
+ logger.info("Training AlertBERT model ...")
442
+ self.mlm.fit(train_data, val_data)
443
+ self.config.train_model = False
444
+ logger.info("AlertBERT model training complete.")
445
+ self._set_up_alertbert()
446
+ if self.config.train_model:
447
+ raise ValueError("The model should not be trained after training!")
@@ -38,7 +38,7 @@ class TimeDeltaAggregation(DensityClusteringAggregation):
38
38
 
39
39
  def __init__(
40
40
  self,
41
- config: TimeDeltaAggregationConfig | dict[str, Any],
41
+ config: TimeDeltaAggregationConfig | dict[str, Any] = TimeDeltaAggregationConfig(),
42
42
  name: str = "TimeDeltaAggregator",
43
43
  ) -> None:
44
44
  """Initialize the TimeDeltaAggregation class.
@@ -78,6 +78,7 @@ class CoreAlertAggregator(CoreComponent):
78
78
 
79
79
  if (alert := self.aggregate_alerts(input_=input_, output_=output_)):
80
80
  output_["aggregatedTimestamp"] = get_timestamp()
81
+ output_["aggregateID"] = str(self.id_generator())
81
82
 
82
83
  return alert
83
84
 
@@ -60,6 +60,12 @@ class CoreConfig(BasicConfig):
60
60
  "removed after training reads it; a killed process leaves it behind."
61
61
  ),
62
62
  )
63
+ connector_name: str | None = Field(
64
+ default=None, description="Add a connector to the output schema.",
65
+ )
66
+ connector_url: str | None = Field(
67
+ default=None, description="Set up the URL of the connector for the output schema.",
68
+ )
63
69
 
64
70
 
65
71
  class CoreComponent(Component[TInput, TOutput], FedOperations):
@@ -67,7 +73,7 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
67
73
 
68
74
  def __init__(
69
75
  self,
70
- name: str,
76
+ name: str = "Component",
71
77
  type_: str = "Core",
72
78
  config: CoreConfig | dict[str, Any] = CoreConfig(),
73
79
  args_buffer: ArgsBuffer = ArgsBuffer(BufferMode.NO_BUF),
@@ -172,7 +178,7 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
172
178
  logger.debug(f"<<{self.name}>> finalizing training")
173
179
  self.post_train()
174
180
 
175
- output_ = self.output_schema()
181
+ output_ = self.output_schema(connector=self.config.connector_name, url=self.config.connector_url)
176
182
  logger.debug(f"<<{self.name}>> processing data")
177
183
  return_schema = self.run(input_=data_buffered, output_=output_)
178
184
  if not return_schema:
@@ -180,4 +186,4 @@ class CoreComponent(Component[TInput, TOutput], FedOperations):
180
186
  return None
181
187
 
182
188
  logger.debug(f"<<{self.name}>> processed:\n{output_}")
183
- return SchemaPipeline.postprocess(output_, is_byte=is_byte)
189
+ return SchemaPipeline.postprocess(output_.store(), is_byte=is_byte)