detectmatelibrary 0.3.1__tar.gz → 0.3.2__tar.gz

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectmatelibrary
-Version: 0.3.1
+Version: 0.3.2
 Summary: DetectMate Library for log processing components
 Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
 License-Expression: EUPL-1.2
@@ -37,13 +37,14 @@ Main library to run the different components in DetectMate.
 
 The library contains the next components:
 
-* **Parsers**: parse the logs receive from the reader.
+* **Parsers**: parse the logs received from the reader.
 * **Detectors**: return alerts if anomalies are detected.
+* **Alert Aggregation**: aggregate the alerts produced by the detectors.
 * **Schemas**: standard data classes use in DetectMate.
 ```
- +--------+     +-----------+
- | Parser | --> |  Detector |
- +--------+     +-----------+
+ +--------+     +-----------+    +-------------------+
+ | Parser | --> |  Detector | -> | Alert Aggregation |
+ +--------+     +-----------+    +-------------------+
 ```
 ## Developer setup:
 

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/README.md

@@ -6,13 +6,14 @@ Main library to run the different components in DetectMate.
 
 The library contains the next components:
 
-* **Parsers**: parse the logs receive from the reader.
+* **Parsers**: parse the logs received from the reader.
 * **Detectors**: return alerts if anomalies are detected.
+* **Alert Aggregation**: aggregate the alerts produced by the detectors.
 * **Schemas**: standard data classes use in DetectMate.
 ```
- +--------+     +-----------+
- | Parser | --> |  Detector |
- +--------+     +-----------+
+ +--------+     +-----------+    +-------------------+
+ | Parser | --> |  Detector | -> | Alert Aggregation |
+ +--------+     +-----------+    +-------------------+
 ```
 ## Developer setup:
 

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/detectmatelibrary.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectmatelibrary
-Version: 0.3.1
+Version: 0.3.2
 Summary: DetectMate Library for log processing components
 Author: André García Gómez, Viktor Beck, Thorina Boenke, Wolfgang Hotwagner, Anna Erdi, Ernst Leierzopf
 License-Expression: EUPL-1.2
@@ -37,13 +37,14 @@ Main library to run the different components in DetectMate.
 
 The library contains the next components:
 
-* **Parsers**: parse the logs receive from the reader.
+* **Parsers**: parse the logs received from the reader.
 * **Detectors**: return alerts if anomalies are detected.
+* **Alert Aggregation**: aggregate the alerts produced by the detectors.
 * **Schemas**: standard data classes use in DetectMate.
 ```
- +--------+     +-----------+
- | Parser | --> |  Detector |
- +--------+     +-----------+
+ +--------+     +-----------+    +-------------------+
+ | Parser | --> |  Detector | -> | Alert Aggregation |
+ +--------+     +-----------+    +-------------------+
 ```
 ## Developer setup:
 

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/detectmatelibrary.egg-info/SOURCES.txt

@@ -7,67 +7,12 @@ detectmatelibrary.egg-info/dependency_links.txt
 detectmatelibrary.egg-info/entry_points.txt
 detectmatelibrary.egg-info/requires.txt
 detectmatelibrary.egg-info/top_level.txt
-detectmatelibrary_tests/__init__.py
-detectmatelibrary_tests/test_common/__init__.py
-detectmatelibrary_tests/test_common/test_config.py
-detectmatelibrary_tests/test_common/test_config_roundtrip.py
-detectmatelibrary_tests/test_common/test_core.py
-detectmatelibrary_tests/test_common/test_core_detector.py
-detectmatelibrary_tests/test_common/test_core_parser.py
-detectmatelibrary_tests/test_common/test_extract_timestamp.py
-detectmatelibrary_tests/test_common/test_fit_logic.py
-detectmatelibrary_tests/test_common/test_persist_config.py
-detectmatelibrary_tests/test_detectors/__init__.py
-detectmatelibrary_tests/test_detectors/dummy_detector.py
-detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py
-detectmatelibrary_tests/test_detectors/test_charset_detector.py
-detectmatelibrary_tests/test_detectors/test_mismatch_warnings.py
-detectmatelibrary_tests/test_detectors/test_new_event_detector.py
-detectmatelibrary_tests/test_detectors/test_new_value_combo_detector.py
-detectmatelibrary_tests/test_detectors/test_new_value_detector.py
-detectmatelibrary_tests/test_detectors/test_persist_integration.py
-detectmatelibrary_tests/test_detectors/test_random_detector.py
-detectmatelibrary_tests/test_detectors/test_rule_detector.py
-detectmatelibrary_tests/test_detectors/test_value_range_detector.py
-detectmatelibrary_tests/test_folder/__init__.py
-detectmatelibrary_tests/test_helper/__init__.py
-detectmatelibrary_tests/test_helper/test_from_to.py
-detectmatelibrary_tests/test_others/__init__.py
-detectmatelibrary_tests/test_others/test_logging.py
-detectmatelibrary_tests/test_parsers/__init__.py
-detectmatelibrary_tests/test_parsers/dummy_parser.py
-detectmatelibrary_tests/test_parsers/test_json_parser.py
-detectmatelibrary_tests/test_parsers/test_logbatcher_parser.py
-detectmatelibrary_tests/test_parsers/test_template_matcher.py
-detectmatelibrary_tests/test_parsers/test_tree_macther.py
-detectmatelibrary_tests/test_persistency/__init__.py
-detectmatelibrary_tests/test_persistency/test_stability_tracker.py
-detectmatelibrary_tests/test_pipelines/__init__.py
-detectmatelibrary_tests/test_pipelines/test_bad_players.py
-detectmatelibrary_tests/test_pipelines/test_basic_pipelines.py
-detectmatelibrary_tests/test_pipelines/test_configuration_engine.py
-detectmatelibrary_tests/test_pipelines/test_dummy_methods.py
-detectmatelibrary_tests/test_schemas/__init__.py
-detectmatelibrary_tests/test_schemas/test_default_values.py
-detectmatelibrary_tests/test_schemas/test_ops.py
-detectmatelibrary_tests/test_schemas/test_schema_class.py
-detectmatelibrary_tests/test_utils/__init__.py
-detectmatelibrary_tests/test_utils/test_aux.py
-detectmatelibrary_tests/test_utils/test_data_buffer.py
-detectmatelibrary_tests/test_utils/test_data_normalizer.py
-detectmatelibrary_tests/test_utils/test_events_seen.py
-detectmatelibrary_tests/test_utils/test_log_format_utils.py
-detectmatelibrary_tests/test_utils/test_persistency.py
-detectmatelibrary_tests/test_utils/test_persistency_dump_load.py
-detectmatelibrary_tests/test_utils/test_persistency_saver.py
-detectmatelibrary_tests/test_utils/test_stability_tracking.py
-detectmatelibrary_tests/test_workspace/__init__.py
-detectmatelibrary_tests/test_workspace/test_create_workspace.py
-detectmatelibrary_tests/test_workspace/test_utils.py
 src/detectmatelibrary/__init__.py
 src/detectmatelibrary/constants.py
 src/detectmatelibrary/metadata.py
+src/detectmatelibrary/alert_aggregation/basic_concat.py
 src/detectmatelibrary/common/__init__.py
+src/detectmatelibrary/common/alert_aggregator.py
 src/detectmatelibrary/common/core.py
 src/detectmatelibrary/common/detector.py
 src/detectmatelibrary/common/parser.py

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/detectmatelibrary.egg-info/top_level.txt

@@ -1,3 +1,2 @@
 detectmatelibrary
-detectmatelibrary_tests
 detectmatelibrary_tools

detectmatelibrary-0.3.2/src/detectmatelibrary/alert_aggregation/basic_concat.py

@@ -0,0 +1,29 @@
+from detectmatelibrary.common.alert_aggregator import (
+    CoreAlertAggregatorConfig, CoreAlertAggregator
+)
+
+from detectmatelibrary.schemas import DetectorSchema, OutputSchema
+
+from typing import Any, Optional
+
+
+class BasicConcatAggregationConfig(CoreAlertAggregatorConfig):
+    method_type: str = "basic_concat_aggregator"
+    buffer_size: int = 3
+
+
+class BasicConcatAggregation(CoreAlertAggregator):
+    def __init__(
+        self,
+        name: str = "BasicConcatAggregator",
+        config: Optional[BasicConcatAggregationConfig | dict[str, Any]] = BasicConcatAggregationConfig(),
+    ) -> None:
+        if isinstance(config, dict):
+            config = BasicConcatAggregationConfig.from_dict(config, name)
+
+        buffer_size: int = config.buffer_size  # type: ignore
+        super().__init__(name=name, buffer_size=buffer_size, config=config)
+
+    def aggregate_alerts(self, input_: list[DetectorSchema], output_: OutputSchema) -> bool:  # type: ignore
+        output_["description"] = "Basic aggregation by alert concatenation"
+        return True

detectmatelibrary-0.3.2/src/detectmatelibrary/common/alert_aggregator.py

@@ -0,0 +1,99 @@
+
+from detectmatelibrary.common.core import CoreComponent, CoreConfig
+
+from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode
+from detectmatelibrary.utils.aux import get_timestamp
+
+from detectmatelibrary.schemas import DetectorSchema, OutputSchema
+
+from typing import Any, Optional
+from typing_extensions import override
+
+
+def _extract_field(
+    input_: list[DetectorSchema] | DetectorSchema, field: str
+) -> list[str]:
+    if not isinstance(input_, list):
+        input_ = [input_]
+
+    return [str(i[field]) for i in input_]
+
+
+def _extract_list_field(input_: list[DetectorSchema] | DetectorSchema, field: str) -> list[str]:
+    if not isinstance(input_, list):
+        input_ = [input_]
+
+    final = []
+    for i in input_:
+        final.extend(i[field])
+
+    return final
+
+
+class CoreAlertAggregatorConfig(CoreConfig):
+    component_type: str = "alert_aggregators"
+    method_type: str = "core_alert_aggregator"
+
+
+class CoreAlertAggregator(CoreComponent):
+    def __init__(
+        self,
+        name: str = "CoreAlertAggregator",
+        buffer_mode: BufferMode = BufferMode.WINDOW,
+        buffer_size: Optional[int] = 1,
+        config: Optional[CoreAlertAggregatorConfig | dict[str, Any]] = CoreAlertAggregatorConfig(),
+    ) -> None:
+        if isinstance(config, dict):
+            config = CoreAlertAggregatorConfig.from_dict(config, name)
+
+        super().__init__(
+            name=name,
+            type_=config.method_type,  # type: ignore
+            config=config,  # type: ignore
+            args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size),
+            input_schema=DetectorSchema,
+            output_schema=OutputSchema,
+        )
+
+    @override
+    def run(
+        self, input_: list[DetectorSchema] | DetectorSchema, output_: OutputSchema  # type: ignore
+    ) -> bool:
+
+        output_["detectorIDs"] = _extract_field(input_, "detectorID")
+        output_["detectorTypes"] = _extract_field(input_, "detectorType")
+        output_["alertIDs"] = _extract_field(input_, "alertID")
+        output_["logIDs"] = _extract_list_field(input_, "logIDs")
+        output_["extractedTimestamps"] = _extract_list_field(input_, "extractedTimestamps")
+
+        if (alert := self.aggregate_alerts(input_=input_, output_=output_)):
+            output_["outputTimestamp"] = get_timestamp()
+
+        return alert
+
+    def aggregate_alerts(
+        self,
+        input_: list[DetectorSchema] | DetectorSchema,
+        output_: OutputSchema,
+    ) -> bool:
+        return True
+
+    @override
+    def train(
+        self, input_: DetectorSchema | list[DetectorSchema]  # type: ignore
+    ) -> None:
+        pass
+
+    @override
+    def configure(
+        self, input_: DetectorSchema | list[DetectorSchema]  # type: ignore
+    ) -> None:
+        pass
+
+    @override
+    def set_configuration(self) -> None:
+        pass
+
+    @override
+    def post_train(self) -> None:
+        pass

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/src/detectmatelibrary/common/core.py

@@ -128,6 +128,58 @@ class CoreComponent(Component):
         )
         self.buffer_train = TrainBuffer()
 
+    def export_state(
+        self,
+        path: str | None = None,
+        storage_options: dict[str, Any] | None = None,
+    ) -> bytes | None:
+        """Save this component's EventPersistency state.
+
+        When path is None, returns the state as bytes (zip archive).
+        When path is given, writes to that fsspec URI and returns None.
+        Returns None if no persistency is configured. Thread-safe when a
+        PersistencySaver is running: acquires the saver lock before saving
+        (guards against the background save timer and concurrent ingest).
+        """
+        # ponytail: local import keeps common.core free of a persistency
+        # package import at module load (see _Stoppable).
+        from detectmatelibrary.utils import persistency
+
+        ep = getattr(self, "persistency", None)
+        if ep is None:
+            logger.debug(f"{self.name}: no persistency configured, nothing to export")
+            return None
+        saver = getattr(self, "saver", None)
+        if saver is not None:
+            with saver.locked():
+                return persistency.save(ep, path, storage_options)
+        return persistency.save(ep, path, storage_options)
+
+    def import_state(
+        self,
+        path: str | bytes,
+        storage_options: dict[str, Any] | None = None,
+    ) -> None:
+        """Restore this component's EventPersistency state.
+
+        path may be an fsspec URI string or bytes returned by
+        export_state(). No-op if no persistency is configured. Thread-safe
+        when a PersistencySaver is running: acquires the saver lock before
+        loading (guards against the background save timer).
+        """
+        from detectmatelibrary.utils import persistency
+
+        ep = getattr(self, "persistency", None)
+        if ep is None:
+            logger.debug(f"{self.name}: no persistency configured, nothing to import")
+            return
+        saver = getattr(self, "saver", None)
+        if saver is not None:
+            with saver.locked():
+                persistency.load(ep, path, storage_options)
+        else:
+            persistency.load(ep, path, storage_options)
+
     def update_state(self, state: StatesL) -> None:
         self.fitlogic.update_state(state)
 

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/src/detectmatelibrary/metadata.py

@@ -7,6 +7,6 @@ __deprecated__ = False
 __website__ = "https://aecid.ait.ac.at"
 __license__ = "EUPL-1.2"
 __status__ = "Development"
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 __all__ = ['__authors__', '__contact__', '__copyright__', '__date__', '__deprecated__', '__website__',
            '__license__', '__status__', '__version__']

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/src/detectmatelibrary/parsers/logbatcher/engine/util.py

@@ -49,12 +49,13 @@ def count_prompt_tokens(prompt: str, model_name: str) -> int:
     Count the number of tokens in the prompt
     Models supported: gpt-4o-mini, gpt-3.5-turbo
     """
-    if model_name == 'gpt-4o-mini':
-        encoder = tiktoken.encoding_for_model('gpt-4o-mini')
-    elif model_name == 'gpt-3.5-turbo':
-        encoder = tiktoken.encoding_for_model('gpt-3.5-turbo')
-    else:
-        raise ValueError("Unsupported model: {}".format(model_name))
+    try:
+        encoder = tiktoken.encoding_for_model(model_name)
+    except KeyError:
+        # ponytail: token count is estimate-only bookkeeping; use the modern
+        # OpenAI encoding for models tiktoken doesn't know rather than raising
+        # (raising here silently discards a successful LLM answer upstream).
+        encoder = tiktoken.get_encoding("o200k_base")
 
     # 计算编码后的token数
     prompt_tokens = encoder.encode(prompt)
@@ -66,12 +67,13 @@ def count_message_tokens(messages: List[Dict[str, str]], model_name: str = "gpt-
     Count the number of tokens in the messages
     Models supported: gpt-4o-mini, gpt-3.5-turbo
     """
-    if model_name == 'gpt-4o-mini':
-        encoder = tiktoken.encoding_for_model('gpt-4o-mini')
-    elif model_name == 'gpt-3.5-turbo':
-        encoder = tiktoken.encoding_for_model('gpt-3.5-turbo')
-    else:
-        raise ValueError("Unsupported model: {}".format(model_name))
+    try:
+        encoder = tiktoken.encoding_for_model(model_name)
+    except KeyError:
+        # ponytail: token count is estimate-only bookkeeping; use the modern
+        # OpenAI encoding for models tiktoken doesn't know rather than raising
+        # (raising here silently discards a successful LLM answer upstream).
+        encoder = tiktoken.get_encoding("o200k_base")
 
     token_count = 0
 

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/src/detectmatelibrary/utils/persistency/__init__.py

@@ -1,5 +1,5 @@
 from .event_persistency import EventPersistency
-from .persistency_saver import PersistencySaver, PersistencySaverConfig, PersistencyLoadError
+from .persistency_saver import PersistencySaver, PersistencySaverConfig, PersistencyLoadError, save, load
 from .event_data_structures.base import EventDataStructure
 from .event_data_structures.dataframes.event_dataframe import EventDataFrame
 from .event_data_structures.dataframes.chunked_event_dataframe import ChunkedEventDataFrame
@@ -14,4 +14,6 @@ __all__ = [
     "EventDataFrame",
     "ChunkedEventDataFrame",
     "EventStabilityTracker",
+    "save",
+    "load",
 ]

{detectmatelibrary-0.3.1 → detectmatelibrary-0.3.2}/src/detectmatelibrary/utils/persistency/event_persistency.py

@@ -1,3 +1,4 @@
+import threading
 from typing import Any, Callable, Dict, List, Optional, Type
 
 from .event_data_structures.base import EventDataStructure
@@ -33,6 +34,10 @@ class EventPersistency:
         self.event_templates: Dict[int | str, str] = {}
         self._events_since_save: int = 0
         self._on_ingest_callbacks: list[Callable[[], None]] = []
+        # ponytail: RLock shared with PersistencySaver so ingest/save/load are
+        # mutually exclusive. On-ingest callbacks fire outside this lock, so
+        # re-entrancy is no longer required — kept as RLock (harmless, safer).
+        self._lock = threading.RLock()
 
     def ingest_event(
         self,
@@ -42,22 +47,29 @@ class EventPersistency:
         named_variables: Dict[str, Any] = {}
     ) -> None:
         """Ingest event data into the appropriate EventData store."""
-        self._events_since_save += 1
+        with self._lock:
+            self._events_since_save += 1
+            self.events_seen.add(event_id)
+            if variables or named_variables:
+                self.event_templates[event_id] = event_template
+                all_variables = self.get_all_variables(variables, named_variables)
+
+                data_structure = self.events_data.get(event_id)
+                if data_structure is None:
+                    data_structure = self.event_data_class(**self.event_data_kwargs)
+                    self.events_data[event_id] = data_structure
+
+                data = data_structure.to_data(all_variables)
+                data_structure.add_data(data)
+        # ponytail: fire callbacks outside the lock so a count-triggered save
+        # doesn't hold the ingest lock across serialize + file I/O.
         for _cb in self._on_ingest_callbacks:
             _cb()
-        self.events_seen.add(event_id)
-        if not variables and not named_variables:
-            return
-        self.event_templates[event_id] = event_template
-        all_variables = self.get_all_variables(variables, named_variables)
 
-        data_structure = self.events_data.get(event_id)
-        if data_structure is None:
-            data_structure = self.event_data_class(**self.event_data_kwargs)
-            self.events_data[event_id] = data_structure
-
-        data = data_structure.to_data(all_variables)
-        data_structure.add_data(data)
+    @property
+    def events_since_save(self) -> int:
+        """Number of events ingested since the last successful save."""
+        return self._events_since_save
 
     def reset_events_since_save(self) -> None:
         """Reset the events-since-save counter after a successful save."""