logprep 19.2.0__tar.gz → 19.3.0__tar.gz

{logprep-19.2.0 → logprep-19.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logprep
-Version: 19.2.0
+Version: 19.3.0
 Summary: Logprep allows to collect, process and forward log messages from various data sources.
 Keywords: kafka,etl,sre,preprocessing,opensearch,soar,logdata
 License:                   GNU LESSER GENERAL PUBLIC LICENSE
@@ -519,49 +519,38 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Topic :: Security
 Classifier: Topic :: System :: Logging
-Requires-Dist: aiohttp>=3.13.3,<4
 Requires-Dist: attrs<26
-Requires-Dist: certifi>=2023.7.22,<2026
 Requires-Dist: confluent-kafka>2,<3
 Requires-Dist: filelock<4
 Requires-Dist: geoip2>=5.2.0,<6
 Requires-Dist: jsonref<2
 Requires-Dist: luqum<2
-Requires-Dist: more-itertools==8.10.0
 Requires-Dist: numpy>=1.26.0,<3
 Requires-Dist: opensearch-py>=3.2,<4
 Requires-Dist: prometheus-client<1
-Requires-Dist: protobuf>=3.20.2,<7
 Requires-Dist: pycryptodome<4
 Requires-Dist: pyparsing<4
 Requires-Dist: scikit-learn>=1.2.0,<2
-Requires-Dist: scipy>=1.9.2,<2
 Requires-Dist: joblib<2
-Requires-Dist: pyyaml<7
 Requires-Dist: requests>=2.33.0,<3
 Requires-Dist: ruamel-yaml<1
 Requires-Dist: schedule<2
 Requires-Dist: urllib3>=2.7.0,<3
 Requires-Dist: uvicorn<1
-Requires-Dist: deepdiff>=8.6.2,<9
 Requires-Dist: msgspec<1
 Requires-Dist: boto3<2
-Requires-Dist: pydantic<3
 Requires-Dist: click<9
-Requires-Dist: pandas<3
-Requires-Dist: tabulate<1
 Requires-Dist: falcon>=3.1.3,<5
 Requires-Dist: uvloop<1
 Requires-Dist: httptools<1
+Requires-Dist: more-itertools>=11.1.0
 Requires-Dist: black>=26.0.0 ; extra == 'dev'
-Requires-Dist: httpx ; extra == 'dev'
 Requires-Dist: isort ; extra == 'dev'
 Requires-Dist: pylint ; extra == 'dev'
 Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: pytest-cov ; extra == 'dev'
 Requires-Dist: responses ; extra == 'dev'
 Requires-Dist: jinja2 ; extra == 'dev'
-Requires-Dist: maturin ; extra == 'dev'
 Requires-Dist: cibuildwheel ; extra == 'dev'
 Requires-Dist: asgiref ; extra == 'dev'
 Requires-Dist: pytest-asyncio ; extra == 'dev'
@@ -571,6 +560,7 @@ Requires-Dist: psutil ; extra == 'dev'
 Requires-Dist: types-requests ; extra == 'dev'
 Requires-Dist: types-psutil ; extra == 'dev'
 Requires-Dist: types-pyyaml ; extra == 'dev'
+Requires-Dist: deepdiff>=8.6.2,<9 ; extra == 'dev'
 Requires-Dist: sphinx>=8.2.0,<9 ; extra == 'doc'
 Requires-Dist: sphinx-rtd-theme ; extra == 'doc'
 Requires-Dist: sphinxcontrib-datatemplates ; extra == 'doc'
@@ -579,6 +569,7 @@ Requires-Dist: sphinx-copybutton ; extra == 'doc'
 Requires-Dist: nbsphinx>=0.9 ; extra == 'doc'
 Requires-Dist: ipython>=8.7 ; extra == 'doc'
 Requires-Dist: openpyxl ; extra == 'doc'
+Requires-Dist: pandas<3 ; extra == 'doc'
 Requires-Python: >=3.11
 Project-URL: Homepage, https://github.com/fkie-cad/Logprep
 Project-URL: Documentation, https://logprep.readthedocs.io/en/latest/

{logprep-19.2.0 → logprep-19.3.0}/logprep/connector/dummy/output.py

@@ -19,15 +19,11 @@ Example
 
 import logging
 import typing
-from typing import TYPE_CHECKING
 
 from attrs import define, field, validators
 
 from logprep.abc.output import Output
 
-if TYPE_CHECKING:
-    from logprep.abc.connector import Connector  # pragma: no cover
-
 logger = logging.getLogger("DummyOutput")
 
 

{logprep-19.2.0 → logprep-19.3.0}/logprep/filter/expression/filter_expression.py

@@ -3,10 +3,12 @@
 import re
 from abc import ABC, abstractmethod
 from itertools import chain, zip_longest
-from typing import Any, Sequence
+from typing import Any, Generic, Sequence, TypeVar
 
 from logprep.util.helper import field_list_to_dotted_field
 
+RangeBoundary = TypeVar("RangeBoundary", int, float, str)
+
 
 class FilterExpressionError(Exception):
     """Base class for FilterExpression related exceptions."""
@@ -289,37 +291,88 @@ class FloatFilterExpression(KeyValueBasedFilterExpression):
         return value == self._expected_value
 
 
-class RangeBasedFilterExpression(KeyBasedFilterExpression):
+class RangeBasedFilterExpression(KeyBasedFilterExpression, Generic[RangeBoundary]):
     """Base class of filter expressions that match for a range of values."""
 
-    def __init__(self, key: Sequence[str], lower_bound: float, upper_bound: float):
+    def __init__(
+        self,
+        key: Sequence[str],
+        lower_bound: RangeBoundary,
+        upper_bound: RangeBoundary,
+        include_lower_bound: bool = True,
+        include_upper_bound: bool = True,
+    ):
         super().__init__(key)
-        self._lower_bound = lower_bound
-        self._upper_bound = upper_bound
+        self._lower_bound: RangeBoundary = lower_bound
+        self._upper_bound: RangeBoundary = upper_bound
+        self._include_lower_bound = include_lower_bound
+        self._include_upper_bound = include_upper_bound
 
     def __repr__(self) -> str:
-        return f"{self.key_as_dotted_string}:[{self._lower_bound} TO {self._upper_bound}]"
+        opening_bracket = "[" if self._include_lower_bound else "{"
+        closing_bracket = "]" if self._include_upper_bound else "}"
+
+        return (
+            f"{self.key_as_dotted_string}:"
+            f"{opening_bracket}{self._lower_bound} TO {self._upper_bound}"
+            f"{closing_bracket}"
+        )
+
+    def _matches_lower_bound(self, value: RangeBoundary) -> bool:
+        if self._include_lower_bound:
+            return value >= self._lower_bound
+        return value > self._lower_bound
+
+    def _matches_upper_bound(self, value: RangeBoundary) -> bool:
+        if self._include_upper_bound:
+            return value <= self._upper_bound
+        return value < self._upper_bound
+
+    def _matches_range(self, value: RangeBoundary) -> bool:
+        return self._matches_lower_bound(value) and self._matches_upper_bound(value)
 
     def does_match(self, document: dict):
         raise NotImplementedError
 
 
-class IntegerRangeFilterExpression(RangeBasedFilterExpression):
+class IntegerRangeFilterExpression(RangeBasedFilterExpression[int]):
     """Range based filter expression that matches for integers."""
 
     def does_match(self, document: dict) -> bool:
         value = self._get_value(self.key, document)
 
-        return self._lower_bound <= value <= self._upper_bound
+        if not isinstance(value, int) or isinstance(value, bool):
+            return False
+
+        return self._matches_range(value)
 
 
-class FloatRangeFilterExpression(RangeBasedFilterExpression):
+class FloatRangeFilterExpression(RangeBasedFilterExpression[float]):
     """Range based filter expression that matches for floats."""
 
     def does_match(self, document: dict) -> bool:
         value = self._get_value(self.key, document)
 
-        return self._lower_bound <= value <= self._upper_bound
+        if not isinstance(value, float) or isinstance(value, bool):
+            return False
+
+        return self._matches_range(value)
+
+
+class StringRangeFilterExpression(RangeBasedFilterExpression[str]):
+    """Range based filter expression that matches for strings.
+
+    String ranges are compared lexicographically.
+    """
+
+    def does_match(self, document: dict) -> bool:
+        value = self._get_value(self.key, document)
+
+        # Avoid comparing strings with other value types, which would raise a TypeError.
+        if not isinstance(value, str):
+            return False
+
+        return self._matches_range(value)
 
 
 class RegExFilterExpression(KeyValueBasedFilterExpression):

{logprep-19.2.0 → logprep-19.3.0}/logprep/filter/lucene_filter.py

@@ -1,4 +1,6 @@
 # pylint: disable=anomalous-backslash-in-string
+# pylint: disable=too-many-positional-arguments
+# pylint: disable=too-many-return-statements
 r"""
 Filter
 ======
@@ -58,6 +60,101 @@ This example is not complete, since rules are specific to processors and require
 
     { "filter": "event_id: 1 AND NOT ip_address: 192.168.0.1" }
 
+Range-Filter
+------------
+
+It is possible to use range expressions to match integer, floating-point, and
+string values. Square brackets include a boundary, while curly brackets exclude
+a boundary. String ranges are compared lexicographically.
+
+The lower and upper boundaries of a range must have the same type. Mixed ranges
+such as integer-to-float, integer-to-string, or float-to-string are not
+supported.
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'age:[18 TO 65]'
+
+
+The example matches log messages in which the value of :code:`age` is greater
+than or equal to :code:`18` and less than or equal to :code:`65`.
+
+Exclusive boundaries are also supported:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'age:{18 TO 65}'
+
+
+The example matches log messages in which the value of :code:`age` is greater
+than :code:`18` and less than :code:`65`.
+
+Floating-point ranges are supported as well:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'temperature:[18.5 TO 25.0]'
+
+
+String ranges are compared lexicographically:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'status:[alpha TO stable]'
+
+
+This also allows matching consistently formatted ISO-8601 timestamps as string
+ranges:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'timestamp:[2024-01-01T00:00:00Z TO 2024-12-31T23:59:59Z]'
+
+
+The example matches log messages in which the value of :code:`timestamp` is
+lexicographically greater than or equal to :code:`2024-01-01T00:00:00Z` and
+less than or equal to :code:`2024-12-31T23:59:59Z`.
+
+Range boundaries containing Lucene special characters must be quoted. This is
+required, for example, for ISO-8601 timestamps with timezone offsets:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'timestamp:["2024-01-01T00:00:00+01:00" TO "2024-12-31T23:59:59+01:00"]'
+
+
+Range expressions can also be used within field groups:
+
+
+..  code-block:: yaml
+    :linenos:
+    :caption: Example
+
+    filter: 'temperature:([18.5 TO 25.0])'
+
+
+Open boundaries using :code:`*`, non-finite numeric boundaries, mixed boundary
+types, and unquoted boundaries containing Lucene special characters are not
+supported.
+
 RegEx-Filter
 ------------
 
@@ -73,7 +170,8 @@ To be recognized as a regular expression, the filter field has to start with
     filter: 'ip_address: /192\.168\.0\..*/'
 
 
-[Deprecated, but still functional] The field with the regex pattern must be added to the optional field
+[Deprecated, but still functional] The field with the regex pattern
+must be added to the optional field
 :code:`regex_fields` in the rule definition.
 
 In the following example the field :code:`ip_address` is defined as regex field.
@@ -93,6 +191,7 @@ require additional options.
 """
 
 import logging
+import math
 import re
 from itertools import chain, zip_longest
 from typing import Sequence
@@ -106,24 +205,31 @@ from luqum.tree import (
     Not,
     OrOperation,
     Phrase,
+    Prohibit,
+    Range,
     Regex,
     SearchField,
     Word,
 )
+
 from logprep.abc.exceptions import LogprepException
 from logprep.filter.expression.filter_expression import (
     Always,
     And,
     Exists,
     FilterExpression,
+    FloatRangeFilterExpression,
+    IntegerRangeFilterExpression,
 )
 from logprep.filter.expression.filter_expression import Not as NotExpression
 from logprep.filter.expression.filter_expression import (
     Null,
     Or,
+    RangeBoundary,
     RegExFilterExpression,
     SigmaFilterExpression,
     StringFilterExpression,
+    StringRangeFilterExpression,
 )
 from logprep.util.helper import field_list_to_dotted_field, get_dotted_field_list
 
@@ -275,6 +381,14 @@ class LuceneTransformer:
             # pylint: enable=no-value-for-parameter
         if isinstance(tree, Group):
             return self._parse_tree(tree.children[0])
+
+        if isinstance(tree, Range):
+            if self._last_search_field is None:
+                raise LuceneFilterError(f'The expression "{str(tree)}" is invalid!')
+
+            key = get_dotted_field_list(self._last_search_field)
+            return self._parse_range(key, tree)
+
         if isinstance(tree, SearchField):
             if isinstance(tree.expr, FieldGroup):
                 self._last_search_field = tree.name
@@ -288,6 +402,171 @@ class LuceneTransformer:
             return self._create_value_expression(tree)
         raise LuceneFilterError(f'The expression "{str(tree)}" is invalid!')
 
+    def _parse_range(self, key: Sequence[str], expr: Range) -> FilterExpression:
+        lower_value = self._get_range_boundary_value(expr.low)
+        upper_value = self._get_range_boundary_value(expr.high)
+
+        for range_parser in (
+            self._parse_integer_range,
+            self._parse_float_range,
+            self._parse_string_range,
+        ):
+            range_filter_expression = range_parser(
+                key,
+                lower_value,
+                upper_value,
+                expr.include_low,
+                expr.include_high,
+                expr,
+            )
+
+            if range_filter_expression is not None:
+                return range_filter_expression
+
+        raise LuceneFilterError(f'The expression "{expr}" is invalid!')
+
+    @staticmethod
+    def _parse_integer_range(
+        key: Sequence[str],
+        lower_value: str,
+        upper_value: str,
+        include_lower_bound: bool,
+        include_upper_bound: bool,
+        expr: Range,
+    ) -> IntegerRangeFilterExpression | None:
+        """Create an integer range expression if both boundaries are integers."""
+        try:
+            lower_bound = int(lower_value)
+            upper_bound = int(upper_value)
+        except ValueError:
+            return None
+
+        LuceneTransformer._validate_range_boundaries(lower_bound, upper_bound, expr)
+
+        return IntegerRangeFilterExpression(
+            key,
+            lower_bound,
+            upper_bound,
+            include_lower_bound,
+            include_upper_bound,
+        )
+
+    @staticmethod
+    def _parse_float_range(
+        key: Sequence[str],
+        lower_value: str,
+        upper_value: str,
+        include_lower_bound: bool,
+        include_upper_bound: bool,
+        expr: Range,
+    ) -> FloatRangeFilterExpression | None:
+        """Create a float range expression if both boundaries are finite numbers."""
+        try:
+            lower_bound = float(lower_value)
+            upper_bound = float(upper_value)
+        except ValueError:
+            return None
+
+        if not math.isfinite(lower_bound) or not math.isfinite(upper_bound):
+            raise LuceneFilterError(
+                f'The expression "{expr}" is invalid. ' "Range boundaries must be finite numbers."
+            )
+
+        LuceneTransformer._validate_range_boundaries(lower_bound, upper_bound, expr)
+
+        return FloatRangeFilterExpression(
+            key,
+            lower_bound,
+            upper_bound,
+            include_lower_bound,
+            include_upper_bound,
+        )
+
+    @staticmethod
+    def _parse_string_range(
+        key: Sequence[str],
+        lower_value: str,
+        upper_value: str,
+        include_lower_bound: bool,
+        include_upper_bound: bool,
+        expr: Range,
+    ) -> StringRangeFilterExpression | None:
+        """Create a lexicographic string range expression for non-numeric boundaries."""
+        if lower_value == "*" or upper_value == "*":
+            raise LuceneFilterError(f'The expression "{expr}" is invalid!')
+
+        if LuceneTransformer._is_finite_numeric_range_boundary(
+            lower_value
+        ) or LuceneTransformer._is_finite_numeric_range_boundary(upper_value):
+            return None
+
+        LuceneTransformer._validate_range_boundaries(lower_value, upper_value, expr)
+
+        return StringRangeFilterExpression(
+            key,
+            lower_value,
+            upper_value,
+            include_lower_bound,
+            include_upper_bound,
+        )
+
+    @staticmethod
+    def _is_finite_numeric_range_boundary(value: str) -> bool:
+        """Return whether a range boundary can be interpreted as a finite number."""
+        try:
+            numeric_value = float(value)
+        except ValueError:
+            return False
+
+        return math.isfinite(numeric_value)
+
+    @staticmethod
+    def _get_range_boundary_value(token: luqum.tree) -> str:
+        """Return a range boundary as a normalized string.
+
+        Luqum parses a negative boundary such as ``-10`` as
+        ``Prohibit(Word("10"))`` instead of ``Word("-10")``. Quoted boundaries are
+        parsed as phrases and are required for values containing Lucene special
+        characters, such as ISO-8601 timestamps with timezone offsets.
+        """
+
+        if isinstance(token, Word):
+            return token.value
+
+        if isinstance(token, Phrase):
+            return token.value.strip('"')
+
+        if isinstance(token, Prohibit) and len(token.children) == 1:
+            child = token.children[0]
+
+            if isinstance(child, Word):
+                return f"-{child.value}"
+
+        raise LuceneFilterError(f'The range boundary "{token}" is invalid!')
+
+    @staticmethod
+    def _validate_range_boundaries(
+        lower_bound: RangeBoundary,
+        upper_bound: RangeBoundary,
+        expr: Range,
+    ) -> None:
+        """Validate that the range boundaries form an ascending range.
+
+        Range filter expressions require the lower boundary to be less than or equal
+        to the upper boundary. This is independent of whether the boundaries are
+        inclusive or exclusive.
+
+        Numeric boundaries are compared numerically. String boundaries are compared
+        lexicographically. Without this validation, a reversed range such as
+        ``[10 TO 0]`` or ``[foo TO bar]`` would be accepted but could never match
+        any value, which would hide a likely configuration error.
+        """
+
+        if lower_bound > upper_bound:
+            raise LuceneFilterError(
+                "The lower range boundary must not exceed " f'the upper range boundary: "{expr}"'
+            )
+
     def _create_field_group_expression(
         self, tree: luqum.tree, dotted_field: str
     ) -> FilterExpression:
@@ -321,22 +600,27 @@ class LuceneTransformer:
         return expressions
 
     def _create_field(self, tree: luqum.tree) -> FilterExpression:
+        key = get_dotted_field_list(tree.name)
+
         if isinstance(tree.expr, (Phrase, Word)):
-            key = get_dotted_field_list(tree.name)
             if tree.expr.value == "null":
                 return Null(key)
 
             value = self._strip_quote_from_string(tree.expr.value)
             value = self._remove_lucene_escaping(value)
             return self._get_filter_expression(key, value)
+
         if isinstance(tree.expr, Regex):
-            key = get_dotted_field_list(tree.name)
             if tree.expr.value == "null":
                 return Null(key)
 
             value = self._strip_quote_from_string(tree.expr.value)
             value = self._remove_lucene_escaping(value)
             return self._get_filter_expression_regex(key, value)
+
+        if isinstance(tree.expr, Range):
+            return self._parse_range(key, tree.expr)
+
         raise LuceneFilterError(f'The expression "{str(tree)}" is invalid!')
 
     @staticmethod

{logprep-19.2.0 → logprep-19.3.0}/logprep/ng/connector/dummy/output.py

@@ -19,7 +19,6 @@ Example
 
 import logging
 import typing
-from typing import TYPE_CHECKING
 
 from attrs import define, field, validators
 
@@ -27,9 +26,6 @@ from logprep.ng.abc.event import Event
 from logprep.ng.abc.output import Output
 from logprep.ng.event.log_event import LogEvent
 
-if TYPE_CHECKING:
-    from logprep.abc.connector import Connector  # pragma: no cover
-
 logger = logging.getLogger("DummyOutput")
 
 
@@ -88,7 +84,7 @@ class DummyOutput(Output):
         return typing.cast("DummyOutput.Config", self._config)
 
     @Output._handle_errors
-    def store(self, event: Event) -> None:
+    def store(self, event: LogEvent) -> None:
         """Store the document in the output destination.
 
         Parameters

{logprep-19.2.0 → logprep-19.3.0}/logprep/ng/processor/clusterer/processor.py

@@ -44,7 +44,6 @@ Processor Configuration
 """
 
 import typing
-from typing import Tuple
 
 from attrs import define, field, validators
 
@@ -57,11 +56,11 @@ from logprep.processor.clusterer.signature_calculation.signature_phase import (
     SignaturePhaseStreaming,
 )
 from logprep.util.helper import (
+    MISSING,
+    FieldValue,
     add_fields_to,
     get_dotted_field_value,
     get_field_value_no_slice,
-    MISSING,
-    FieldValue,
 )
 
 

logprep-19.3.0/logprep/ng/processor/deduplicator/processor.py

@@ -0,0 +1,50 @@
+"""
+Deduplicator
+============
+
+The `deduplicator` is a processor that removes duplicate values from specified fields.
+
+Processor Configuration
+^^^^^^^^^^^^^^^^^^^^^^^
+..  code-block:: yaml
+    :linenos:
+
+    - deduplicatorname:
+        type: deduplicator
+        rules:
+            - tests/testdata/rules/rules
+
+.. autoclass:: logprep.processor.deduplicator.processor.Deduplicator.Config
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :noindex:
+
+.. automodule:: logprep.processor.deduplicator.rule
+"""
+
+import typing
+
+from logprep.ng.abc.processor import Processor
+from logprep.processor.base.rule import Rule
+from logprep.processor.deduplicator.rule import DeduplicatorRule
+from logprep.util.helper import get_dotted_field_value, add_fields_to
+
+
+class Deduplicator(Processor):
+    """A processor that removes duplicates from lists"""
+
+    rule_class = DeduplicatorRule  # type: ignore
+
+    def _apply_rules(self, event: dict, rule: Rule):
+        rule = typing.cast(DeduplicatorRule, rule)
+        for field in rule.fields:
+            value_list = get_dotted_field_value(event, field)
+            if not isinstance(value_list, list):
+                continue
+
+            unique = []
+            for value in value_list:
+                if value not in unique:
+                    unique.append(value)
+            add_fields_to(event, {field: unique}, rule, False, True)