pySigma 1.3.2__tar.gz → 1.4.0__tar.gz

{pysigma-1.3.2 → pysigma-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pySigma
-Version: 1.3.2
+Version: 1.4.0
 Summary: Sigma rule processing and conversion tools
 License-Expression: LGPL-2.1-only
 License-File: LICENSE
@@ -17,8 +17,9 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Security
 Requires-Dist: diskcache (>=5.6.3,<6.0.0)
-Requires-Dist: diskcache-stubs (>=5.6.3.6.20240818,<6.0.0.0.0)
+Requires-Dist: diskcache-stubs (>=5.6.3)
 Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
+Requires-Dist: jq (>=1.6,<2.0)
 Requires-Dist: packaging (>=26.0,<27.0)
 Requires-Dist: pyparsing (>=3.2.5,<4.0.0)
 Requires-Dist: pyyaml (>=6.0.3,<7.0.0)

{pysigma-1.3.2 → pysigma-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pySigma"
-version = "1.3.2"
+version = "1.4.0"
 license = "LGPL-2.1-only"
 description = "Sigma rule processing and conversion tools"
 authors = [
@@ -37,20 +37,21 @@ jinja2 = "^3.1.6"
 types-pyyaml = "^6.0.12.20250915"
 typing-extensions = "^4.15.0"
 diskcache = "^5.6.3"
-diskcache-stubs = "^5.6.3.6.20240818"
+diskcache-stubs = ">=5.6.3"
+jq = "^1.6"
 
 [tool.poetry.group.dev.dependencies]
 black = "^26.3.1"
-mypy = "^1.18"
 pip = "^26.0.1"
 pre-commit = "^4.4"
 pylint = "^4.0"
-pytest = "^9.0"
+pytest = "^9.1"
 pytest-cov = "^7.0"
 pytest-mypy = "^1.0"
 Sphinx = "^8"
 defusedxml = "^0.7"
 types-requests = "^2.32.4.20250913"
+mypy = "^2.1"
 
 [tool.black]
 line-length = 100

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/conditions.py

@@ -223,10 +223,18 @@ class ConditionSelector(ConditionItem):
         else:
             r = re.compile(self.pattern.replace("*", ".*"))
 
+        # When a filter is applied to a rule its detection identifiers are renamed to
+        # start with a `_filt_<random>_` prefix, and its condition patterns receive the
+        # same prefix.  We therefore allow `_`-prefixed identifiers to be matched when
+        # the pattern itself starts with `_` (i.e. it is a filter-internal pattern).
+        # For patterns that do NOT start with `_` (i.e. rule-level patterns such as
+        # "1 of selection_*") the original restriction is kept so that filter identifiers
+        # are never accidentally pulled into the rule's own selectors.
         return [
             ConditionIdentifier([identifier])
             for identifier in detections.detections.keys()
-            if r.match(identifier) and not identifier.startswith("_")
+            if r.match(identifier)
+            and (self.pattern.startswith("_") or not identifier.startswith("_"))
         ]
 
     def postprocess(

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/data/mitre_attack.py

@@ -65,7 +65,7 @@ def _load_mitre_attack_data() -> dict[str, Any]:
     - mitre_attack_mitigations: dict[str, str] mapping mitigation IDs to names
     """
     cache = _get_cache()
-    cache_key = f"mitre_attack_data_{_custom_url or 'default'}"
+    cache_key = "mitre_attack_data"
 
     # Try to get from cache first
     cached_data = cache.get(cache_key)

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/data/mitre_d3fend.py

@@ -51,7 +51,7 @@ def _load_mitre_d3fend_data() -> dict[str, Any]:
     - mitre_d3fend_artifacts: dict[str, str] mapping artifact IDs to names
     """
     cache = _get_cache()
-    cache_key = f"mitre_d3fend_data_{_custom_url or 'default'}"
+    cache_key = "mitre_d3fend_data"
 
     # Try to get from cache first
     cached_data = cache.get(cache_key)

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/filters.py

@@ -206,23 +206,53 @@ class SigmaFilter(SigmaRuleBase):
 
         return True
 
+    # Keywords that must not be prefixed when rewriting filter conditions
+    _CONDITION_KEYWORDS: frozenset[str] = frozenset({"not", "and", "or", "all", "any", "of", "1"})
+
     def apply_on_rule(
         self: Self, rule: SigmaRule | SigmaCorrelationRule
     ) -> SigmaRule | SigmaCorrelationRule:
         if not self._should_apply_on_rule(rule) or isinstance(rule, SigmaCorrelationRule):
             return rule
 
-        filter_condition = self.filter.condition[0]
-        for original_cond_name, condition in self.filter.detections.items():
-            cond_name = "_filt_" + ("".join(random.choices(string.ascii_lowercase, k=10)))
+        # Generate one random prefix shared by all filter identifiers in this application.
+        # Using a single prefix (rather than a fresh random name per identifier) preserves
+        # the structure of the original identifier names so that wildcard patterns in the
+        # filter condition (e.g. "1 of selection_*") continue to work after renaming.
+        prefix = "_filt_" + "".join(random.choices(string.ascii_lowercase, k=10))
 
-            # Replace each instance of the original condition name with the new condition name to avoid conflicts
-            filter_condition = re.sub(
-                rf"(\s|\(|^){original_cond_name}(\s|$|\))",
-                r"\1" + cond_name + r"\2",
-                filter_condition,
-            )
-            rule.detection.detections[cond_name] = condition
+        # Rename every filter detection identifier with the shared prefix.
+        for original_cond_name, condition in self.filter.detections.items():
+            rule.detection.detections[prefix + "_" + original_cond_name] = condition
+
+        # Rewrite the filter condition string so that every identifier/pattern token is
+        # prefixed.  This handles:
+        #   - exact names:        "selection"    -> "PREFIX_selection"
+        #   - suffix wildcards:   "selection_*"  -> "PREFIX_selection_*"
+        #   - prefix wildcards:   "*_allow"      -> "PREFIX_*_allow"
+        #   - the "them" keyword: "1 of them"    -> "1 of PREFIX_*"
+        # Sigma keywords (not, and, or, all, any, of, 1) are left unchanged.
+        #
+        # The regex matches a single Sigma condition token: an optional leading `*`
+        # (wildcard prefix) or a letter, followed by alphanumerics, `*`, `_`, or `-`.
+        # Wildcards are only valid at the start or end of a Sigma identifier pattern
+        # but this regex accepts any occurrence; the Sigma condition parser is
+        # responsible for rejecting syntactically invalid patterns at parse time.
+        def _replace_token(m: re.Match[str]) -> str:
+            token = m.group(0)
+            if token.lower() in self._CONDITION_KEYWORDS:
+                return token
+            if token == "them":
+                # "them" means all detections; replace with a pattern that matches all
+                # filter identifiers carrying the current prefix.
+                return prefix + "_*"
+            return prefix + "_" + token
+
+        filter_condition = re.sub(
+            r"[a-zA-Z*][a-zA-Z0-9*_-]*",
+            _replace_token,
+            self.filter.condition[0],
+        )
 
         for i, condition_str in enumerate(rule.detection.condition):
             rule.detection.condition[i] = f"({condition_str}) and " + f"({filter_condition})"

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/modifiers.py

@@ -387,6 +387,14 @@ class SigmaAllModifier(SigmaListModifier[Any, Any]):
         return val
 
 
+class SigmaNegateModifier(SigmaListModifier[Any, Any]):
+    """Negate the match - turns the detection item into a NOT match."""
+
+    def modify(self, val: Any) -> Any:
+        self.detection_item.negated = True
+        return val
+
+
 class SigmaCompareModifier(SigmaValueModifier[SigmaNumber, SigmaCompareExpression]):
     """Base class for numeric comparison operator modifiers."""
 
@@ -526,6 +534,7 @@ class SigmaTimestampYearModifier(SigmaTimestampModifier):
 # Mapping from modifier identifier strings to modifier classes
 modifier_mapping: dict[str, Type[SigmaModifier[Any, Any]]] = {
     "all": SigmaAllModifier,
+    "neq": SigmaNegateModifier,
     "base64": SigmaBase64Modifier,
     "base64offset": SigmaBase64OffsetModifier,
     "cased": SigmaCaseSensitiveModifier,

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/processing/pipeline.py

@@ -25,6 +25,7 @@ from sigma.processing.finalization import Finalizer, NestedFinalizer, finalizers
 from sigma.processing.templates import TemplateBase
 from sigma.processing.tracking import FieldMappingTracking
 from sigma.processing.transformations import transformations
+from sigma.processing.transformations.external import ExternalSourceBaseTransformation
 from sigma.rule import SigmaDetectionItem, SigmaRule
 from sigma.correlations import SigmaCorrelationRule
 from sigma.processing.transformations.base import PreprocessingTransformation, Transformation
@@ -74,6 +75,7 @@ class ProcessingItemBase:
         transformations: dict[str, Type[Transformation]],
         allow_template_vars: bool = False,
         vars_allowed_paths: tuple[str, ...] | None = None,
+        allow_external_sources: bool = False,
     ) -> dict[str, Any]:
         """Return class instantiation parameters for attributes contained in base class for further
         usage in similar methods of classes inherited from this class."""
@@ -103,6 +105,7 @@ class ProcessingItemBase:
                 transformations,
                 allow_template_vars=allow_template_vars,
                 vars_allowed_paths=vars_allowed_paths,
+                allow_external_sources=allow_external_sources,
             ),
         }
 
@@ -307,6 +310,7 @@ class ProcessingItemBase:
         transformations: dict[str, Type[Transformation]],
         allow_template_vars: bool = False,
         vars_allowed_paths: tuple[str, ...] | None = None,
+        allow_external_sources: bool = False,
     ) -> Transformation:
         try:
             transformation_class_name = d["type"]
@@ -341,11 +345,14 @@ class ProcessingItemBase:
                 "id",
                 "allow_template_vars",
                 "vars_allowed_paths",
+                "allow_external_sources",
             }
         }
         if issubclass(transformation_class, TemplateBase):
             params["allow_template_vars"] = allow_template_vars
             params["vars_allowed_paths"] = vars_allowed_paths
+        if issubclass(transformation_class, ExternalSourceBaseTransformation):
+            params["allow_external_sources"] = allow_external_sources
         try:
             return transformation_class(**params)
         except (SigmaConfigurationError, TypeError) as e:
@@ -420,9 +427,17 @@ class ProcessingItem(ProcessingItemBase):
     field_name_condition_expression: ConditionExpression | None = None
 
     @classmethod
-    def from_dict(cls, d: dict[str, Any]) -> "ProcessingItem":
+    def from_dict(
+        cls,
+        d: dict[str, Any],
+        allow_external_sources: bool = False,
+    ) -> "ProcessingItem":
         """Instantiate processing item from parsed definition and variables."""
-        kwargs = super()._base_args_from_dict(d, transformations)
+        kwargs = super()._base_args_from_dict(
+            d,
+            transformations,
+            allow_external_sources=allow_external_sources,
+        )
 
         detection_item_conds = cls._parse_conditions(
             cast(Mapping[str, Type[ProcessingCondition]], detection_item_conditions),
@@ -762,6 +777,7 @@ class ProcessingPipeline:
         d: dict[str, Any],
         allow_template_vars: bool = False,
         vars_allowed_paths: tuple[str, ...] | None = None,
+        allow_external_sources: bool = False,
     ) -> "ProcessingPipeline":
         """Instantiate processing pipeline from a parsed processing item description."""
 
@@ -788,7 +804,9 @@ class ProcessingPipeline:
         processing_items = list()
         for i, item in enumerate(items):
             try:
-                processing_item = ProcessingItem.from_dict(item)
+                processing_item = ProcessingItem.from_dict(
+                    item, allow_external_sources=allow_external_sources
+                )
                 processing_items.append(processing_item)
             except SigmaConfigurationError as e:
                 raise SigmaConfigurationError(f"Error in processing rule {i + 1}: {str(e)}") from e
@@ -812,6 +830,7 @@ class ProcessingPipeline:
         for fd in fds:
             fd.pop("allow_template_vars", None)  # Strip untrusted YAML value
             fd.pop("vars_allowed_paths", None)  # Strip untrusted YAML value
+            fd.pop("allow_external_sources", None)  # Strip untrusted YAML value
             try:
                 finalizer_type = fd.pop("type")
             except KeyError:
@@ -860,6 +879,7 @@ class ProcessingPipeline:
         allow_template_vars: bool = False,
         vars_allowed_paths: tuple[str, ...] | None = None,
         source_path: str | None = None,
+        allow_external_sources: bool = False,
     ) -> "ProcessingPipeline":
         """Convert YAML input string into processing pipeline.
 
@@ -879,6 +899,7 @@ class ProcessingPipeline:
             parsed_pipeline,
             allow_template_vars=allow_template_vars,
             vars_allowed_paths=vars_allowed_paths,
+            allow_external_sources=allow_external_sources,
         )
 
     def apply(self, rule: SigmaRule | SigmaCorrelationRule) -> SigmaRule | SigmaCorrelationRule:

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/processing/transformations/__init__.py

@@ -23,6 +23,12 @@ from sigma.processing.transformations.placeholder import (
     ValueListPlaceholderTransformation,
     WildcardPlaceholderTransformation,
 )
+from sigma.processing.transformations.external import (
+    ExternalSourceBaseTransformation,
+    FilePlaceholderTransformation,
+    HTTPPlaceholderTransformation,
+    CommandPlaceholderTransformation,
+)
 from sigma.processing.transformations.rule import (
     ChangeLogsourceTransformation,
     SetCustomAttributeTransformation,
@@ -49,6 +55,9 @@ transformations: dict[str, Type[Transformation]] = {
     "wildcard_placeholders": WildcardPlaceholderTransformation,
     "value_placeholders": ValueListPlaceholderTransformation,
     "query_expression_placeholders": QueryExpressionPlaceholderTransformation,
+    "file_placeholders": FilePlaceholderTransformation,
+    "http_placeholders": HTTPPlaceholderTransformation,
+    "command_placeholders": CommandPlaceholderTransformation,
     "add_condition": AddConditionTransformation,
     "change_logsource": ChangeLogsourceTransformation,
     "add_field": AddFieldTransformation,
@@ -81,6 +90,10 @@ __all__ = [
     "WildcardPlaceholderTransformation",
     "ValueListPlaceholderTransformation",
     "QueryExpressionPlaceholderTransformation",
+    "ExternalSourceBaseTransformation",
+    "FilePlaceholderTransformation",
+    "HTTPPlaceholderTransformation",
+    "CommandPlaceholderTransformation",
     "AddConditionTransformation",
     "ChangeLogsourceTransformation",
     "AddFieldTransformation",

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/processing/transformations/base.py

@@ -183,6 +183,29 @@ class FieldMappingTransformationBase(DetectionItemTransformation):
         else:
             return [field]
 
+    def apply_detection(self, detection: SigmaDetection) -> None:
+        for i, detection_item in enumerate(detection.detection_items):
+            if isinstance(detection_item, SigmaDetection):  # recurse into nested detection items
+                self.apply_detection(detection_item)
+            else:
+                # Save a reference to the value list *before* the transformation so we can
+                # detect whether the transformation replaced it with a new list object.
+                # Field-only renames keep the same list instance; value-modifying operations
+                # (e.g. keyword-to-field mapping that adds wildcards, field-reference remapping)
+                # always assign a new list to detection_item.value.
+                value_before = detection_item.value
+                if (
+                    self.processing_item is None
+                    or self.processing_item.match_detection_item(detection_item)
+                ) and (r := self.apply_detection_item(detection_item)) is not None:
+                    if isinstance(r, SigmaDetectionItem) and r.value is not value_before:
+                        # The value list was replaced, so original_value is no longer in sync
+                        # with the current values. Disable conversion to prevent to_plain()
+                        # from producing stale output.
+                        r.disable_conversion_to_plain()
+                    detection.detection_items[i] = r
+                    self.processing_item_applied(r)
+
     def apply(
         self,
         rule: SigmaRule | SigmaCorrelationRule,

pysigma-1.4.0/sigma/processing/transformations/external.py

@@ -0,0 +1,390 @@
+"""External data source placeholder transformations.
+
+These transformations replace Sigma placeholders with values fetched from external sources
+such as local files, HTTP endpoints, or command output.
+
+Security note: Because these transformations access external sources, they are **disabled by
+default** and must be explicitly enabled by passing ``allow_external_sources=True`` when
+loading a :class:`~sigma.processing.pipeline.ProcessingPipeline` or by setting the
+environment variable ``PYSIGMA_ALLOW_EXTERNAL_SOURCES=1``.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+import re
+import subprocess
+from abc import abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+import yaml
+
+from sigma.exceptions import SigmaConfigurationError, SigmaSecurityError, SigmaValueError
+from sigma.processing.transformations.placeholder import BasePlaceholderTransformation
+from sigma.types import Placeholder, SigmaString
+
+PYSIGMA_ALLOW_EXTERNAL_SOURCES_ENV = "PYSIGMA_ALLOW_EXTERNAL_SOURCES"
+
+# Default cap on the amount of data accepted from an external source (10 MiB).
+DEFAULT_MAX_RESPONSE_BYTES = 10 * 1024 * 1024
+
+# Data formats understood by the external source parsers.
+SUPPORTED_FORMATS = ("plaintext", "csv", "json", "yaml")
+
+
+@dataclass
+class ExternalSourceBaseTransformation(BasePlaceholderTransformation):
+    """Base class for placeholder transformations that fetch replacement values from
+    an external source (file, HTTP, command).
+
+    **Supported formats** (controlled by the *format* parameter):
+
+    * ``"plaintext"`` — one value per line; an optional *filter* regex must
+      match for a line to be included.
+    * ``"csv"`` — CSV data; *csv_column* selects the column (column header
+      name **or** 0-based integer index); *csv_has_header* (default ``True``)
+      controls whether the first row is treated as a header; an optional
+      *filter* regex is applied to each extracted cell value.
+    * ``"json"`` — JSON data; *jq_expression* selects the value(s).
+    * ``"yaml"`` — YAML data; *jq_expression* selects the value(s).
+
+    **Security**: external-source transformations are disabled by default.
+    Enable them by passing ``allow_external_sources=True`` when loading the
+    pipeline or by setting the environment variable
+    ``PYSIGMA_ALLOW_EXTERNAL_SOURCES=1``.
+    """
+
+    format: str = "plaintext"
+    filter: str | None = None
+    csv_column: str | int | None = None
+    csv_has_header: bool = True
+    jq_expression: str | None = None
+    allow_external_sources: bool = False
+
+    _values_cache: list[str] | None = field(init=False, default=None, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        if self.format not in SUPPORTED_FORMATS:
+            raise SigmaConfigurationError(
+                f"Unknown external source format '{self.format}'. "
+                f"Supported formats: {', '.join(SUPPORTED_FORMATS)}."
+            )
+        super().__post_init__()
+
+    def _external_sources_allowed(self) -> bool:
+        """Return *True* if external data sources are permitted."""
+        if self.allow_external_sources:
+            return True
+        return os.environ.get(PYSIGMA_ALLOW_EXTERNAL_SOURCES_ENV, "").lower() in (
+            "1",
+            "true",
+        )
+
+    @abstractmethod
+    def _fetch_data(self) -> str:
+        """Fetch raw text data from the external source.
+
+        Subclasses **must** override this method.
+        """
+
+    def _get_values(self) -> list[str]:
+        """Return the list of replacement values, fetching and caching them if necessary.
+
+        Raises :class:`~sigma.exceptions.SigmaSecurityError` when external
+        data sources are not enabled.
+        """
+        if self._values_cache is not None:
+            return self._values_cache
+
+        if not self._external_sources_allowed():
+            raise SigmaSecurityError(
+                "External data source transformations are disabled by default for security "
+                "reasons. Enable them with allow_external_sources=True when loading the "
+                "pipeline or by setting the environment variable "
+                f"{PYSIGMA_ALLOW_EXTERNAL_SOURCES_ENV}=1."
+            )
+
+        data = self._fetch_data()
+        self._values_cache = self._parse_data(data)
+        return self._values_cache
+
+    def _parse_data(self, data: str) -> list[str]:
+        """Dispatch to the appropriate format parser."""
+        if self.format == "plaintext":
+            return self._parse_plaintext(data)
+        elif self.format == "csv":
+            return self._parse_csv(data)
+        elif self.format == "json":
+            return self._parse_json(data)
+        elif self.format == "yaml":
+            return self._parse_yaml_data(data)
+        else:
+            raise SigmaConfigurationError(
+                f"Unknown external source format '{self.format}'. "
+                f"Supported formats: {', '.join(SUPPORTED_FORMATS)}."
+            )
+
+    def _parse_plaintext(self, data: str) -> list[str]:
+        values = [line for line in (line.strip() for line in data.splitlines()) if line]
+        if self.filter:
+            pattern = re.compile(self.filter)
+            values = [v for v in values if pattern.search(v)]
+        return values
+
+    def _parse_csv(self, data: str) -> list[str]:
+        if self.csv_column is None:
+            raise SigmaConfigurationError("'csv_column' must be specified when format is 'csv'")
+
+        values: list[str] = []
+
+        if isinstance(self.csv_column, str):
+            if not self.csv_has_header:
+                raise SigmaConfigurationError(
+                    "CSV column referenced by name requires a header row (csv_has_header=True)"
+                )
+            reader_dict = csv.DictReader(io.StringIO(data))
+            for row in reader_dict:
+                if self.csv_column not in row:
+                    raise SigmaConfigurationError(
+                        f"CSV column '{self.csv_column}' not found in data"
+                    )
+                val = row[self.csv_column]
+                if val is not None:
+                    values.append(val)
+        else:
+            col_idx = self.csv_column
+            rows = iter(csv.reader(io.StringIO(data)))
+            if self.csv_has_header:
+                next(rows, None)  # discard header row for consistency with name-based mode
+            for csv_row in rows:
+                if col_idx < len(csv_row):
+                    values.append(csv_row[col_idx])
+
+        if self.filter:
+            pattern = re.compile(self.filter)
+            values = [v for v in values if pattern.search(v)]
+        return values
+
+    def _parse_json(self, data: str) -> list[str]:
+        import jq  # type: ignore[import-not-found]
+
+        if self.jq_expression is None:
+            raise SigmaConfigurationError("'jq_expression' must be specified when format is 'json'")
+        try:
+            parsed = json.loads(data)
+        except json.JSONDecodeError as e:
+            raise SigmaValueError(f"Failed to parse JSON data: {e}") from e
+        try:
+            result = jq.all(self.jq_expression, parsed)
+        except ValueError as e:
+            raise SigmaConfigurationError(f"Invalid jq expression: {e}") from e
+        return self._jq_results_to_values(result)
+
+    def _parse_yaml_data(self, data: str) -> list[str]:
+        import jq
+
+        if self.jq_expression is None:
+            raise SigmaConfigurationError("'jq_expression' must be specified when format is 'yaml'")
+        try:
+            parsed = yaml.safe_load(data)
+        except yaml.YAMLError as e:
+            raise SigmaValueError(f"Failed to parse YAML data: {e}") from e
+        try:
+            result = jq.all(self.jq_expression, parsed)
+        except ValueError as e:
+            raise SigmaConfigurationError(f"Invalid jq expression: {e}") from e
+        return self._jq_results_to_values(result)
+
+    @staticmethod
+    def _jq_results_to_values(result: Iterable[Any]) -> list[str]:
+        """Convert jq output into scalar placeholder values.
+
+        Each placeholder replacement becomes an individual field match value,
+        so a jq result must be a scalar. An expression that yields an array or
+        object is rejected with guidance to project to scalars (e.g. use
+        ``.items[]`` instead of ``.items``). ``None`` results are skipped.
+        """
+        values: list[str] = []
+        for v in result:
+            if v is None:
+                continue
+            if isinstance(v, (dict, list)):
+                raise SigmaConfigurationError(
+                    "jq_expression must select scalar values for placeholder "
+                    f"replacement, but a {type(v).__name__} was returned; project "
+                    "to scalars (e.g. '.items[]' instead of '.items')"
+                )
+            values.append(str(v))
+        return values
+
+    def placeholder_replacements(self, p: Placeholder) -> Iterable[SigmaString]:
+        return [SigmaString(v) for v in self._get_values()]
+
+
+@dataclass
+class FilePlaceholderTransformation(ExternalSourceBaseTransformation):
+    """Replace placeholders with values read from a local file.
+
+    Parameters:
+    * **path** — path to the file (required)
+    * **format** — data format: ``"plaintext"`` (default), ``"csv"``, ``"json"``, ``"yaml"``
+    * **filter** — optional regex that each value must match (plaintext/csv)
+    * **csv_column** — column name (str) or 0-based index (int) for CSV format
+    * **csv_has_header** — whether the first CSV row is a header (default ``True``)
+    * **jq_expression** — path expression for JSON/YAML formats (e.g. ``.items[]``)
+    * **include** / **exclude** — placeholder name lists (from
+      :class:`~sigma.processing.transformations.placeholder.BasePlaceholderTransformation`)
+    """
+
+    path: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.path:
+            raise SigmaConfigurationError(
+                "FilePlaceholderTransformation requires a non-empty 'path'"
+            )
+        super().__post_init__()
+
+    def _fetch_data(self) -> str:
+        try:
+            with open(self.path, encoding="utf-8") as fh:
+                return fh.read()
+        except OSError as e:
+            raise SigmaValueError(
+                f"FilePlaceholderTransformation: failed to read '{self.path}': {e}"
+            ) from e
+
+
+@dataclass
+class HTTPPlaceholderTransformation(ExternalSourceBaseTransformation):
+    """Replace placeholders with values fetched from an HTTP(S) endpoint.
+
+    Parameters:
+    * **url** — URL to fetch (required)
+    * **method** — HTTP method (default: ``"GET"``)
+    * **timeout** — request timeout in seconds (default: 10)
+    * **headers** — optional dict of custom HTTP request headers
+    * **params** — optional dict of URL query parameters
+    * **form_data** — optional dict to send as a form-encoded request body
+      (``application/x-www-form-urlencoded``)
+    * **json_body** — optional dict to send as a JSON request body
+      (``application/json``)
+    * **max_body_size** — maximum response body size in bytes; the fetch is
+      aborted with an error once this many bytes have been read (default 10 MiB)
+    * **format** — data format: ``"plaintext"`` (default), ``"csv"``, ``"json"``, ``"yaml"``
+    * **filter** — optional regex that each value must match (plaintext/csv)
+    * **csv_column** — column name (str) or 0-based index (int) for CSV format
+    * **csv_has_header** — whether the first CSV row is a header (default ``True``)
+    * **jq_expression** — path expression for JSON/YAML formats
+    * **include** / **exclude** — placeholder name lists
+    """
+
+    url: str = ""
+    method: str = "GET"
+    timeout: int = 10
+    headers: dict[str, str] | None = None
+    params: dict[str, str] | None = None
+    form_data: dict[str, Any] | None = None
+    json_body: dict[str, Any] | None = None
+    max_body_size: int = DEFAULT_MAX_RESPONSE_BYTES
+
+    def __post_init__(self) -> None:
+        if not self.url:
+            raise SigmaConfigurationError(
+                "HTTPPlaceholderTransformation requires a non-empty 'url'"
+            )
+        super().__post_init__()
+
+    def _fetch_data(self) -> str:
+        import requests
+
+        try:
+            with requests.request(
+                method=self.method,
+                url=self.url,
+                timeout=self.timeout,
+                headers=self.headers,
+                params=self.params,
+                data=self.form_data,
+                json=self.json_body,
+                stream=True,
+            ) as response:
+                response.raise_for_status()
+                content = bytearray()
+                for chunk in response.iter_content(chunk_size=8192):
+                    content.extend(chunk)
+                    if len(content) > self.max_body_size:
+                        raise SigmaValueError(
+                            f"HTTPPlaceholderTransformation: response from '{self.url}' "
+                            f"exceeds max_body_size ({self.max_body_size} bytes)"
+                        )
+                encoding = response.encoding or response.apparent_encoding or "utf-8"
+                return content.decode(encoding, errors="replace")
+        except requests.RequestException as e:
+            raise SigmaValueError(
+                f"HTTPPlaceholderTransformation: failed to fetch '{self.url}': {e}"
+            ) from e
+
+
+@dataclass
+class CommandPlaceholderTransformation(ExternalSourceBaseTransformation):
+    """Replace placeholders with the stdout output of a shell command.
+
+    Parameters:
+    * **cmd** — command string (passed to ``/bin/sh -c``) or a list of
+      arguments (required)
+    * **timeout** — maximum execution time in seconds (default: 30)
+    * **max_stdout** — maximum accepted stdout size in bytes; output larger
+      than this is rejected with an error (default 10 MiB)
+    * **format** — data format: ``"plaintext"`` (default), ``"csv"``, ``"json"``, ``"yaml"``
+    * **filter** — optional regex that each value must match (plaintext/csv)
+    * **csv_column** — column name (str) or 0-based index (int) for CSV format
+    * **csv_has_header** — whether the first CSV row is a header (default ``True``)
+    * **jq_expression** — path expression for JSON/YAML formats
+    * **include** / **exclude** — placeholder name lists
+    """
+
+    cmd: str | list[str] = ""
+    timeout: int = 30
+    max_stdout: int = DEFAULT_MAX_RESPONSE_BYTES
+
+    def __post_init__(self) -> None:
+        if not self.cmd:
+            raise SigmaConfigurationError(
+                "CommandPlaceholderTransformation requires a non-empty 'cmd'"
+            )
+        super().__post_init__()
+
+    def _fetch_data(self) -> str:
+        try:
+            result = subprocess.run(
+                self.cmd,
+                shell=isinstance(self.cmd, str),
+                capture_output=True,
+                text=True,
+                timeout=self.timeout,
+            )
+        except subprocess.TimeoutExpired as e:
+            raise SigmaValueError(
+                f"CommandPlaceholderTransformation: command timed out: {e}"
+            ) from e
+        except OSError as e:
+            raise SigmaValueError(
+                f"CommandPlaceholderTransformation: failed to execute command: {e}"
+            ) from e
+
+        if result.returncode != 0:
+            raise SigmaValueError(
+                f"CommandPlaceholderTransformation: command exited with code "
+                f"{result.returncode}: {result.stderr.strip()}"
+            )
+        if len(result.stdout.encode("utf-8", errors="replace")) > self.max_stdout:
+            raise SigmaValueError(
+                f"CommandPlaceholderTransformation: command output exceeds "
+                f"max_stdout ({self.max_stdout} bytes)"
+            )
+        return result.stdout

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/rule/base.py

@@ -25,7 +25,7 @@ class SigmaYAMLLoader(yaml.CSafeLoader):
     def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Any, Any]:
         keys = set()
         for k, v in node.value:
-            key = self.construct_object(k, deep=deep)  # type: ignore
+            key = self.construct_object(k, deep=deep)
             if key in keys:
                 raise yaml.error.YAMLError("Duplicate key '{k}'")
             else:

{pysigma-1.3.2 → pysigma-1.4.0}/sigma/rule/detection.py

@@ -12,6 +12,7 @@ from sigma.conditions import (
     ConditionAND,
     ConditionFieldEqualsValueExpression,
     ConditionItem,
+    ConditionNOT,
     ConditionOR,
     ConditionValueExpression,
     ParentChainMixin,
@@ -67,6 +68,7 @@ class SigmaDetectionItem(ProcessingItemTrackingMixin, ParentChainMixin):
     modifiers: list[type[SigmaModifier[Any, Any]]]
     value: list[SigmaType]
     value_linking: type[ConditionAND | ConditionOR] = ConditionOR
+    negated: bool = False
     source: SigmaRuleLocation | None = dataclasses.field(default=None, compare=False)
     original_value: list[SigmaType] | None = dataclasses.field(
         init=False, repr=False, hash=False, compare=False
@@ -241,16 +243,16 @@ class SigmaDetectionItem(ProcessingItemTrackingMixin, ParentChainMixin):
                     "Null value must be bound to a field", source=self.source
                 )
             else:
-                return ConditionFieldEqualsValueExpression(self.field, SigmaNull()).postprocess(
+                cond = ConditionFieldEqualsValueExpression(self.field, SigmaNull()).postprocess(
                     detections, self, self.source
                 )
-        if len(self.value) == 1:  # single value: return key/value or value-only expression
+        elif len(self.value) == 1:  # single value: return key/value or value-only expression
             if self.field is None:
-                return ConditionValueExpression(self.value[0]).postprocess(
+                cond = ConditionValueExpression(self.value[0]).postprocess(
                     detections, self, self.source
                 )
             else:
-                return ConditionFieldEqualsValueExpression(self.field, self.value[0]).postprocess(
+                cond = ConditionFieldEqualsValueExpression(self.field, self.value[0]).postprocess(
                     detections, self, self.source
                 )
         else:  # more than one value, return logically linked values or an "in" expression
@@ -261,7 +263,16 @@ class SigmaDetectionItem(ProcessingItemTrackingMixin, ParentChainMixin):
                     [ConditionFieldEqualsValueExpression(self.field, v) for v in self.value]
                 )
             cond.postprocess(detections, parent, self.source)
-            return cond
+
+        if self.negated:
+            if cond is None:
+                return None
+            not_cond = ConditionNOT([cond])
+            not_cond.parent = parent
+            not_cond.source = self.source
+            cond.parent = not_cond
+            return not_cond
+        return cond
 
     def is_keyword(self: Self) -> bool:
         """Returns True if detection item is a keyword detection without field reference."""