detectmatelibrary 0.3.1__py3-none-any.whl

detectmatelibrary/__init__.py

@@ -0,0 +1,24 @@
+from . import common
+from . import detectors
+from . import parsers
+from . import schemas
+from . import utils
+from .metadata import (__authors__, __contact__, __copyright__, __date__, __deprecated__, __website__,
+                       __license__, __status__, __version__)
+
+__all__ = [
+    "common",
+    "detectors",
+    "parsers",
+    "schemas",
+    "utils",
+    "__authors__",
+    "__contact__",
+    "__copyright__",
+    "__date__",
+    "__deprecated__",
+    "__website__",
+    "__license__",
+    "__status__",
+    "__version__"
+]

detectmatelibrary/common/_config/__init__.py

@@ -0,0 +1,119 @@
+from ._compile import ConfigMethods, generate_detector_config
+from ._formats import EventsConfig
+
+__all__ = ["ConfigMethods", "generate_detector_config", "EventsConfig", "BasicConfig"]
+
+from pydantic import BaseModel, ConfigDict
+
+from typing_extensions import Self
+from typing import Any, Dict
+from copy import deepcopy
+
+from numpy.random import choice
+import string
+
+
+def random_id(length: int = 10) -> str:
+    characters = [s for s in string.ascii_letters + string.digits]
+    return "".join(str(choice(characters)) for _ in range(length))
+
+
+class BasicConfig(BaseModel):
+    """Base configuration class with helper methods."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    method_type: str = "default_method_type"
+    component_type: str = "default_type"
+
+    auto_config: bool = False
+
+    def get_config(self) -> Dict[str, Any]:
+        """Return the configuration as a dictionary."""
+        return self.model_dump()
+
+    def update_config(self, new_config: Dict[str, Any]) -> None:
+        """Update the configuration with new values."""
+        for key, value in new_config.items():
+            setattr(self, key, value)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any], method_id: str) -> Self:
+        aux = cls()
+        config_ = ConfigMethods.get_method(
+            deepcopy(data), component_type=aux.component_type, method_id=method_id
+        )
+        ConfigMethods.check_type(config_, method_type=aux.method_type)
+
+        return cls(**ConfigMethods.process(config_))
+
+    def to_dict(self, method_id: str = random_id()) -> Dict[str, Any]:
+        """Convert the config back to YAML-compatible dictionary format.
+
+        This is the inverse of from_dict() and ensures yaml -> pydantic -> yaml preservation.
+
+        Args:
+            method_id: The method identifier to use in the output structure
+
+        Returns:
+            Dictionary with structure: {component_type: {method_id: config_data}}
+        """
+        # Build the config in the format expected by from_dict
+        result: Dict[str, Any] = {
+            "method_type": self.method_type,
+            "auto_config": self.auto_config,
+        }
+
+        # Collect all non-meta fields for params
+        params = {}
+        events_data = None
+        instances_data = None
+        persist_data: dict[str, Any] | None = None
+
+        for field_name, field_value in self:
+            # Skip meta fields
+            if field_name in ("component_type", "method_type", "auto_config"):
+                continue
+
+            # Handle EventsConfig specially
+            if field_name == "events":
+                if field_value is not None:
+                    if isinstance(field_value, EventsConfig):
+                        events_data = field_value.to_dict()
+                    else:
+                        events_data = field_value
+            # Handle global instances specially (top-level, not in params)
+            # Serialized as "global" in YAML (Python field is "global_instances")
+            elif field_name == "global_instances" and field_value:
+                instances_data = {
+                    name: inst.to_dict()
+                    for name, inst in field_value.items()
+                }
+            elif field_name == "persist":
+                if field_value is not None:
+                    persist_data = field_value.model_dump()
+            else:
+                # All other fields go into params
+                params[field_name] = field_value
+
+        # Add params if there are any
+        if params:
+            result["params"] = params
+
+        # Add global instances if they exist (serialized as "global" in YAML)
+        if instances_data is not None:
+            result["global"] = instances_data
+
+        # Add events if they exist
+        if events_data is not None:
+            result["events"] = events_data
+
+        if persist_data is not None:
+            result["persist"] = persist_data
+
+        # Wrap in the component_type and method_id structure
+        return {
+            self.component_type: {
+                method_id: result
+            }
+        }

detectmatelibrary/common/_config/_compile.py

@@ -0,0 +1,212 @@
+from detectmatelibrary.common._config._formats import EventsConfig, _EventInstance
+
+from typing import Any, Dict, List, Sequence, Tuple, Union
+import warnings
+import re
+
+
+def _classify_variables(
+    var_names: Sequence[str], var_pattern: re.Pattern[str]
+) -> Dict[str, Any]:
+    """Classify variable names into positional and header variables.
+
+    Returns an instance config dict with 'params', 'variables', and
+    'header_variables' keys.
+    """
+    positional_variables = []
+    header_variables = []
+
+    for var_name in var_names:
+        match = var_pattern.match(var_name)
+        if match:
+            position = int(match.group(1))
+            positional_variables.append({
+                "pos": position,
+                "name": var_name,
+                "params": {}
+            })
+        else:
+            header_variables.append({
+                "pos": var_name,
+                "params": {}
+            })
+
+    instance_config: Dict[str, Any] = {"params": {}}
+    if positional_variables:
+        instance_config["variables"] = positional_variables
+    if header_variables:
+        instance_config["header_variables"] = header_variables
+
+    return instance_config
+
+
+class MethodNotFoundError(Exception):
+    def __init__(self, method_id: str, component_type: str) -> None:
+        super().__init__(
+            f"Method '{method_id}' of type '{component_type}' not found in configuration."
+        )
+
+
+class TypeNotFoundError(Exception):
+    def __init__(self, type_name: str) -> None:
+        super().__init__(f"Type '{type_name}' not found in configuration.")
+
+
+class MethodTypeNotMatch(Exception):
+    def __init__(self, expected_type: str, actual_type: str) -> None:
+        super().__init__(
+            f"Type mismatch: expected '{expected_type}', got '{actual_type}'."
+        )
+
+
+class MissingParamsWarning(UserWarning):
+    def __init__(self) -> None:
+        super().__init__(
+            "'auto_config = False' and no 'params', 'events', 'global', or 'persist' provided. "
+            "Is that intended?"
+        )
+
+
+class AutoConfigWarning(UserWarning):
+    def __init__(self) -> None:
+        super().__init__("'auto_config = True' will overwrite 'events' and 'params'.")
+
+
+class ConfigMethods:
+    @staticmethod
+    def get_method(
+        config: Dict[str, Dict[str, Dict[str, Any]]], method_id: str, component_type: str
+    ) -> Dict[str, Any]:
+
+        if component_type not in config:
+            raise TypeNotFoundError(component_type)
+
+        args = config[component_type]
+        if method_id not in args:
+            raise MethodNotFoundError(method_id, component_type)
+
+        return args[method_id]
+
+    @staticmethod
+    def check_type(config: Dict[str, Any], method_type: str) -> None:
+        if config["method_type"] != method_type:
+            raise MethodTypeNotMatch(expected_type=method_type, actual_type=config["method_type"])
+
+    @staticmethod
+    def process(config: Dict[str, Any]) -> Dict[str, Any]:
+        has_params = "params" in config
+        has_events = "events" in config
+        has_instances = "global" in config
+        has_persist = "persist" in config
+
+        no_data = not has_params and not has_events and not has_instances and not has_persist
+        if no_data and not config.get("auto_config", False):
+            warnings.warn(MissingParamsWarning())
+
+        if has_params:
+            if config.get("auto_config", False):
+                warnings.warn(AutoConfigWarning())
+
+            config.update(config["params"])
+            config.pop("params")
+
+        # Handle "events" key at top level
+        if has_events:
+            config["events"] = EventsConfig._init(config["events"])
+
+        # Handle "global" key: event-ID-independent global instances
+        # Renamed to "global_instances" to avoid collision with Python keyword
+        if has_instances:
+            config["global_instances"] = {
+                name: _EventInstance._init(**data)
+                for name, data in config.pop("global").items()
+            }
+
+        return config
+
+
+def generate_detector_config(
+    variable_selection: Dict[int | str, List[Union[str, Tuple[str, ...]]]],
+    detector_name: str,
+    method_type: str,
+    **additional_params: Any
+) -> Dict[str, Any]:
+    """Generate a detector configuration dictionary from variable selections.
+
+    Transforms a variable selection mapping into the nested configuration
+    structure required by detector configs. Supports both individual variable
+    names (strings) and tuples of variable names. Each tuple produces a
+    separate detector instance in the config.
+
+    Args:
+        variable_selection: Maps event_id to list of variable names or tuples
+            of variable names. Strings are grouped into a single instance.
+            Each tuple becomes its own instance. Variable names matching
+            'var_\\d+' are positional template variables; others are header
+            variables.
+        detector_name: Name of the detector, used as the base instance_id.
+        method_type: Type of detection method (e.g., "new_value_detector").
+        **additional_params: Additional parameters for the detector's params
+            dict (e.g., max_combo_size=3).
+
+    Returns:
+        Dictionary with structure compatible with detector config classes.
+
+    Examples:
+        Single variable names (one instance per event)::
+
+            config = generate_detector_config(
+                variable_selection={1: ["var_0", "var_1", "level"]},
+                detector_name="MyDetector",
+                method_type="new_value_detector",
+            )
+
+        Tuples of variable names (one instance per tuple)::
+
+            config = generate_detector_config(
+                variable_selection={1: [("username", "src_ip"), ("var_0", "var_1")]},
+                detector_name="MyDetector",
+                method_type="new_value_combo_detector",
+                max_combo_size=2,
+            )
+    """
+    var_pattern = re.compile(r"^var_(\d+)$")
+
+    events_config: Dict[int | str, Dict[str, Any]] = {}
+
+    for event_id, variable_names in variable_selection.items():
+        instances: Dict[str, Any] = {}
+
+        # Separate plain strings from tuples
+        single_vars: List[str] = []
+        tuple_vars: List[Tuple[str, ...]] = []
+
+        for entry in variable_names:
+            if isinstance(entry, tuple):
+                tuple_vars.append(entry)
+            else:
+                single_vars.append(entry)
+
+        # Plain strings -> single instance keyed by detector_name
+        if single_vars:
+            instances[detector_name] = _classify_variables(single_vars, var_pattern)
+
+        # Each tuple -> its own instance, keyed by joined variable names
+        for combo in tuple_vars:
+            instance_id = f"{detector_name}_{'_'.join(combo)}"
+            instances[instance_id] = _classify_variables(combo, var_pattern)
+
+        events_config[event_id] = instances
+
+    config_dict = {
+        "detectors": {
+            detector_name: {
+                "method_type": method_type,
+                "auto_config": False,
+                "params": additional_params,
+                "events": events_config
+            }
+        }
+    }
+
+    return config_dict

detectmatelibrary/common/_config/_formats.py

@@ -0,0 +1,147 @@
+from pydantic import BaseModel
+
+from typing_extensions import Self
+from typing import Dict, Any
+
+
+# Sub-formats ********************************************************+
+class Variable(BaseModel):
+    pos: str | int
+    name: str = ""
+    params: Dict[str, Any] = {}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Variable to YAML-compatible dictionary."""
+        result: Dict[str, Any] = {
+            "pos": self.pos,
+        }
+        if self.name:
+            result["name"] = self.name
+        if self.params:
+            result["params"] = self.params
+        return result
+
+
+class Header(BaseModel):
+    pos: str
+    params: Dict[str, Any] = {}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Header to YAML-compatible dictionary."""
+        result: Dict[str, Any] = {
+            "pos": self.pos,
+        }
+        if self.params:
+            result["params"] = self.params
+        return result
+
+
+class _EventInstance(BaseModel):
+    """Configuration for a specific instance within an event."""
+    params: Dict[str, Any] = {}
+    variables: Dict[str | int, Variable] = {}
+    header_variables: Dict[str, Header] = {}
+
+    @classmethod
+    def _init(cls, **kwargs: dict[str, Any]) -> "_EventInstance":
+        for var, cl in zip(["variables", "header_variables"], [Variable, Header]):
+            if var in kwargs:
+                new_dict = {}
+                for v in kwargs[var]:
+                    aux = cl(**v)  # type: ignore
+                    new_dict[aux.pos] = aux
+                kwargs[var] = new_dict
+        return cls(**kwargs)  # type: ignore
+
+    def get_all(self) -> Dict[Any, Header | Variable]:
+        return {**self.variables, **self.header_variables}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert _EventInstance to YAML-compatible dictionary."""
+        result: Dict[str, Any] = {}
+        # Always include params even if empty (to match YAML structure)
+        result["params"] = self.params
+        if self.variables:
+            result["variables"] = [var.to_dict() for var in self.variables.values()]
+        if self.header_variables:
+            result["header_variables"] = [header.to_dict() for header in self.header_variables.values()]
+        return result
+
+
+class _EventConfig(BaseModel):
+    """Configuration for all instances of a specific event."""
+    instances: Dict[str, _EventInstance]
+
+    @classmethod
+    def _init(cls, instances_dict: Dict[str, Dict[str, Any]]) -> "_EventConfig":
+        instances = {}
+        for instance_id, instance_data in instances_dict.items():
+            instances[instance_id] = _EventInstance._init(**instance_data)
+        return cls(instances=instances)
+
+    @property
+    def variables(self) -> Dict[str | int, Variable]:
+        """Pass-through to first instance for compatibility."""
+        if self.instances:
+            return next(iter(self.instances.values())).variables
+        return {}
+
+    @property
+    def header_variables(self) -> Dict[str, Header]:
+        """Pass-through to first instance for compatibility."""
+        if self.instances:
+            return next(iter(self.instances.values())).header_variables
+        return {}
+
+    def get_all(self) -> Dict[Any, Header | Variable]:
+        """Pass-through to first instance for compatibility."""
+        if self.instances:
+            return next(iter(self.instances.values())).get_all()
+        return {}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert _EventConfig to YAML-compatible dictionary."""
+        return {
+            instance_id: instance.to_dict()
+            for instance_id, instance in self.instances.items()
+        }
+
+
+# Main-formats ********************************************************+
+class EventsConfig(BaseModel):
+    """Events configuration dict keyed by event_id."""
+    events: Dict[Any, _EventConfig]
+
+    @classmethod
+    def _init(cls, events_dict: Dict[Any, Dict[str, Any]]) -> Self:
+        new_dict = {}
+        for event_id, instances in events_dict.items():
+            new_dict[event_id] = _EventConfig._init(instances)
+        return cls(events=new_dict)
+
+    def __getitem__(self, idx: str | int) -> _EventConfig | None:
+        if idx not in self.events:
+            return None
+        return self.events[idx]
+
+    def __contains__(self, idx: str | int) -> bool:
+        return idx in self.events
+
+    def to_dict(self) -> Dict[Any, Any]:
+        """Convert EventsConfig to YAML-compatible dictionary.
+
+        This unwraps the internal 'events' dict structure to match YAML
+        format.
+        """
+        result = {}
+        for event_id, event_config in self.events.items():
+            # Skip events with no instances
+            if not event_config.instances:
+                continue
+            # Convert string keys back to int if they were originally int
+            try:
+                key = int(event_id)
+            except (ValueError, TypeError):
+                key = event_id
+            result[key] = event_config.to_dict()
+        return result

detectmatelibrary/common/_core_op/_fit_logic.py

@@ -0,0 +1,153 @@
+
+from typing import Literal, get_args
+from enum import Enum
+import warnings
+
+
+class TrainState(Enum):
+    DEFAULT = 0
+    STOP_TRAINING = 1
+    KEEP_TRAINING = 2
+
+    def describe(self) -> str:
+        descriptions = [
+            "Follow default training behavior.",
+            "Force stop training.",
+            "Keep training regardless of default behavior."
+        ]
+
+        return descriptions[self.value]
+
+
+class ConfigState(Enum):
+    DEFAULT = 0
+    STOP_CONFIGURE = 1
+    KEEP_CONFIGURE = 2
+
+    def describe(self) -> str:
+        descriptions = [
+            "Follow default configuration behavior.",
+            "Force stop configuration.",
+            "Keep configuring regardless of default behavior."
+        ]
+        return descriptions[self.value]
+
+
+StatesL = Literal["keep_training", "stop_training", "keep_configuring", "stop_configuring"]
+
+def update_state(
+    state: StatesL, train_state: TrainState, config_state: ConfigState
+) -> tuple[TrainState, ConfigState]:
+    if state == "keep_training":
+        train_state = TrainState.KEEP_TRAINING
+    elif state == "stop_training":
+        train_state = TrainState.STOP_TRAINING
+    elif state == "keep_configuring":
+        config_state = ConfigState.KEEP_CONFIGURE
+    elif state == "stop_configuring":
+        config_state = ConfigState.STOP_CONFIGURE
+    else:
+        warnings.warn(f"State {state} unknown, use: {get_args(StatesL)}")
+
+    return train_state, config_state
+
+
+def do_training(
+    data_use_training: int | None, index: int, train_state: TrainState
+) -> bool:
+    if train_state == TrainState.STOP_TRAINING:
+        return False
+    elif train_state == TrainState.KEEP_TRAINING:
+        return True
+
+    return data_use_training is not None and data_use_training > index
+
+
+def do_configure(
+    data_use_configure: int | None, index: int, configure_state: ConfigState
+) -> bool:
+    if configure_state == ConfigState.STOP_CONFIGURE:
+        return False
+    elif configure_state == ConfigState.KEEP_CONFIGURE:
+        return True
+
+    return data_use_configure is not None and data_use_configure > index
+
+
+class FitLogicState(Enum):
+    DO_CONFIG = 0
+    DO_TRAIN = 1
+    NOTHING = 2
+
+    def describe(self) -> str:
+        descriptions = [
+            "Configuring",
+            "Training.",
+            "Default"
+        ]
+        return descriptions[self.value]
+
+
+class FitLogic:
+    def __init__(
+        self, data_use_configure: int | None, data_use_training: int | None
+    ) -> None:
+        
+        self.train_state = TrainState.DEFAULT
+        self.configure_state = ConfigState.DEFAULT
+        self.last_state = FitLogicState.NOTHING
+
+        self.data_used_train, self.data_used_configure = 0, 0
+        self._configuration_done, self.config_finished = False, False
+        self._training_done, self.training_finished = False, False
+
+        self.data_use_configure = data_use_configure
+        self.data_use_training = data_use_training
+
+    def get_last_state(self) -> str:
+        return self.last_state.describe()
+
+    def update_state(self, state: StatesL) -> None:
+        self.train_state, self.configure_state = update_state(
+            state=state, train_state=self.train_state, config_state=self.configure_state
+        )
+
+    def finish_config(self) -> bool:
+        if self._configuration_done and not self.config_finished:
+            self.config_finished = True
+            return True
+        return False
+
+    def finish_training(self) -> bool:
+        if self._training_done and not self.training_finished:
+            self.training_finished = True
+            return True
+        return False
+
+    def __check_state(self) -> FitLogicState:
+        if do_configure(
+            data_use_configure=self.data_use_configure,
+            index=self.data_used_configure,
+            configure_state=self.configure_state
+        ):
+            self.data_used_configure += 1
+            return FitLogicState.DO_CONFIG
+        else:
+            if self.data_used_configure > 0 and not self._configuration_done:
+                self._configuration_done = True
+
+            if do_training(
+                data_use_training=self.data_use_training,
+                index=self.data_used_train,
+                train_state=self.train_state
+            ):
+                self.data_used_train += 1
+                return FitLogicState.DO_TRAIN
+            elif self.data_used_train > 0 and not self._training_done:
+                self._training_done = True
+
+        return FitLogicState.NOTHING
+    
+    def run(self) -> FitLogicState:
+        self.last_state = self.__check_state()
+        return self.last_state

detectmatelibrary/common/_core_op/_schema_pipeline.py

@@ -0,0 +1,28 @@
+from detectmatelibrary.schemas import BaseSchema
+
+
+from typing import Tuple
+
+
+class SchemaPipeline:
+    @staticmethod
+    def preprocess(
+        input_: BaseSchema, data: BaseSchema | bytes
+    ) -> Tuple[bool, BaseSchema]:
+
+        is_byte = False
+        if isinstance(data, bytes):
+            is_byte = True
+            input_.deserialize(data)
+            data = input_.copy()
+        else:
+            data = data.copy()
+
+        return is_byte, data
+
+    @staticmethod
+    def postprocess(
+        data: BaseSchema, is_byte: bool
+    ) -> BaseSchema | bytes:
+
+        return data if not is_byte else data.serialize()