mloda 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

{mloda_core → mloda/core}/abstract_plugins/components/parallelization_modes.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
 
-class ParallelizationModes(Enum):
+class ParallelizationMode(Enum):
     SYNC = "sync"
     THREADING = "threading"
     MULTIPROCESSING = "multiprocessing"

{mloda_core → mloda/core}/abstract_plugins/components/plugin_option/plugin_collector.py

@@ -1,11 +1,11 @@
 from typing import Set, Type
 
-from mloda_core.abstract_plugins.abstract_feature_group import AbstractFeatureGroup
+from mloda.core.abstract_plugins.feature_group import FeatureGroup
 
 
-class PlugInCollector:
+class PluginCollector:
     """
-    The PlugInCollector class is a helper class with the purpose to disable or enable feature groups.
+    The PluginCollector class is a helper class with the purpose to disable or enable feature groups.
 
     This class is useful for rapid prototype development, where you want to disable or enable feature groups,
     when the other, competing feature groups are found.
@@ -15,16 +15,16 @@ class PlugInCollector:
     """
 
     def __init__(self) -> None:
-        self.disabled_feature_group_classes: Set[Type[AbstractFeatureGroup]] = set()
-        self.enabled_feature_group_classes: Set[Type[AbstractFeatureGroup]] = set()
+        self.disabled_feature_group_classes: Set[Type[FeatureGroup]] = set()
+        self.enabled_feature_group_classes: Set[Type[FeatureGroup]] = set()
 
-    def add_disabled_feature_group_classes(self, feature_group_cls: Set[Type[AbstractFeatureGroup]]) -> None:
+    def add_disabled_feature_group_classes(self, feature_group_cls: Set[Type[FeatureGroup]]) -> None:
         self.disabled_feature_group_classes.update(feature_group_cls)
 
-    def add_enabled_feature_group_classes(self, feature_group_cls: Set[Type[AbstractFeatureGroup]]) -> None:
+    def add_enabled_feature_group_classes(self, feature_group_cls: Set[Type[FeatureGroup]]) -> None:
         self.enabled_feature_group_classes.update(feature_group_cls)
 
-    def applicable_feature_group_class(self, feature_group_cls: Type[AbstractFeatureGroup]) -> bool:
+    def applicable_feature_group_class(self, feature_group_cls: Type[FeatureGroup]) -> bool:
         if feature_group_cls in self.disabled_feature_group_classes:
             return False
 
@@ -38,22 +38,22 @@ class PlugInCollector:
 
     @staticmethod
     def disabled_feature_groups(
-        feature_group_cls: Set[Type[AbstractFeatureGroup]] | Type[AbstractFeatureGroup],
-    ) -> "PlugInCollector":
+        feature_group_cls: Set[Type[FeatureGroup]] | Type[FeatureGroup],
+    ) -> "PluginCollector":
         if not isinstance(feature_group_cls, Set):
             feature_group_cls = {feature_group_cls}
 
-        plugin_collector = PlugInCollector()
+        plugin_collector = PluginCollector()
         plugin_collector.add_disabled_feature_group_classes(feature_group_cls)
         return plugin_collector
 
     @staticmethod
     def enabled_feature_groups(
-        feature_group_cls: Set[Type[AbstractFeatureGroup]] | Type[AbstractFeatureGroup],
-    ) -> "PlugInCollector":
+        feature_group_cls: Set[Type[FeatureGroup]] | Type[FeatureGroup],
+    ) -> "PluginCollector":
         if not isinstance(feature_group_cls, Set):
             feature_group_cls = {feature_group_cls}
 
-        plugin_collector = PlugInCollector()
+        plugin_collector = PluginCollector()
         plugin_collector.add_enabled_feature_group_classes(feature_group_cls)
         return plugin_collector

mloda/core/abstract_plugins/components/validators/datatype_validator.py

@@ -0,0 +1,96 @@
+from typing import Any
+
+
+from mloda.core.abstract_plugins.components.data_types import DataType
+
+
+class DataTypeMismatchError(ValueError):
+    """Raised when feature data type doesn't match declared type."""
+
+    def __init__(self, feature_name: str, declared: DataType, actual: DataType) -> None:
+        self.feature_name = feature_name
+        self.declared = declared
+        self.actual = actual
+        super().__init__(
+            f"Feature '{feature_name}': declared {declared.name}, got {actual.name}, coercion not supported"
+        )
+
+
+class DataTypeValidator:
+    """Validates feature data matches declared DataType."""
+
+    _COMPATIBLE_TYPES = {
+        DataType.INT64: {DataType.INT32, DataType.INT64},
+        DataType.DOUBLE: {DataType.FLOAT, DataType.DOUBLE, DataType.INT32, DataType.INT64},
+        DataType.TIMESTAMP_MICROS: {DataType.TIMESTAMP_MILLIS, DataType.TIMESTAMP_MICROS},
+    }
+
+    @classmethod
+    def _types_compatible(cls, declared: DataType, actual: DataType) -> bool:
+        """Check if actual type is compatible with declared (allows widening)."""
+        if declared == actual:
+            return True
+        return actual in cls._COMPATIBLE_TYPES.get(declared, set())
+
+    @classmethod
+    def _types_loosely_compatible(cls, declared: DataType, actual: DataType) -> bool:
+        """Check if types are loosely compatible (allows any numeric/timestamp pairing).
+
+        Lenient mode allows data type mismatches within the same category:
+        - All numeric types (INT32, INT64, FLOAT, DOUBLE) are interchangeable
+        - All timestamp types are interchangeable
+        - Other types must match exactly
+
+        This fixes legacy FeatureGroups that declare INT32 but return DOUBLE.
+        """
+        if declared == actual:
+            return True
+
+        numeric_types = {DataType.INT32, DataType.INT64, DataType.FLOAT, DataType.DOUBLE}
+        if declared in numeric_types and actual in numeric_types:
+            return True
+
+        timestamp_types = {DataType.TIMESTAMP_MILLIS, DataType.TIMESTAMP_MICROS}
+        if declared in timestamp_types and actual in timestamp_types:
+            return True
+
+        return False
+
+    @classmethod
+    def validate(cls, data: Any, features: Any, strict_only: bool = False) -> None:
+        """Validate that data columns match declared feature types.
+
+        Args:
+            data: PyArrow table or similar with column data
+            features: FeatureSet containing features to validate
+            strict_only: If True, only validate when strict_type_enforcement is enabled.
+                        This maintains backward compatibility with existing code.
+        """
+        from mloda_plugins.feature_group.experimental.default_options_key import DefaultOptionKeys
+
+        for feature in features.features:
+            if feature.data_type is None:
+                continue
+
+            col_name = feature.get_name()
+            if col_name not in data.column_names:
+                continue
+
+            arrow_type = data.schema.field(col_name).type
+
+            try:
+                actual_type = DataType.from_arrow_type(arrow_type)
+            except ValueError:
+                continue
+
+            strict_mode = False
+            if feature.options:
+                strict_value = feature.options.get(DefaultOptionKeys.strict_type_enforcement)
+                strict_mode = strict_value if strict_value is not None else False
+
+            if strict_mode:
+                if not cls._types_compatible(feature.data_type, actual_type):
+                    raise DataTypeMismatchError(col_name, feature.data_type, actual_type)
+            else:
+                if not cls._types_loosely_compatible(feature.data_type, actual_type):
+                    raise DataTypeMismatchError(col_name, feature.data_type, actual_type)

mloda/core/abstract_plugins/components/validators/feature_set_validator.py

@@ -0,0 +1,38 @@
+from typing import Any, Optional, Set, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mloda.core.abstract_plugins.components.feature import Feature
+
+
+class FeatureSetValidator:
+    @staticmethod
+    def validate_options_initialized(options: Any, context: str = "FeatureSet") -> None:
+        if options is None:
+            raise ValueError(f"Options not initialized in {context}")
+
+    @staticmethod
+    def validate_equal_options(features: Set["Feature"]) -> None:
+        if len(features) <= 1:
+            return
+
+        options_list = [feature.options for feature in features]
+        first_options = options_list[0]
+
+        for options in options_list[1:]:
+            if options != first_options:
+                raise ValueError("Features have different options")
+
+    @staticmethod
+    def validate_feature_added(feature_name: Optional[str], context: str = "feature") -> None:
+        if feature_name is None:
+            raise ValueError(f"Feature name is None in {context}")
+
+    @staticmethod
+    def validate_filters_not_set(filters: Any) -> None:
+        if filters is not None:
+            raise ValueError("Filters already set")
+
+    @staticmethod
+    def validate_filters_is_set_type(filters: Any) -> None:
+        if not isinstance(filters, set):
+            raise ValueError("Filters must be a Set type")

mloda/core/abstract_plugins/components/validators/feature_validator.py

@@ -0,0 +1,23 @@
+from typing import Optional, Set, Type
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+
+
+class FeatureValidator:
+    @staticmethod
+    def validate_and_resolve_compute_framework(
+        framework_name: str, available_frameworks: Set[Type[ComputeFramework]], source: str = "parameter"
+    ) -> Type[ComputeFramework]:
+        for subclass in available_frameworks:
+            if framework_name == subclass.get_class_name():
+                return subclass
+        raise ValueError(f"Compute framework via {source} {framework_name} not found.")
+
+    @staticmethod
+    def validate_compute_frameworks_resolved(
+        compute_frameworks: Optional[Set[Type[ComputeFramework]]], feature_name: str
+    ) -> None:
+        if compute_frameworks is None:
+            raise ValueError(
+                f"Feature {feature_name} does not have any compute framework. "
+                "This function can only be called when the frameworks were resolved."
+            )

mloda/core/abstract_plugins/components/validators/link_validator.py

@@ -0,0 +1,79 @@
+from typing import TYPE_CHECKING, Any, Optional, Set, Tuple, Union
+
+if TYPE_CHECKING:
+    from mloda.core.abstract_plugins.components.link import Link
+
+
+class LinkValidator:
+    @staticmethod
+    def validate_index_not_empty(index: Union[str, Tuple[str, ...]], context: str = "index") -> None:
+        if not index:
+            raise ValueError(f"{context} cannot be empty")
+
+    @staticmethod
+    def validate_join_type(jointype: Any) -> None:
+        from mloda.core.abstract_plugins.components.link import JoinType
+
+        if not isinstance(jointype, JoinType):
+            raise ValueError(f"Join type {jointype} is not supported")
+
+    @staticmethod
+    def validate_no_double_joins(links: Set["Link"]) -> None:
+        from mloda.core.abstract_plugins.components.link import JoinType
+
+        for i_link in links:
+            for j_link in links:
+                if i_link == j_link:
+                    continue
+                if (
+                    i_link.left_feature_group == j_link.right_feature_group
+                    and i_link.right_feature_group == j_link.left_feature_group
+                    and i_link.jointype not in [JoinType.APPEND, JoinType.UNION]
+                ):
+                    raise ValueError(
+                        f"Link {i_link} and {j_link} have at least two different defined joins. Please remove one."
+                    )
+
+    @staticmethod
+    def validate_no_conflicting_join_types(links: Set["Link"]) -> None:
+        for i_link in links:
+            for j_link in links:
+                if i_link == j_link:
+                    continue
+                if (
+                    i_link.left_feature_group == j_link.left_feature_group
+                    and i_link.right_feature_group == j_link.right_feature_group
+                    and i_link.jointype != j_link.jointype
+                ):
+                    raise ValueError(
+                        f"Link {i_link} and {j_link} have different join types for the same feature groups. Please remove one."
+                    )
+
+    @staticmethod
+    def validate_right_join_constraints(links: Set["Link"]) -> None:
+        from mloda.core.abstract_plugins.components.link import JoinType
+
+        for i_link in links:
+            if i_link.jointype == JoinType.RIGHT:
+                for j_link in links:
+                    if i_link == j_link:
+                        continue
+                    if (
+                        i_link.left_feature_group == j_link.left_feature_group
+                        or i_link.left_feature_group == j_link.right_feature_group
+                    ):
+                        raise ValueError(
+                            f"Link {i_link} and {j_link} have multiple right joins for the same feature group on the left side or switching from left to right side although using right join. Please reconsider your joinlogic and if possible, use left joins instead of rightjoins. This will currently break the planner or during execution."
+                        )
+
+    @classmethod
+    def validate_links(cls, links: Optional[Set["Link"]]) -> None:
+        if links is None:
+            return
+
+        for link in links:
+            cls.validate_join_type(link.jointype)
+
+        cls.validate_no_double_joins(links)
+        cls.validate_no_conflicting_join_types(links)
+        cls.validate_right_join_constraints(links)

mloda/core/abstract_plugins/components/validators/options_validator.py

@@ -0,0 +1,57 @@
+from typing import Any, Dict, Set
+
+
+class OptionsValidator:
+    """Validates Options configuration consistency."""
+
+    @staticmethod
+    def validate_no_duplicate_keys(group: Dict[str, Any], context: Dict[str, Any]) -> None:
+        """
+        Ensure no key exists in both group and context.
+
+        Raises ValueError if any key exists in both, with duplicate keys in message.
+        """
+        duplicate_keys = set(group.keys()) & set(context.keys())
+        if duplicate_keys:
+            raise ValueError(f"Keys cannot exist in both group and context: {duplicate_keys}")
+
+    @staticmethod
+    def validate_can_add_to_group(key: str, value: Any, group: Dict[str, Any], context: Dict[str, Any]) -> None:
+        """
+        Validate that a key can be added to group.
+
+        Checks:
+        1. If key exists in group with different value -> ValueError (include key in message)
+        2. If key exists in context -> ValueError (include key in message)
+        """
+        if key in group:
+            if value != group[key]:
+                raise ValueError(f"Key {key} already exists in group options with a different value: {group[key]}")
+        if key in context:
+            raise ValueError(f"Key {key} already exists in context options. Cannot add to group.")
+
+    @staticmethod
+    def validate_can_add_to_context(key: str, value: Any, group: Dict[str, Any], context: Dict[str, Any]) -> None:
+        """
+        Validate that a key can be added to context.
+
+        Checks:
+        1. If key exists in context with different value -> ValueError (include key in message)
+        2. If key exists in group -> ValueError (include key in message)
+        """
+        if key in context:
+            if value != context[key]:
+                raise ValueError(f"Key {key} already exists in context options with a different value: {context[key]}")
+        if key in group:
+            raise ValueError(f"Key {key} already exists in group options. Cannot add to context.")
+
+    @staticmethod
+    def validate_no_group_context_conflicts(other_group_keys: Set[str], self_context_keys: Set[str]) -> None:
+        """
+        Validate no conflicts between other's group keys and self's context keys.
+
+        Raises ValueError if any key exists in both, with conflicting keys in message.
+        """
+        conflicting_keys = other_group_keys & self_context_keys
+        if conflicting_keys:
+            raise ValueError(f"Cannot update group: keys already exist in context: {conflicting_keys}")

mloda_core/abstract_plugins/compute_frame_work.py → mloda/core/abstract_plugins/compute_framework.py

@@ -1,23 +1,27 @@
 from abc import ABC
 from typing import Any, List, Optional, Set, Type, Union, final
 from uuid import UUID, uuid4
-from mloda_core.abstract_plugins.components.data_access_collection import DataAccessCollection
-from mloda_core.abstract_plugins.components.framework_transformer.cfw_transformer import (
+from mloda.core.abstract_plugins.components.data_access_collection import DataAccessCollection
+from mloda.core.abstract_plugins.components.framework_transformer.cfw_transformer import (
     ComputeFrameworkTransformer,
 )
-from mloda_core.abstract_plugins.components.merge.base_merge_engine import BaseMergeEngine
+from mloda.core.abstract_plugins.components.merge.base_merge_engine import BaseMergeEngine
 import pyarrow as pa
 
-from mloda_core.abstract_plugins.function_extender import WrapperFunctionExtender, WrapperFunctionEnum
-from mloda_core.abstract_plugins.components.feature_name import FeatureName
-from mloda_core.abstract_plugins.components.parallelization_modes import ParallelizationModes
-from mloda_core.filter.filter_engine import BaseFilterEngine
-from mloda_core.runtime.flight.flight_server import FlightServer
+from mloda.core.abstract_plugins.function_extender import (
+    Extender,
+    ExtenderHook,
+    _CompositeExtender,
+)
+from mloda.core.abstract_plugins.components.feature_name import FeatureName
+from mloda.core.abstract_plugins.components.parallelization_modes import ParallelizationMode
+from mloda.core.filter.filter_engine import BaseFilterEngine
+from mloda.core.runtime.flight.flight_server import FlightServer
 
 
-class ComputeFrameWork(ABC):
+class ComputeFramework(ABC):
     """
-    Documentation ComputeFrameWork:
+    Documentation ComputeFramework:
 
     This class is used to define the compute framework.
 
@@ -41,10 +45,10 @@ class ComputeFrameWork(ABC):
 
     def __init__(
         self,
-        mode: ParallelizationModes,
+        mode: ParallelizationMode,
         children_if_root: frozenset[UUID],
         uuid: UUID = uuid4(),
-        function_extender: Optional[Set[WrapperFunctionExtender]] = None,
+        function_extender: Optional[Set[Extender]] = None,
     ) -> None:
         """This class is initialized step execution."""
         self.mode = mode
@@ -64,18 +68,19 @@ class ComputeFrameWork(ABC):
         # connection object for frameworks that need persistent connections (e.g., DuckDB, Spark)
         self.framework_connection_object: Optional[Any] = None
 
-    @staticmethod
-    def expected_data_framework() -> Any:
+    @classmethod
+    def expected_data_framework(cls) -> Any:
         """
         This function should return the expected data framework for the compute framework.
         However, we only need to set it if we really want to be sure that the datatype is correct.
         """
         return None
 
-    def filter_engine(self) -> Type[BaseFilterEngine]:
+    @classmethod
+    def filter_engine(cls) -> Type[BaseFilterEngine]:
         """
         This function should return the filtered data.
-        The BaseFilterEngine should be overwritten by the appropriate ComputeFrameWork if needed
+        The BaseFilterEngine should be overwritten by the appropriate ComputeFramework if needed
         """
         raise NotImplementedError
 
@@ -122,16 +127,15 @@ class ComputeFrameWork(ABC):
         """
         return data
 
-    def merge_engine(self) -> Type[BaseMergeEngine]:
+    @classmethod
+    def merge_engine(cls) -> Type[BaseMergeEngine]:
         """
         This function should return a subclass of the BaseMergeEngine.
         With this, we can merge data from the same compute framework.
 
         This implementation is optional.
         """
-        raise NotImplementedError(
-            f"Merge functionality is for this compute framework not implemented {self.__class__.__name__}."
-        )
+        raise NotImplementedError(f"Merge functionality is for this compute framework not implemented {cls.__name__}.")
 
     def set_framework_connection_object(self, framework_connection_object: Optional[Any] = None) -> None:
         """
@@ -241,7 +245,7 @@ class ComputeFrameWork(ABC):
         if self.data is None:
             return
 
-        extender = self.get_function_extender(WrapperFunctionEnum.VALIDATE_INPUT_FEATURE)
+        extender = self.get_function_extender(ExtenderHook.VALIDATE_INPUT_FEATURE)
         if extender is None:
             result = feature_group.validate_input_features(self.data, features)
         else:
@@ -255,7 +259,12 @@ class ComputeFrameWork(ABC):
     def run_validate_output_features(self, feature_group: Any, features: Any) -> Any:
         if self.data is None:
             return
-        extender = self.get_function_extender(WrapperFunctionEnum.VALIDATE_OUTPUT_FEATURE)
+
+        from mloda.core.abstract_plugins.components.validators.datatype_validator import DataTypeValidator
+
+        DataTypeValidator.validate(self.data, features, strict_only=True)
+
+        extender = self.get_function_extender(ExtenderHook.VALIDATE_OUTPUT_FEATURE)
         if extender is None:
             result = feature_group.validate_output_features(self.data, features)
         else:
@@ -276,7 +285,7 @@ class ComputeFrameWork(ABC):
 
     @final
     def __eq__(self, other: object) -> bool:
-        if not isinstance(other, ComputeFrameWork):
+        if not isinstance(other, ComputeFramework):
             return False
         return self.get_class_name() == other.get_class_name() and self.children_if_root == other.children_if_root
 
@@ -323,23 +332,23 @@ class ComputeFrameWork(ABC):
         return False
 
     @final
-    def get_function_extender(self, wrapper_function_enum: WrapperFunctionEnum) -> Optional[WrapperFunctionExtender]:
-        found_extender = None
+    def get_function_extender(self, wrapper_function_enum: ExtenderHook) -> Optional[Extender]:
+        matching_extenders = []
         for extender in self.function_extender:
             if wrapper_function_enum in extender.wraps():
-                if found_extender is not None:
-                    raise ValueError(
-                        f"Multiple function_extender found for {wrapper_function_enum}, {found_extender.__class__.__name__}, {extender.__class__.__name__}"
-                    )
-                found_extender = extender
-
-        if found_extender is not None:
-            return found_extender
-        return None
+                matching_extenders.append(extender)
+
+        if len(matching_extenders) == 0:
+            return None
+        if len(matching_extenders) == 1:
+            return matching_extenders[0]
+
+        sorted_extenders = sorted(matching_extenders, key=lambda e: e.priority)
+        return _CompositeExtender(sorted_extenders, wrapper_function_enum)
 
     @final
     def run_calculate_feature(self, feature_group: Any, features: Any) -> Any:
-        extender = self.get_function_extender(WrapperFunctionEnum.FEATURE_GROUP_CALCULATE_FEATURE)
+        extender = self.get_function_extender(ExtenderHook.FEATURE_GROUP_CALCULATE_FEATURE)
 
         try:
             if extender is None:
@@ -365,8 +374,8 @@ When a feature depends on multiple input features, you must provide explicit Lin
 how to merge them. Without Links, the framework cannot determine how to combine the data.
 
 Example:
-    from mloda_core.abstract_plugins.components.link import Link
-    from mloda_core.abstract_plugins.components.index.index import Index
+    from mloda.core.abstract_plugins.components.link import Link
+    from mloda.core.abstract_plugins.components.index.index import Index
 
     links = {{
         Link.inner(