mloda 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

mloda_core/abstract_plugins/abstract_feature_group.py → mloda/core/abstract_plugins/feature_group.py

@@ -3,26 +3,26 @@ from __future__ import annotations
 from typing import Any, Callable, Dict, List, Optional, Set, Type, Union, final
 from abc import ABC
 
-from mloda_core.abstract_plugins.components.base_artifact import BaseArtifact
-from mloda_core.abstract_plugins.components.data_access_collection import DataAccessCollection
-from mloda_core.abstract_plugins.components.data_types import DataType
-
-from mloda_core.abstract_plugins.components.domain import Domain
-from mloda_core.abstract_plugins.components.feature_group_version import FeatureGroupVersion
-from mloda_core.abstract_plugins.components.feature_name import FeatureName
-from mloda_core.abstract_plugins.components.input_data.api.api_input_data import ApiInputData
-from mloda_core.abstract_plugins.components.input_data.base_input_data import BaseInputData
-from mloda_core.abstract_plugins.components.input_data.creator.data_creator import DataCreator
-from mloda_core.abstract_plugins.components.match_data.match_data import MatchData
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.abstract_plugins.components.feature import Feature
-from mloda_core.abstract_plugins.components.feature_set import FeatureSet
-from mloda_core.abstract_plugins.components.options import Options
-from mloda_core.abstract_plugins.components.index.index import Index
-from mloda_core.abstract_plugins.components.utils import get_all_subclasses
-
-
-class AbstractFeatureGroup(ABC):
+from mloda.core.abstract_plugins.components.base_artifact import BaseArtifact
+from mloda.core.abstract_plugins.components.data_access_collection import DataAccessCollection
+from mloda.core.abstract_plugins.components.data_types import DataType
+
+from mloda.core.abstract_plugins.components.domain import Domain
+from mloda.core.abstract_plugins.components.feature_group_version import FeatureGroupVersion
+from mloda.core.abstract_plugins.components.feature_name import FeatureName
+from mloda.core.abstract_plugins.components.input_data.api.api_input_data import ApiInputData
+from mloda.core.abstract_plugins.components.input_data.base_input_data import BaseInputData
+from mloda.core.abstract_plugins.components.input_data.creator.data_creator import DataCreator
+from mloda.core.abstract_plugins.components.match_data.match_data import MatchData
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.components.feature import Feature
+from mloda.core.abstract_plugins.components.feature_set import FeatureSet
+from mloda.core.abstract_plugins.components.options import Options
+from mloda.core.abstract_plugins.components.index.index import Index
+from mloda.core.abstract_plugins.components.utils import get_all_subclasses
+
+
+class FeatureGroup(ABC):
     """
     Mostly implement:
     input_features, except it is a primary source
@@ -47,7 +47,7 @@ class AbstractFeatureGroup(ABC):
         the base class's docstring. Otherwise, it falls back to the class name.
         This behavior allows subclasses to easily customize their description.
         """
-        base_doc = (AbstractFeatureGroup.__doc__ or "").strip()
+        base_doc = (FeatureGroup.__doc__ or "").strip()
         current_doc = (cls.__doc__ or "").strip()
 
         if current_doc and current_doc != base_doc:
@@ -82,14 +82,30 @@ class AbstractFeatureGroup(ABC):
     @classmethod
     def validate_input_features(cls, data: Any, features: FeatureSet) -> Optional[bool]:
         """
-        This function should be used to validate the input data.
+        Validate the input data before feature calculation.
+
+        Override this method to implement custom input validation logic.
+        The base implementation returns None (no validation).
+
+        Returns:
+            None: No validation needed/not implemented (neutral - validation passes by default)
+            True: Validation explicitly passed
+            False: Validation failed
         """
         return None
 
     @classmethod
     def validate_output_features(cls, data: Any, features: FeatureSet) -> Optional[bool]:
         """
-        This function should be used to validate the output data.
+        Validate the output data after feature calculation.
+
+        Override this method to implement custom output validation logic.
+        The base implementation returns None (no validation).
+
+        Returns:
+            None: No validation needed/not implemented (neutral - validation passes by default)
+            True: Validation explicitly passed
+            False: Validation failed
         """
         return None
 
@@ -238,11 +254,18 @@ class AbstractFeatureGroup(ABC):
     @classmethod
     def supports_index(cls, index: Index) -> Optional[bool]:
         """
-        Indicates whether this feature group supports the given index.
+        Check if this feature group supports the given index.
+
+        This method checks the index against the feature group's supported index columns.
+        If no index columns are defined, any index is accepted.
+
+        Args:
+            index: The index to check for support.
 
-        This method should return True if the feature group supports the given index,
-        False otherwise. If the feature group does not have any specific index
-        requirements, it should return None.
+        Returns:
+            None: No index constraint defined (accepts any index)
+            True: Index is supported
+            False: Index is not supported
         """
         supported_index_columns = cls.index_columns()
 
@@ -355,7 +378,7 @@ class AbstractFeatureGroup(ABC):
         return Domain.get_default_domain()
 
     @classmethod
-    def compute_framework_rule(cls) -> Union[bool, Set[Type[ComputeFrameWork]]]:
+    def compute_framework_rule(cls) -> Union[bool, Set[Type[ComputeFramework]]]:
         """
         Defines the rule for determining the compute framework to use for this feature group.
 
@@ -366,7 +389,7 @@ class AbstractFeatureGroup(ABC):
 
     @final
     @classmethod
-    def compute_framework_definition(cls) -> Set[Type[ComputeFrameWork]]:
+    def compute_framework_definition(cls) -> Set[Type[ComputeFramework]]:
         """
         Determines the set of compute frameworks supported by this feature group based on the
         `compute_framework_rule`.
@@ -376,7 +399,7 @@ class AbstractFeatureGroup(ABC):
 
         """If FG creator does not care, we allow every framework."""
         if rule is True:
-            return get_all_subclasses(ComputeFrameWork)
+            return get_all_subclasses(ComputeFramework)
         if isinstance(rule, bool):
             raise Exception("Compute framework rule for is not a set of compute frameworks.")
         return rule
@@ -393,9 +416,9 @@ class AbstractFeatureGroup(ABC):
         """
         Checks if this feature group is equal to another object.
         """
-        if isinstance(another, AbstractFeatureGroup):
+        if isinstance(another, FeatureGroup):
             return self.get_class_name() == another.get_class_name()
-        raise Exception(f"Cannot compare AbstractFeatureGroup with another type. {another} ")
+        raise Exception(f"Cannot compare FeatureGroup with another type. {another} ")
 
     def __hash__(self) -> int:
         """
@@ -463,7 +486,7 @@ class AbstractFeatureGroup(ABC):
 
         To be used, create a class like this:
 
-        class MyMatchData(AbstractFeatureGroup, MatchData):
+        class MyMatchData(FeatureGroup, MatchData):
             ...
 
         and then create the function match_data_access.

mloda/core/abstract_plugins/function_extender.py

@@ -0,0 +1,78 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any, List, Optional, Set
+import logging
+
+
+class ExtenderHook(Enum):
+    FEATURE_GROUP_CALCULATE_FEATURE = "feature_group_calculate_feature"
+    VALIDATE_INPUT_FEATURE = "validate_input_feature"
+    VALIDATE_OUTPUT_FEATURE = "validate_output_feature"
+
+
+class Extender(ABC):
+    """
+    - Automated Metadata harvestor connector
+    - Messaging Integration ( email )
+    - Automation Tools
+    - data lineage mapping
+    - Impact Analysis
+    - Audit Trail
+    - Monitoring alerts
+    - metadata capture
+    - Event logging
+    - metrics on feature calculation
+    - visibility / observibility
+    - Performance
+    """
+
+    @property
+    def priority(self) -> int:
+        """Lower priority runs first. Default is 100."""
+        if hasattr(self, "_priority"):
+            return self._priority
+        return 100
+
+    @priority.setter
+    def priority(self, value: int) -> None:
+        self._priority = value
+
+    @abstractmethod
+    def wraps(self) -> Set[ExtenderHook]:
+        pass
+
+    @abstractmethod
+    def __call__(self, func: Any, *args: Any, **kwargs: Any) -> Any:
+        pass
+
+
+class _CompositeExtender(Extender):
+    """Internal class that chains multiple Extenders in priority order."""
+
+    def __init__(self, extenders: List[Extender], function_type: Optional[ExtenderHook] = None):
+        self.extenders = sorted(extenders, key=lambda e: e.priority)
+        self.function_type = function_type
+
+    def wraps(self) -> Set[ExtenderHook]:
+        if self.function_type:
+            return {self.function_type}
+        result = set()
+        for extender in self.extenders:
+            result.update(extender.wraps())
+        return result
+
+    def __call__(self, func: Any, *args: Any, **kwargs: Any) -> Any:
+        def make_wrapper(ext: Extender, inner_func: Any) -> Any:
+            def wrapper(*a: Any, **kw: Any) -> Any:
+                try:
+                    return ext.__call__(inner_func, *a, **kw)
+                except Exception as e:
+                    logging.error(f"{ext.__class__.__name__} {ext.name if hasattr(ext, 'name') else ''} {str(e)}")
+                    return inner_func(*a, **kw)
+
+            return wrapper
+
+        wrapped_func = func
+        for extender in reversed(self.extenders):
+            wrapped_func = make_wrapper(extender, wrapped_func)
+        return wrapped_func(*args, **kwargs)

mloda/core/api/plugin_docs.py

@@ -0,0 +1,220 @@
+"""
+Plugin documentation discovery functions.
+
+These functions return documentation and metadata for currently loaded plugins.
+They report the current state - ensure plugins are loaded before calling.
+
+Example:
+    from mloda.core.abstract_plugins.plugin_loader.plugin_loader import PluginLoader
+    from mloda.core.api.plugin_docs import get_feature_group_docs
+
+    # Load plugins first
+    PluginLoader.all()
+
+    # Then get documentation
+    docs = get_feature_group_docs()
+"""
+
+from typing import List, Optional, Type, Union
+
+from mloda.core.abstract_plugins.feature_group import FeatureGroup
+from mloda.core.abstract_plugins.components.plugin_option.plugin_collector import PluginCollector
+from mloda.core.abstract_plugins.components.utils import get_all_subclasses
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.function_extender import Extender
+from mloda.core.api.plugin_info import ComputeFrameworkInfo, ExtenderInfo, FeatureGroupInfo
+
+
+def get_feature_group_docs(
+    name: Optional[str] = None,
+    search: Optional[str] = None,
+    compute_framework: Optional[Union[str, Type[ComputeFramework]]] = None,
+    version_contains: Optional[str] = None,
+    plugin_collector: Optional[PluginCollector] = None,
+) -> List[FeatureGroupInfo]:
+    """
+    Get documentation for feature groups with optional filtering.
+
+    Returns the current state of loaded feature groups. Ensure plugins are loaded
+    before calling this function (e.g., via PluginLoader.all()).
+
+    Args:
+        name: Filter by feature group name (case-insensitive partial match).
+        search: Search in feature group description (case-insensitive partial match).
+        compute_framework: Filter by compute framework name or class.
+        version_contains: Filter by version substring.
+        plugin_collector: Filter using plugin collector's applicability check.
+
+    Returns:
+        List of FeatureGroupInfo objects sorted by name.
+    """
+    all_feature_groups = get_all_subclasses(FeatureGroup)
+    results = []
+
+    for fg_class in all_feature_groups:
+        fg_name = fg_class.get_class_name()
+        description = fg_class.description()
+        version = fg_class.version()
+        module = fg_class.__module__
+        compute_frameworks = [cfw.__name__ for cfw in fg_class.compute_framework_definition()]
+        supported_feature_names = fg_class.feature_names_supported()
+        prefix = fg_class.prefix()
+
+        if name is not None and name.lower() not in fg_name.lower():
+            continue
+
+        if search is not None and search.lower() not in description.lower():
+            continue
+
+        if compute_framework is not None:
+            cfw_name = compute_framework if isinstance(compute_framework, str) else compute_framework.__name__
+            if cfw_name not in compute_frameworks:
+                continue
+
+        if version_contains is not None and version_contains not in version:
+            continue
+
+        if plugin_collector is not None and not plugin_collector.applicable_feature_group_class(fg_class):
+            continue
+
+        results.append(
+            FeatureGroupInfo(
+                name=fg_name,
+                description=description,
+                version=version,
+                module=module,
+                compute_frameworks=compute_frameworks,
+                supported_feature_names=supported_feature_names,
+                prefix=prefix,
+            )
+        )
+
+    return sorted(results, key=lambda x: x.name)
+
+
+def get_compute_framework_docs(
+    name: Optional[str] = None,
+    search: Optional[str] = None,
+    available_only: bool = True,
+) -> List[ComputeFrameworkInfo]:
+    """
+    Get documentation for compute frameworks with optional filtering.
+
+    Returns the current state of loaded compute frameworks. Ensure plugins are loaded
+    before calling this function (e.g., via PluginLoader.all()).
+
+    Args:
+        name: Filter by compute framework name (case-insensitive partial match).
+        search: Search in compute framework description (case-insensitive partial match).
+        available_only: If True, only return available frameworks (default True).
+
+    Returns:
+        List of ComputeFrameworkInfo objects sorted by name.
+    """
+    all_compute_frameworks = get_all_subclasses(ComputeFramework)
+    results = []
+
+    for cfw_class in all_compute_frameworks:
+        cfw_name = cfw_class.__name__
+        description = (cfw_class.__doc__ or "").strip() or cfw_class.__name__
+        module = cfw_class.__module__
+
+        is_available = cfw_class.is_available()
+
+        try:
+            expected_data_framework = str(cfw_class.expected_data_framework())
+        except Exception:  # nosec
+            expected_data_framework = "unavailable"
+
+        try:
+            has_merge_engine = cfw_class.merge_engine() is not None
+        except Exception:  # nosec
+            has_merge_engine = False
+
+        try:
+            has_filter_engine = cfw_class.filter_engine() is not None
+        except Exception:  # nosec
+            has_filter_engine = False
+
+        if available_only and not is_available:
+            continue
+
+        if name is not None and name.lower() not in cfw_name.lower():
+            continue
+
+        if search is not None and search.lower() not in description.lower():
+            continue
+
+        results.append(
+            ComputeFrameworkInfo(
+                name=cfw_name,
+                description=description,
+                module=module,
+                is_available=is_available,
+                expected_data_framework=expected_data_framework,
+                has_merge_engine=has_merge_engine,
+                has_filter_engine=has_filter_engine,
+            )
+        )
+
+    return sorted(results, key=lambda x: x.name)
+
+
+def get_extender_docs(
+    name: Optional[str] = None,
+    search: Optional[str] = None,
+    wraps: Optional[str] = None,
+) -> List[ExtenderInfo]:
+    """
+    Get documentation for extenders with optional filtering.
+
+    Returns the current state of loaded extenders. Ensure plugins are loaded
+    before calling this function (e.g., via PluginLoader.all()).
+
+    Args:
+        name: Filter by extender name (case-insensitive partial match).
+        search: Search in extender description (case-insensitive partial match).
+        wraps: Filter by wrapped function type (case-insensitive exact match).
+
+    Returns:
+        List of ExtenderInfo objects sorted by name.
+    """
+    all_extenders = get_all_subclasses(Extender)
+    results = []
+
+    for ext_class in all_extenders:
+        ext_name = ext_class.__name__
+        description = (ext_class.__doc__ or "").strip() or ext_class.__name__
+        module = ext_class.__module__
+
+        if ext_name in ("Extender", "_CompositeExtender"):
+            continue
+
+        wraps_list: List[str] = []
+        try:
+            instance = ext_class()
+            wraps_list = [w.value for w in instance.wraps()]
+        except Exception:  # nosec
+            pass
+
+        if name is not None and name.lower() not in ext_name.lower():
+            continue
+
+        if search is not None and search.lower() not in description.lower():
+            continue
+
+        if wraps is not None:
+            wraps_lower = wraps.lower()
+            if not any(wraps_lower == w.lower() for w in wraps_list):
+                continue
+
+        results.append(
+            ExtenderInfo(
+                name=ext_name,
+                description=description,
+                module=module,
+                wraps=wraps_list,
+            )
+        )
+
+    return sorted(results, key=lambda x: x.name)

mloda/core/api/plugin_info.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+from typing import List, Set
+
+
+@dataclass
+class FeatureGroupInfo:
+    name: str
+    description: str
+    version: str
+    module: str
+    compute_frameworks: List[str]
+    supported_feature_names: Set[str]
+    prefix: str
+
+
+@dataclass
+class ComputeFrameworkInfo:
+    name: str
+    description: str
+    module: str
+    is_available: bool
+    expected_data_framework: str
+    has_merge_engine: bool
+    has_filter_engine: bool
+
+
+@dataclass
+class ExtenderInfo:
+    name: str
+    description: str
+    module: str
+    wraps: List[str]

{mloda_core → mloda/core}/api/prepare/setup_compute_framework.py

@@ -1,22 +1,22 @@
 from typing import Optional, Set, Type, Union, cast
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.abstract_plugins.components.feature_collection import Features
-from mloda_core.abstract_plugins.components.utils import get_all_subclasses
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.components.feature_collection import Features
+from mloda.core.abstract_plugins.components.utils import get_all_subclasses
 
 
 class SetupComputeFramework:
     """A class to create the compute framework and do basic validation."""
 
     def __init__(
-        self, user_compute_frameworks: Union[Set[Type[ComputeFrameWork]], Optional[list[str]]], features: Features
+        self, user_compute_frameworks: Union[Set[Type[ComputeFramework]], Optional[list[str]]], features: Features
     ) -> None:
-        available_compute_frameworks = get_all_subclasses(ComputeFrameWork)
+        available_compute_frameworks = get_all_subclasses(ComputeFramework)
 
         if user_compute_frameworks:
             if isinstance(user_compute_frameworks, list):
-                user_set_compute_frameworks: set[str | Type[ComputeFrameWork]] = set(user_compute_frameworks)
+                user_set_compute_frameworks: set[str | Type[ComputeFramework]] = set(user_compute_frameworks)
             else:
-                user_set_compute_frameworks = cast(set[str | Type[ComputeFrameWork]], user_compute_frameworks)
+                user_set_compute_frameworks = cast(set[str | Type[ComputeFramework]], user_compute_frameworks)
 
             available_compute_frameworks = self.filter_user_set_in_available_sub_classes(
                 user_set_compute_frameworks, available_compute_frameworks
@@ -29,7 +29,7 @@ class SetupComputeFramework:
         self.compute_frameworks = available_compute_frameworks
 
     def validate_if_at_least_one_feature_compute_framework_is_in_available_compute_framework(
-        self, features: Features, available_compute_frameworks: set[type[ComputeFrameWork]]
+        self, features: Features, available_compute_frameworks: set[type[ComputeFramework]]
     ) -> None:
         for feature in features.collection:
             if feature.compute_frameworks and not any(
@@ -41,9 +41,9 @@ class SetupComputeFramework:
 
     def filter_user_set_in_available_sub_classes(
         self,
-        api_request_compute_frameworks: set[str | Type[ComputeFrameWork]],
-        sub_classes: set[type[ComputeFrameWork]],
-    ) -> set[type[ComputeFrameWork]]:
+        api_request_compute_frameworks: set[str | Type[ComputeFramework]],
+        sub_classes: set[type[ComputeFramework]],
+    ) -> set[type[ComputeFramework]]:
         compute_frameworks = set()
         compute_frameworks = {
             sub