mloda 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

{mloda_core → mloda/core}/prepare/identify_feature_group.py

@@ -1,11 +1,11 @@
 from typing import Optional, Set, Tuple, Type
 
-from mloda_core.prepare.accessible_plugins import FeatureGroupEnvironmentMapping
-from mloda_core.abstract_plugins.components.data_access_collection import DataAccessCollection
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.abstract_plugins.abstract_feature_group import AbstractFeatureGroup
-from mloda_core.abstract_plugins.components.feature import Feature
-from mloda_core.abstract_plugins.components.link import Link
+from mloda.core.prepare.accessible_plugins import FeatureGroupEnvironmentMapping
+from mloda.core.abstract_plugins.components.data_access_collection import DataAccessCollection
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.feature_group import FeatureGroup
+from mloda.core.abstract_plugins.components.feature import Feature
+from mloda.core.abstract_plugins.components.link import Link
 
 import logging
 
@@ -53,9 +53,7 @@ class IdentifyFeatureGroupClass:
         _identified_feature_groups = self.filter_subclasses(_identified_feature_groups)
         return _identified_feature_groups
 
-    def _filter_feature_group_by_links(
-        self, feature_group: Type[AbstractFeatureGroup], links: Optional[Set[Link]]
-    ) -> bool:
+    def _filter_feature_group_by_links(self, feature_group: Type[FeatureGroup], links: Optional[Set[Link]]) -> bool:
         # Case index columns not given, so no validation possible
         if feature_group.index_columns() is None:
             return True
@@ -76,18 +74,18 @@ class IdentifyFeatureGroupClass:
 
     def _filter_feature_group_by_criteria(
         self,
-        feature_group: Type[AbstractFeatureGroup],
+        feature_group: Type[FeatureGroup],
         feature: Feature,
         data_access_collection: Optional[DataAccessCollection],
     ) -> bool:
         return feature_group.match_feature_group_criteria(feature.name, feature.options, data_access_collection)
 
-    def _filter_feature_group_by_domain(self, feature_group: Type[AbstractFeatureGroup], feature: Feature) -> bool:
+    def _filter_feature_group_by_domain(self, feature_group: Type[FeatureGroup], feature: Feature) -> bool:
         return not feature.domain or feature_group.get_domain() == feature.domain
 
     def _filter_feature_group_by_framework(
         self,
-        compute_frameworks: Set[Type[ComputeFrameWork]],
+        compute_frameworks: Set[Type[ComputeFramework]],
         feature: Feature,
     ) -> bool:
         if feature.compute_frameworks is None:
@@ -114,7 +112,7 @@ class IdentifyFeatureGroupClass:
         if not compute_frameworks:
             raise ValueError(f"Feature {feature.name} {feature_group_class.get_class_name()} has no compute framework.")
 
-    def get(self) -> Tuple[Type[AbstractFeatureGroup], Set[Type[ComputeFrameWork]]]:
+    def get(self) -> Tuple[Type[FeatureGroup], Set[Type[ComputeFramework]]]:
         return next(iter(self.feature_group_compute_framework_mapping.items()))
 
     def filter_subclasses(
@@ -123,7 +121,7 @@ class IdentifyFeatureGroupClass:
         """
         This functionality ensures that only subclass feature groups are kept.
         """
-        fgs_to_pop: Set[Type[AbstractFeatureGroup]] = set()
+        fgs_to_pop: Set[Type[FeatureGroup]] = set()
 
         for i_feature_group, i_compute_frameworks in _identified_feature_groups.items():
             for o_feature_group, o_compute_frameworks in _identified_feature_groups.items():

{mloda_core → mloda/core}/prepare/joinstep_collection.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
 from typing import Dict, Set, Type
 from uuid import UUID
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.core.step.join_step import JoinStep
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.core.step.join_step import JoinStep
 
 
 class JoinStepCollection:
@@ -10,7 +10,7 @@ class JoinStepCollection:
         self.collection: Dict[JoinStep, Set[UUID]] = defaultdict(set)
 
     def similar_dependent_joins_uuids(
-        self, left_framework: Type[ComputeFrameWork], right_framework: Type[ComputeFrameWork]
+        self, left_framework: Type[ComputeFramework], right_framework: Type[ComputeFramework]
     ) -> Set[UUID]:
         """
         This functionality makes sure that we do not write on the same datasets due to overlapping joins at once.

{mloda_core → mloda/core}/prepare/resolve_compute_frameworks.py

@@ -2,10 +2,10 @@ from copy import deepcopy
 from typing import Any, Dict, List, Set, Type
 from collections import defaultdict
 from uuid import UUID
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.prepare.graph.graph import Graph
-from mloda_core.prepare.resolve_links import LinkFrameworkTrekker, LinkTrekker
-from mloda_core.abstract_plugins.components.link import JoinType, Link
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.prepare.graph.graph import Graph
+from mloda.core.prepare.resolve_links import LinkFrameworkTrekker, LinkTrekker
+from mloda.core.abstract_plugins.components.link import JoinType, Link
 
 
 class ResolveComputeFrameworks:
@@ -108,8 +108,8 @@ class ResolveComputeFrameworks:
         self.to_invert_trekker_collection = []
 
     def resolve_trekked_links(
-        self, trekked_links: List[LinkFrameworkTrekker], compute_frameworks: Set[Type[ComputeFrameWork]]
-    ) -> Set[Type[ComputeFrameWork]]:
+        self, trekked_links: List[LinkFrameworkTrekker], compute_frameworks: Set[Type[ComputeFramework]]
+    ) -> Set[Type[ComputeFramework]]:
         new_cfws = set()
 
         for link, left_cfw, right_cfw in trekked_links:

{mloda_core → mloda/core}/prepare/resolve_graph.py

@@ -1,23 +1,23 @@
 from collections import defaultdict
 from typing import Dict, List, Optional, Set, Tuple, Type, Union
 from uuid import UUID
-from mloda_core.prepare.graph.graph import Graph
-from mloda_core.prepare.resolve_compute_frameworks import ResolveComputeFrameworks
-from mloda_core.prepare.resolve_links import LinkFrameworkTrekker, ResolveLinks
-from mloda_core.abstract_plugins.abstract_feature_group import AbstractFeatureGroup
-from mloda_core.abstract_plugins.components.feature import Feature
-from mloda_core.abstract_plugins.components.link import Link
+from mloda.core.prepare.graph.graph import Graph
+from mloda.core.prepare.resolve_compute_frameworks import ResolveComputeFrameworks
+from mloda.core.prepare.resolve_links import LinkFrameworkTrekker, ResolveLinks
+from mloda.core.abstract_plugins.feature_group import FeatureGroup
+from mloda.core.abstract_plugins.components.feature import Feature
+from mloda.core.abstract_plugins.components.link import Link
 
 
-LinkFeatureQueue = List[Union[LinkFrameworkTrekker, Tuple[Feature, Type[AbstractFeatureGroup]]]]
+LinkFeatureQueue = List[Union[LinkFrameworkTrekker, Tuple[Feature, Type[FeatureGroup]]]]
 
-PlannedQueue = List[Union[LinkFrameworkTrekker, Tuple[Type[AbstractFeatureGroup], Set[Feature]]]]
+PlannedQueue = List[Union[LinkFrameworkTrekker, Tuple[Type[FeatureGroup], Set[Feature]]]]
 
 
 class ResolveGraph:
     def __init__(self, graph: Graph, links: Optional[Set[Link]] = None):
         self.graph = graph
-        self.nodes_per_feature_group: Dict[Type[AbstractFeatureGroup], Set[Feature]] = {}
+        self.nodes_per_feature_group: Dict[Type[FeatureGroup], Set[Feature]] = {}
         self.resolver_compute_framework = ResolveComputeFrameworks(self.graph)
         self.resolver_links = ResolveLinks(self.graph, links)
 
@@ -73,8 +73,8 @@ class ResolveGraph:
 
         return feature_link_queue
 
-    def get_nodes_with_same_feature_group_class(self) -> Dict[Type[AbstractFeatureGroup], Set[Feature]]:
-        collection: Dict[Type[AbstractFeatureGroup], Set[Feature]] = defaultdict(set)
+    def get_nodes_with_same_feature_group_class(self) -> Dict[Type[FeatureGroup], Set[Feature]]:
+        collection: Dict[Type[FeatureGroup], Set[Feature]] = defaultdict(set)
 
         for node in self.graph.queue:
             node_properties = self.graph.get_nodes()[node]

{mloda_core → mloda/core}/prepare/resolve_links.py

@@ -2,12 +2,13 @@ from collections import OrderedDict, defaultdict
 from typing import Dict, List, Optional, Set, Tuple, Type, Union
 from uuid import UUID
 
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.prepare.graph.graph import Graph
-from mloda_core.abstract_plugins.components.link import Link
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.prepare.graph.graph import Graph
+from mloda.core.abstract_plugins.components.link import Link
+from mloda.core.prepare.validators.resolve_link_validator import ResolveLinkValidator
 
 
-LinkFrameworkTrekker = Tuple[Link, Type[ComputeFrameWork], Type[ComputeFrameWork]]
+LinkFrameworkTrekker = Tuple[Link, Type[ComputeFramework], Type[ComputeFramework]]
 
 
 class LinkTrekker:
@@ -23,7 +24,7 @@ class LinkTrekker:
         self.data[key].add(value)
 
     def invert_link(
-        self, link: Link, left_cfw: Type[ComputeFrameWork], right_cfw: Type[ComputeFrameWork], uuid: UUID
+        self, link: Link, left_cfw: Type[ComputeFramework], right_cfw: Type[ComputeFramework], uuid: UUID
     ) -> None:
         """
         The purpose of this function is to invert left and right of an index during a run.
@@ -45,7 +46,7 @@ class LinkTrekker:
             del self.data[(link, left_cfw, right_cfw)]
             del self.data_ordered[(link, left_cfw, right_cfw)]
 
-    def get_position(self, link: Link, left_cfw: Type[ComputeFrameWork], right_cfw: Type[ComputeFrameWork]) -> int:
+    def get_position(self, link: Link, left_cfw: Type[ComputeFramework], right_cfw: Type[ComputeFramework]) -> int:
         for i, (k, _) in enumerate(self.data_ordered.items()):
             if k == (link, left_cfw, right_cfw):
                 return i
@@ -203,8 +204,7 @@ class LinkTrekker:
             if k not in self.data_ordered:
                 self.data_ordered[k] = v
 
-        if len(self.data.items()) != len(self.data_ordered.items()):
-            raise ValueError("Data ordered is not the same length as data!")
+        ResolveLinkValidator.validate_data_consistency(self.data, self.data_ordered)
 
 
 class ResolveLinks:
@@ -241,27 +241,7 @@ class ResolveLinks:
         self.graph.set_root_parents_by_direct_()
 
         self.go_through_each_child_and_its_parents_and_look_for_links()
-        self.validate_link_trekker()
-
-    def validate_link_trekker(self) -> None:
-        for link_fw_trekker, _ in self.link_trekker.data.items():
-            for other_fw_trekker, _ in self.link_trekker.data.items():
-                if link_fw_trekker == other_fw_trekker:
-                    continue
-
-                link, _, _ = link_fw_trekker
-                other, _, _ = other_fw_trekker
-
-                # if feature group match
-                if (
-                    link.left_feature_group.get_class_name() == other.left_feature_group.get_class_name()
-                    and link.right_feature_group.get_class_name() == other.right_feature_group.get_class_name()
-                ):
-                    # case join different
-                    if link.jointype != other.jointype:
-                        raise Exception(
-                            f"Link {link} and {other} have the same feature groups, but different join types!"
-                        )
+        ResolveLinkValidator.validate_no_conflicting_join_types(self.link_trekker.data)
 
     def get_link_trekker(self) -> LinkTrekker:
         return self.link_trekker
@@ -374,8 +354,8 @@ class ResolveLinks:
     def create_link_trekker_key(
         self,
         link: Link,
-        left_frameworks: Optional[Set[Type[ComputeFrameWork]]] = None,
-        right_frameworks: Optional[Set[Type[ComputeFrameWork]]] = None,
+        left_frameworks: Optional[Set[Type[ComputeFramework]]] = None,
+        right_frameworks: Optional[Set[Type[ComputeFramework]]] = None,
     ) -> LinkFrameworkTrekker:
         if left_frameworks is None or right_frameworks is None:
             raise ValueError("Left or right frameworks are not set!")

mloda/core/prepare/validators/resolve_link_validator.py

@@ -0,0 +1,32 @@
+from collections import OrderedDict
+from typing import Any, Dict, Set
+from uuid import UUID
+
+
+class ResolveLinkValidator:
+    @staticmethod
+    def validate_data_consistency(
+        data: Dict[Any, Set[UUID]],
+        data_ordered: "OrderedDict[Any, Set[UUID]]",
+    ) -> None:
+        if len(data.items()) != len(data_ordered.items()):
+            raise ValueError("Data and data_ordered have different lengths")
+
+    @staticmethod
+    def validate_no_conflicting_join_types(data: Dict[Any, Set[UUID]]) -> None:
+        seen_pairs: Dict[Any, Any] = {}
+        for key in data.keys():
+            link, _, _ = key
+            left_fg = link.left_feature_group
+            right_fg = link.right_feature_group
+            jointype = link.jointype
+
+            pair_key = (left_fg, right_fg)
+
+            if pair_key in seen_pairs:
+                if seen_pairs[pair_key] != jointype:
+                    raise Exception(
+                        f"Conflicting join types for {left_fg.get_class_name()} and {right_fg.get_class_name()}"
+                    )
+            else:
+                seen_pairs[pair_key] = jointype

mloda/core/runtime/compute_framework_executor.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import multiprocessing
+import threading
+import traceback
+import logging
+from typing import Any, Callable, Dict, Optional, Set, Type
+from uuid import UUID, uuid4
+
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.components.parallelization_modes import ParallelizationMode
+from mloda.core.core.cfw_manager import CfwManager
+from mloda.core.core.step.feature_group_step import FeatureGroupStep
+from mloda.core.core.step.join_step import JoinStep
+from mloda.core.core.step.transform_frame_work_step import TransformFrameworkStep
+from mloda.core.runtime.worker_manager import WorkerManager
+from mloda.core.runtime.worker.thread_worker import thread_worker
+from mloda.core.runtime.worker.multiprocessing_worker import worker
+
+logger = logging.getLogger(__name__)
+
+
+class ComputeFrameworkExecutor:
+    """
+    Manages compute framework initialization and step execution.
+
+    Extracted from Runner class to handle CFW lifecycle and step execution logic.
+    """
+
+    def __init__(self, cfw_register: CfwManager, worker_manager: WorkerManager) -> None:
+        """
+        Initialize the executor with dependencies.
+
+        Args:
+            cfw_register: The CFW manager for registering compute frameworks.
+            worker_manager: The worker manager for handling parallel execution.
+        """
+        self.cfw_collection: Dict[UUID, ComputeFramework] = {}
+        self.cfw_register = cfw_register
+        self.worker_manager = worker_manager
+
+    def init_compute_framework(
+        self,
+        cf_class: Type[ComputeFramework],
+        parallelization_mode: ParallelizationMode,
+        children_if_root: Set[UUID],
+        uuid: Optional[UUID] = None,
+    ) -> UUID:
+        """
+        Initializes a compute framework.
+
+        Returns:
+            The UUID of the compute framework.
+        """
+        # get function_extender
+        function_extender = self.cfw_register.get_function_extender()
+
+        # init framework
+        new_cfw = cf_class(
+            parallelization_mode,
+            frozenset(children_if_root),
+            uuid or uuid4(),
+            function_extender=function_extender,
+        )
+
+        # add to register
+        self.cfw_register.add_cfw_to_compute_frameworks(new_cfw.get_uuid(), cf_class.get_class_name(), children_if_root)
+
+        # add to collection
+        self.cfw_collection[new_cfw.get_uuid()] = new_cfw
+
+        return new_cfw.get_uuid()
+
+    def add_compute_framework(
+        self,
+        step: Any,
+        parallelization_mode: ParallelizationMode,
+        feature_uuid: UUID,
+        children_if_root: Set[UUID],
+    ) -> UUID:
+        """
+        Adds a compute framework to the CFW register and CFW collection.
+
+        Returns:
+            The UUID of the compute framework.
+        """
+        with multiprocessing.Lock():
+            cfw_uuid = self.cfw_register.get_cfw_uuid(step.compute_framework.get_class_name(), feature_uuid)
+            # if cfw does not exist, create a new one
+            if cfw_uuid is None:
+                cfw_uuid = self.init_compute_framework(step.compute_framework, parallelization_mode, children_if_root)
+
+            return cfw_uuid
+
+    def get_cfw(self, compute_framework: Type[ComputeFramework], feature_uuid: UUID) -> ComputeFramework:
+        """
+        Retrieves a compute framework based on its type and a feature UUID.
+
+        Args:
+            compute_framework: The type of compute framework to retrieve.
+            feature_uuid: The UUID of the feature associated with the compute framework.
+        """
+        cfw_uuid = self.cfw_register.get_initialized_compute_framework_uuid(
+            compute_framework, feature_uuid=feature_uuid
+        )
+        if cfw_uuid is None:
+            raise ValueError(f"cfw_uuid should not be none: {compute_framework}.")
+        return self.cfw_collection[cfw_uuid]
+
+    def _get_execution_function(
+        self, mode_by_cfw_register: Set[ParallelizationMode], mode_by_step: Set[ParallelizationMode]
+    ) -> Callable[[Any], None]:
+        """
+        Identifies the execution mode and returns the corresponding execute step function.
+
+        Returns:
+            The execute step function corresponding to the identified mode.
+        """
+        modes = mode_by_cfw_register.intersection(mode_by_step)
+
+        if ParallelizationMode.MULTIPROCESSING in modes:
+            return self.multi_execute_step
+        elif ParallelizationMode.THREADING in modes:
+            return self.thread_execute_step
+        return self.sync_execute_step
+
+    def prepare_execute_step(self, step: Any, parallelization_mode: ParallelizationMode) -> UUID:
+        """
+        Prepares a step for execution by initializing or retrieving the associated CFW.
+        """
+        cfw_uuid: Optional[UUID] = None
+
+        if isinstance(step, FeatureGroupStep):
+            for tfs_id in step.tfs_ids:
+                cfw_uuid = self.cfw_register.get_cfw_uuid(step.compute_framework.get_class_name(), tfs_id)
+                if cfw_uuid:
+                    return cfw_uuid
+
+            feature_uuid = step.features.any_uuid
+
+            if feature_uuid is None:
+                raise ValueError(f"from_feature_uuid should not be none. {step, feature_uuid}")
+
+            cfw_uuid = self.add_compute_framework(step, parallelization_mode, feature_uuid, set(step.children_if_root))
+        elif isinstance(step, TransformFrameworkStep):
+            from_feature_uuid, from_cfw_uuid = None, None
+            for r_f in step.required_uuids:
+                from_cfw_uuid = self.cfw_register.get_cfw_uuid(step.from_framework.get_class_name(), r_f)
+                if from_cfw_uuid:
+                    from_feature_uuid = r_f
+                    break
+
+            if from_feature_uuid is None or from_cfw_uuid is None:
+                raise ValueError(
+                    f"from_feature_uuid or from_cfw_uuid should not be none. {step, from_feature_uuid, from_cfw_uuid}"
+                )
+
+            from_cfw = self.cfw_collection[from_cfw_uuid]
+            childrens = set(from_cfw.children_if_root)
+
+            if step.link_id:
+                from_feature_uuid = step.link_id
+                childrens.add(from_feature_uuid)
+
+            with multiprocessing.Lock():
+                cfw_uuid = self.init_compute_framework(step.to_framework, parallelization_mode, childrens, step.uuid)
+
+        elif isinstance(step, JoinStep):
+            cfw_uuid = self.cfw_register.get_cfw_uuid(
+                step.left_framework.get_class_name(), next(iter(step.left_framework_uuids))
+            )
+
+        if cfw_uuid is None:
+            raise ValueError(f"This should not occur. {step}")
+
+        return cfw_uuid
+
+    def prepare_tfs_right_cfw(self, step: TransformFrameworkStep) -> UUID:
+        """
+        Prepares the right CFW for a TransformFrameworkStep.
+        """
+        uuid = step.right_framework_uuid if step.right_framework_uuid else next(iter(step.required_uuids))
+
+        cfw_uuid = self.cfw_register.get_cfw_uuid(step.from_framework.get_class_name(), uuid)
+
+        if cfw_uuid is None or isinstance(cfw_uuid, UUID) is False:
+            raise ValueError(
+                f"cfw_uuid should not be none in prepare_tfs: {step.from_framework.get_class_name()}, {uuid}"
+            )
+
+        return cfw_uuid
+
+    def prepare_tfs_and_joinstep(self, step: Any) -> Any:
+        """
+        Prepares CFWs required for TransformFrameworkStep or JoinStep.
+        """
+        from_cfw: Optional[Any] = None
+        if isinstance(step, TransformFrameworkStep):
+            from_cfw = self.prepare_tfs_right_cfw(step)
+            from_cfw = self.cfw_collection[from_cfw]
+        elif isinstance(step, JoinStep):
+            # Left framework here, because it is already transformed beforehand
+            from_cfw_uuid = self.cfw_register.get_cfw_uuid(step.left_framework.get_class_name(), step.link.uuid)
+
+            if from_cfw_uuid is None:
+                from_cfw_uuid = self.cfw_register.get_cfw_uuid(
+                    step.left_framework.get_class_name(), next(iter(step.right_framework_uuids))
+                )
+
+            if from_cfw_uuid is None:
+                raise ValueError(
+                    f"from_cfw_uuid should not be none: {step.left_framework.get_class_name()}, {step.link.uuid}"
+                )
+
+            from_cfw = self.cfw_collection[from_cfw_uuid]
+        return from_cfw
+
+    def sync_execute_step(self, step: Any) -> None:
+        """
+        Executes a step synchronously.
+        """
+        cfw_uuid = self.prepare_execute_step(step, ParallelizationMode.SYNC)
+
+        try:
+            from_cfw = self.prepare_tfs_and_joinstep(step) or None
+            step.execute(self.cfw_register, self.cfw_collection[cfw_uuid], from_cfw=from_cfw)
+            step.step_is_done = True
+
+        except Exception as e:
+            error_message = f"An error occurred: {e}"
+            msg = f"{error_message}\nFull traceback:\n{traceback.format_exc()}"
+            logging.error(msg)
+            exc_info = traceback.format_exc()
+            self.cfw_register.set_error(msg, exc_info)
+
+    def thread_execute_step(self, step: Any) -> None:
+        """
+        Executes a step in a separate thread.
+        """
+        cfw_uuid = self.prepare_execute_step(step, ParallelizationMode.THREADING)
+        from_cfw = self.prepare_tfs_and_joinstep(step) or None
+
+        task = threading.Thread(
+            target=thread_worker,
+            args=(step, self.cfw_register, self.cfw_collection[cfw_uuid], from_cfw),
+        )
+
+        self.worker_manager.add_thread_task(task)
+
+    def multi_execute_step(self, step: Any) -> None:
+        """
+        Executes a step in a separate process.
+        """
+        cfw_uuid = self.prepare_execute_step(step, ParallelizationMode.MULTIPROCESSING)
+
+        from_cfw = None
+        if isinstance(step, TransformFrameworkStep):
+            from_cfw = self.prepare_tfs_right_cfw(step)
+
+        existing = self.worker_manager.get_process_queues(cfw_uuid)
+
+        if existing is None:
+            process, command_queue, result_queue = self.worker_manager.create_worker_process(
+                cfw_uuid,
+                worker,
+                (self.cfw_register, self.cfw_collection[cfw_uuid], from_cfw),
+            )
+        else:
+            process, command_queue, result_queue = existing
+
+        self.worker_manager.send_command(cfw_uuid, step)