mloda 0.3.3__py3-none-any.whl → 0.4.1__py3-none-any.whl

mloda/core/runtime/data_lifecycle_manager.py

@@ -0,0 +1,160 @@
+from typing import Any, Dict, List, Optional, Set
+from uuid import UUID
+
+from mloda.core.abstract_plugins.components.framework_transformer.cfw_transformer import ComputeFrameworkTransformer
+from mloda.core.abstract_plugins.components.feature_name import FeatureName
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.abstract_plugins.components.feature_set import FeatureSet
+from mloda.core.runtime.flight.flight_server import FlightServer
+
+
+class DataLifecycleManager:
+    """
+    Manages data lifecycle including dropping, result collection, and artifacts.
+
+    This class handles the lifecycle of data in compute frameworks (CFWs),
+    including tracking data to drop, collecting results, and managing artifacts.
+    """
+
+    def __init__(self, transformer: Optional[ComputeFrameworkTransformer] = None) -> None:
+        """
+        Initializes DataLifecycleManager with empty state and transformer.
+
+        Args:
+            transformer: Optional transformer for CFW data conversion.
+                        If None, a new ComputeFrameworkTransformer is created.
+        """
+        self.result_data_collection: Dict[UUID, Any] = {}
+        self.track_data_to_drop: Dict[UUID, Set[UUID]] = {}
+        self.artifacts: Dict[str, Any] = {}
+        self.transformer = transformer if transformer is not None else ComputeFrameworkTransformer()
+
+    def drop_data_for_finished_cfws(
+        self, finished_ids: Set[UUID], cfw_collection: Dict[UUID, ComputeFramework], location: Optional[str] = None
+    ) -> None:
+        """
+        Drops data for CFWs when all their dependent steps are finished.
+
+        Args:
+            finished_ids: Set of step UUIDs that have been completed.
+            cfw_collection: Dictionary of CFWs keyed by UUID.
+            location: Optional location string for remote data dropping.
+        """
+        if not finished_ids:
+            return
+
+        cfw_to_delete = set()
+        for cfw_uuid, step_uuids in self.track_data_to_drop.items():
+            if all(step_id in finished_ids for step_id in step_uuids):
+                self.drop_cfw_data(cfw_uuid, cfw_collection, location)
+                cfw_to_delete.add(cfw_uuid)
+
+        for cfw_uuid in cfw_to_delete:
+            del self.track_data_to_drop[cfw_uuid]
+
+    def drop_cfw_data(
+        self, cfw_uuid: UUID, cfw_collection: Dict[UUID, ComputeFramework], location: Optional[str] = None
+    ) -> None:
+        """
+        Drops data associated with a specific CFW.
+
+        Args:
+            cfw_uuid: The UUID of the CFW to drop data for.
+            cfw_collection: Dictionary of CFWs keyed by UUID.
+            location: Optional location string for remote data dropping.
+        """
+        cfw = cfw_collection[cfw_uuid]
+        if location:
+            cfw.drop_last_data(location)
+        else:
+            cfw.drop_last_data(None)
+
+    def track_flyway_datasets(self, cfw_uuid: UUID, datasets: Set[UUID]) -> None:
+        """
+        Stores flyway datasets for a CFW UUID for later dropping.
+
+        Args:
+            cfw_uuid: The UUID of the CFW.
+            datasets: Set of dataset UUIDs to track for dropping.
+        """
+        self.track_data_to_drop[cfw_uuid] = datasets
+
+    def add_to_result_data_collection(
+        self, cfw: ComputeFramework, features: FeatureSet, step_uuid: UUID, location: Optional[str] = None
+    ) -> None:
+        """
+        Adds result data to the collection if features are requested.
+
+        Args:
+            cfw: The compute framework containing the data.
+            features: The feature set to extract from the CFW.
+            step_uuid: The UUID of the step to associate with the result.
+            location: Optional location string for remote data access.
+        """
+        initial_requested_features = features.get_initial_requested_features()
+        if not initial_requested_features:
+            return
+
+        result = self.get_result_data(cfw, initial_requested_features, location)
+        if result is not None:
+            self.result_data_collection[step_uuid] = result
+
+    def get_result_data(
+        self, cfw: ComputeFramework, selected_feature_names: Set[FeatureName], location: Optional[str] = None
+    ) -> Any:
+        """
+        Gets result data from the compute framework.
+
+        Args:
+            cfw: The compute framework containing the data.
+            selected_feature_names: Set of feature names to select.
+            location: Optional location string for remote data access.
+
+        Returns:
+            The selected data from the CFW.
+
+        Raises:
+            ValueError: If CFW has no data and no location is provided.
+        """
+        if cfw.data is not None:
+            data = cfw.data
+        elif location:
+            data = FlightServer.download_table(location, str(cfw.uuid))
+            data = cfw.convert_flyserver_data_back(data, self.transformer)
+        else:
+            raise ValueError("Not implemented")
+
+        return cfw.select_data_by_column_names(data, selected_feature_names)
+
+    def get_results(self) -> List[Any]:
+        """
+        Returns list of all collected results.
+
+        Returns:
+            List of all result data.
+
+        Raises:
+            ValueError: If no results have been collected.
+        """
+        if not self.result_data_collection:
+            raise ValueError("No results found")
+
+        return list(self.result_data_collection.values())
+
+    def set_artifacts(self, artifacts: Dict[str, Any]) -> None:
+        """
+        Stores artifacts dictionary.
+
+        Args:
+            artifacts: Dictionary of artifacts to store.
+        """
+        self.artifacts = artifacts
+
+    def get_artifacts(self) -> Dict[str, Any]:
+        """
+        Returns stored artifacts.
+
+        Returns:
+            Dictionary of artifacts.
+        """
+        return self.artifacts

{mloda_core → mloda/core}/runtime/flight/runner_flight_server.py

@@ -4,7 +4,7 @@ from typing import Any, List
 
 import logging
 
-from mloda_core.runtime.flight.flight_server import FlightServer, create_location
+from mloda.core.runtime.flight.flight_server import FlightServer, create_location
 
 logger = logging.getLogger(__name__)
 

mloda/core/runtime/run.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import multiprocessing
+import threading
+import time
+from typing import Any, Dict, List, Optional, Set, Union
+from uuid import UUID
+import logging
+
+from mloda.core.abstract_plugins.function_extender import Extender
+from mloda.core.abstract_plugins.components.feature_name import FeatureName
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.prepare.execution_plan import ExecutionPlan
+from mloda.core.runtime.worker_manager import WorkerManager
+from mloda.core.runtime.data_lifecycle_manager import DataLifecycleManager
+from mloda.core.runtime.compute_framework_executor import ComputeFrameworkExecutor
+from mloda.core.core.cfw_manager import CfwManager, MyManager
+from mloda.core.abstract_plugins.components.parallelization_modes import ParallelizationMode
+from mloda.core.runtime.flight.runner_flight_server import ParallelRunnerFlightServer
+from mloda.core.core.step.feature_group_step import FeatureGroupStep
+from mloda.core.core.step.join_step import JoinStep
+from mloda.core.core.step.transform_frame_work_step import TransformFrameworkStep
+from mloda.core.abstract_plugins.components.feature_set import FeatureSet
+
+
+logger = logging.getLogger(__name__)
+
+
+class ExecutionOrchestrator:
+    """
+    Orchestrates the execution of an mloda based on a given execution plan.
+
+    This class manages compute frameworks (CFWs), data dependencies, and parallel execution
+    using threads or multiprocessing. It handles the execution of feature group steps,
+    transform framework steps, and join steps, while also managing data dropping and result collection.
+    """
+
+    def __init__(
+        self,
+        execution_planner: ExecutionPlan,
+        flight_server: Optional[ParallelRunnerFlightServer] = None,
+    ) -> None:
+        """
+        Initializes the ExecutionOrchestrator with an execution plan and optional flight server.
+
+        Args:
+            execution_planner: The execution plan that defines the steps to be executed.
+            flight_server: An optional flight server for data transfer.
+        """
+        self.execution_planner = execution_planner
+
+        self.cfw_register: CfwManager
+
+        # multiprocessing - delegate to WorkerManager
+        self.location: Optional[str] = None
+        self.worker_manager = WorkerManager()
+
+        # Data lifecycle - delegate to DataLifecycleManager
+        self.data_lifecycle_manager = DataLifecycleManager()
+
+        self.flight_server = None
+        if flight_server:
+            self.flight_server = flight_server
+
+    def _is_step_done(self, step_uuids: Set[UUID], finished_ids: Set[UUID]) -> bool:
+        """
+        Checks if all steps identified by the given UUIDs have already been finished.
+        """
+        return all(uuid in finished_ids for uuid in step_uuids)
+
+    def _drop_data_for_finished_cfws(self, finished_ids: Set[UUID]) -> None:
+        """
+        Handles the dropping of intermediate data based on finished steps.
+        """
+        self.data_lifecycle_manager.drop_data_for_finished_cfws(
+            finished_ids, self.executor.cfw_collection, self.location
+        )
+
+    def compute(self) -> None:
+        """
+        Executes the mloda pipeline based on the execution plan.
+
+        This method iterates through the execution plan, checks dependencies,
+        and executes steps using the appropriate parallelization mode.
+        It also handles errors, result collection, and data dropping.
+        """
+        if self.cfw_register is None:
+            raise ValueError("CfwManager not initialized")
+
+        self.executor = ComputeFrameworkExecutor(self.cfw_register, self.worker_manager)
+
+        finished_ids: Set[UUID] = set()
+        to_finish_ids: Set[UUID] = set()
+        currently_running_steps: Set[UUID] = set()
+
+        try:
+            while to_finish_ids != finished_ids or len(finished_ids) == 0:
+                if self.cfw_register:
+                    error = self.cfw_register.get_error()
+                    if error:
+                        logger.error(self.cfw_register.get_error_exc_info())
+                        raise Exception(self.cfw_register.get_error_exc_info(), self.cfw_register.get_error_msg())
+                else:
+                    break
+
+                for step in self.execution_planner:
+                    to_finish_ids.update(step.get_uuids())
+
+                    if isinstance(step, FeatureGroupStep):
+                        self._drop_data_for_finished_cfws(finished_ids)
+
+                    if self._is_step_done(step.get_uuids(), finished_ids):
+                        continue
+
+                    # check if step is currently running
+                    if self.currently_running_step(step.get_uuids(), currently_running_steps):
+                        if self._process_step_result(step):
+                            self._mark_step_as_finished(step.get_uuids(), finished_ids, currently_running_steps)
+                        continue
+
+                    if not self._can_run_step(
+                        step.required_uuids, step.get_uuids(), finished_ids, currently_running_steps
+                    ):
+                        continue
+                    self._execute_step(step)
+
+                time.sleep(0.01)
+
+        finally:
+            self.data_lifecycle_manager.set_artifacts(self.cfw_register.get_artifacts())
+            self.join()
+
+    def _process_step_result(self, step: Any) -> Union[Any, bool]:
+        """
+        Handles the result of a step based on its type.
+
+        This method checks if a step is done, then performs specific actions based
+        on the step's type, such as adding results to the data collection or dropping data.
+        """
+        # set step.is_done from other processes via result queue
+        self.worker_manager.poll_result_queues()
+        if step.uuid in self.worker_manager.result_uuids_collection:
+            step.step_is_done = True
+
+        if not step.step_is_done:
+            return False
+
+        if isinstance(step, (TransformFrameworkStep, JoinStep)):
+            return True
+
+        if isinstance(step, FeatureGroupStep):
+            if step.features.any_uuid is None:
+                raise ValueError(f"from_feature_uuid should not be none. {step}")
+
+            cfw = self.executor.get_cfw(step.compute_framework, step.features.any_uuid)
+            self.add_to_result_data_collection(cfw, step.features, step.uuid)
+            self._drop_data_if_possible(cfw, step)
+
+        return True
+
+    def _drop_data_if_possible(self, cfw: ComputeFramework, step: Any) -> None:
+        """
+        Drops data associated with a compute framework if possible.
+
+        This method checks if data can be dropped based on the CFW's dependencies
+        and either drops the data directly or sends a command to a worker process to do so.
+        """
+        process, command_queue, result_queue = self.worker_manager.process_register.get(cfw.uuid, (None, None, None))
+
+        feature_uuids_to_possible_drop = {f.uuid for f in step.features.features}
+
+        if command_queue is None:
+            data_to_drop = cfw.add_already_calculated_children_and_drop_if_possible(
+                feature_uuids_to_possible_drop, self.location
+            )
+            if isinstance(data_to_drop, frozenset):
+                self.data_lifecycle_manager.track_data_to_drop[cfw.uuid] = set(data_to_drop)
+        else:
+            command_queue.put(feature_uuids_to_possible_drop)
+
+            flyway_datasets = self.cfw_register.get_uuid_flyway_datasets(cfw.uuid)
+            if flyway_datasets:
+                self.data_lifecycle_manager.track_data_to_drop[cfw.uuid] = flyway_datasets
+
+            if result_queue is not None:
+                self._wait_for_drop_completion(result_queue, cfw.uuid)
+
+    def _wait_for_drop_completion(
+        self, result_queue: multiprocessing.Queue[Any], cfw_uuid: UUID, timeout: float = 5.0
+    ) -> None:
+        """
+        Wait for drop operation to complete from worker process.
+
+        Args:
+            result_queue: The queue to receive completion signals from the worker.
+            cfw_uuid: The UUID of the compute framework being dropped.
+            timeout: Maximum time to wait for completion in seconds.
+        """
+        self.worker_manager.wait_for_drop_completion(result_queue, cfw_uuid, timeout)
+
+    def _execute_step(self, step: Any) -> None:
+        """
+        Executes a step based on its parallelization mode.
+        """
+        execution_function = self.executor._get_execution_function(
+            self.cfw_register.get_parallelization_modes(), step.get_parallelization_mode()
+        )
+        execution_function(step)
+
+    def join(self) -> None:
+        """
+        Joins all tasks (threads or processes) and terminates multiprocessing processes.
+        """
+        self.worker_manager.join_all()
+
+    def add_to_result_data_collection(self, cfw: ComputeFramework, features: FeatureSet, step_uuid: UUID) -> None:
+        """
+        Adds the result data to the result data collection.
+        """
+        self.data_lifecycle_manager.add_to_result_data_collection(cfw, features, step_uuid, self.location)
+
+    def get_result_data(
+        self, cfw: ComputeFramework, selected_feature_names: Set[FeatureName], location: Optional[str] = None
+    ) -> Any:
+        """
+        Gets result data from the compute framework.
+        """
+        return self.data_lifecycle_manager.get_result_data(cfw, selected_feature_names, location)
+
+    def currently_running_step(self, step_uuids: Set[UUID], currently_running_steps: Set[UUID]) -> bool:
+        """
+        Checks if a step is currently running.
+
+        Returns:
+            True if the step is currently running, False otherwise.
+        """
+        if next(iter(step_uuids)) not in currently_running_steps:
+            return False
+        return True
+
+    def __enter__(
+        self,
+        parallelization_modes: Set[ParallelizationMode] = {ParallelizationMode.SYNC},
+        function_extender: Optional[Set[Extender]] = None,
+        api_data: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Enters the context of the ExecutionOrchestrator.
+        """
+        MyManager.register("CfwManager", CfwManager)
+        self.manager = MyManager().__enter__()
+        self.cfw_register = self.manager.CfwManager(parallelization_modes, function_extender)  # type: ignore[attr-defined]
+
+        if self.flight_server:
+            if self.flight_server.flight_server_process is None:
+                self.flight_server.start_flight_server_process()
+
+        if self.flight_server:
+            self.location = self.flight_server.get_location()
+
+            if self.location is None:
+                raise ValueError("Location should not be None.")
+
+            self.cfw_register.set_location(self.location)
+
+        if api_data:
+            self.cfw_register.set_api_data(api_data)
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """
+        Exits the context of the ExecutionOrchestrator.
+
+        Args:
+            exc_type: The exception type.
+            exc_val: The exception value.
+            exc_tb: The exception traceback.
+        """
+        self.manager.shutdown()
+
+    def get_artifacts(self) -> Dict[str, Any]:
+        """
+        Gets the artifacts.
+        """
+        return self.data_lifecycle_manager.get_artifacts()
+
+    def _can_run_step(
+        self,
+        required_uuids: Set[UUID],
+        step_uuid: Set[UUID],
+        finished_steps: Set[UUID],
+        currently_running_steps: Set[UUID],
+    ) -> bool:
+        """
+        Checks if a step can be run. If it can, add it to the currently_running_steps set.
+        """
+
+        with threading.Lock():
+            if required_uuids.issubset(finished_steps) and not step_uuid.intersection(currently_running_steps):
+                currently_running_steps.update(step_uuid)
+                return True
+            return False
+
+    def _mark_step_as_finished(
+        self, step_uuid: Set[UUID], finished_steps: Set[UUID], currently_running_steps: Set[UUID]
+    ) -> None:
+        """
+        Marks a step as finished.
+        """
+        with threading.Lock():
+            currently_running_steps.difference_update(step_uuid)
+            finished_steps.update(step_uuid)
+
+    def get_result(self) -> List[Any]:
+        """
+        Gets the results.
+        """
+        return self.data_lifecycle_manager.get_results()

{mloda_core → mloda/core}/runtime/worker/multiprocessing_worker.py

@@ -8,11 +8,11 @@ from typing import Any, Set, Union
 from uuid import UUID
 from queue import Empty
 
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
-from mloda_core.core.cfw_manager import CfwManager
-from mloda_core.core.step.feature_group_step import FeatureGroupStep
-from mloda_core.core.step.join_step import JoinStep
-from mloda_core.core.step.transform_frame_work_step import TransformFrameworkStep
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
+from mloda.core.core.cfw_manager import CfwManager
+from mloda.core.core.step.feature_group_step import FeatureGroupStep
+from mloda.core.core.step.join_step import JoinStep
+from mloda.core.core.step.transform_frame_work_step import TransformFrameworkStep
 
 
 logger = logging.getLogger(__name__)
@@ -26,12 +26,17 @@ def _handle_stop_command(command_queue: multiprocessing.Queue[Any]) -> None:
 
 def _handle_data_dropping(
     command_queue: multiprocessing.Queue[Any],
-    cfw: ComputeFrameWork,
+    cfw: ComputeFramework,
     command: Set[Any],
     location: str,
+    result_queue: multiprocessing.Queue[Any],
 ) -> bool:
     """Handles dropping already calculated data based on the provided command."""
     data_to_drop = cfw.add_already_calculated_children_and_drop_if_possible(command, location)
+
+    # Signal completion back to main thread
+    result_queue.put(("DROP_COMPLETE", cfw.uuid), block=False)
+
     if data_to_drop is True:
         _handle_stop_command(command_queue)
         return True
@@ -41,7 +46,7 @@ def _handle_data_dropping(
 def _execute_command(
     command: Union[JoinStep, TransformFrameworkStep, FeatureGroupStep],
     cfw_register: CfwManager,
-    cfw: ComputeFrameWork,
+    cfw: ComputeFramework,
     data: Any,
     from_cfw: UUID,
 ) -> Any:
@@ -73,7 +78,7 @@ def _execute_command(
 
 def _handle_command_result(
     command: FeatureGroupStep,
-    cfw: ComputeFrameWork,
+    cfw: ComputeFramework,
     location: str,
     data: Any,
     result_queue: multiprocessing.Queue[Any],
@@ -94,7 +99,7 @@ def worker(
     command_queue: multiprocessing.Queue[Any],
     result_queue: multiprocessing.Queue[Any],
     cfw_register: CfwManager,
-    cfw: ComputeFrameWork,
+    cfw: ComputeFramework,
     from_cfw: UUID,
 ) -> None:
     data = None
@@ -115,7 +120,7 @@ def worker(
             break
 
         if isinstance(command, set):
-            if _handle_data_dropping(command_queue, cfw, command, location):
+            if _handle_data_dropping(command_queue, cfw, command, location, result_queue):
                 break
             continue
 

{mloda_core → mloda/core}/runtime/worker/thread_worker.py

@@ -2,13 +2,13 @@ import logging
 import traceback
 from typing import Any
 
-from mloda_core.abstract_plugins.compute_frame_work import ComputeFrameWork
+from mloda.core.abstract_plugins.compute_framework import ComputeFramework
 
 
 logger = logging.getLogger(__name__)
 
 
-def thread_worker(command: Any, cfw_register: Any, cfw: ComputeFrameWork, from_cfw: ComputeFrameWork) -> None:
+def thread_worker(command: Any, cfw_register: Any, cfw: ComputeFramework, from_cfw: ComputeFramework) -> None:
     try:
         command.execute(cfw_register, cfw, from_cfw=from_cfw)
         command.step_is_done = True

mloda/core/runtime/worker_manager.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import multiprocessing
+import queue
+import threading
+import time
+import logging
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
+from uuid import UUID
+
+logger = logging.getLogger(__name__)
+
+
+class WorkerManager:
+    """Manages thread/process lifecycle for parallel execution."""
+
+    def __init__(self) -> None:
+        """Initialize empty state."""
+        self.tasks: List[Union[threading.Thread, multiprocessing.Process]] = []
+        self.process_register: Dict[UUID, Tuple[Any, Any, Any]] = {}
+        self.result_queues_collection: Set[Any] = set()
+        self.result_uuids_collection: Set[UUID] = set()
+
+    def add_thread_task(self, task: threading.Thread) -> None:
+        """Add task to list and call task.start()."""
+        self.tasks.append(task)
+        task.start()
+
+    def create_worker_process(
+        self, cfw_uuid: UUID, target: Callable[..., None], args: Tuple[Any, ...]
+    ) -> Tuple[Any, Any, Any]:
+        """Create worker process with command and result queues."""
+        command_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
+        result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
+
+        process = multiprocessing.Process(target=target, args=(command_queue, result_queue, *args))
+
+        self.process_register[cfw_uuid] = (process, command_queue, result_queue)
+        self.result_queues_collection.add(result_queue)
+        self.tasks.append(process)
+        process.start()
+
+        return process, command_queue, result_queue
+
+    def get_process_queues(self, cfw_uuid: UUID) -> Optional[Tuple[Any, Any, Any]]:
+        """Return registered tuple or None."""
+        return self.process_register.get(cfw_uuid)
+
+    def send_command(self, cfw_uuid: UUID, command: Any) -> None:
+        """Put command in command_queue, raise ValueError if not found."""
+        result = self.process_register.get(cfw_uuid)
+        if result is None:
+            raise ValueError(f"No process found for CFW UUID: {cfw_uuid}")
+        _, command_queue, _ = result
+        command_queue.put(command)
+
+    def poll_result_queues(self) -> None:
+        """Non-blocking poll all result queues, add UUIDs to result_uuids_collection."""
+        for r_queue in self.result_queues_collection:
+            try:
+                result_uuid = r_queue.get(block=False)
+                self.result_uuids_collection.add(UUID(result_uuid))
+            except queue.Empty:
+                continue
+
+    def is_step_done(self, step_uuid: UUID) -> bool:
+        """Return step_uuid in result_uuids_collection."""
+        return step_uuid in self.result_uuids_collection
+
+    def wait_for_drop_completion(self, result_queue: Any, cfw_uuid: UUID, timeout: float = 5.0) -> None:
+        """Poll queue until ("DROP_COMPLETE", cfw_uuid) received or timeout."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                msg = result_queue.get(block=False)
+                if isinstance(msg, tuple) and len(msg) == 2 and msg[0] == "DROP_COMPLETE" and msg[1] == cfw_uuid:
+                    return
+                result_queue.put(msg, block=False)
+            except queue.Empty:
+                time.sleep(0.001)
+        logger.warning(f"Drop operation for CFW {cfw_uuid} timed out after {timeout}s")
+
+    def join_all(self) -> None:
+        """Terminate processes (not threads), join all tasks, raise Exception if any fail."""
+        failed = False
+        for task in self.tasks:
+            try:
+                if isinstance(task, multiprocessing.Process):
+                    task.terminate()
+                task.join()
+            except Exception as e:
+                logger.error(f"Error joining task: {e}")
+                failed = True
+
+        if failed:
+            raise Exception("Error while joining tasks")