executorlib 0.2.0__tar.gz → 0.3.0__tar.gz

{executorlib-0.2.0/executorlib.egg-info → executorlib-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: executorlib
-Version: 0.2.0
+Version: 0.3.0
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Author-email: Jan Janssen <janssen@lanl.gov>
 License: BSD 3-Clause License

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-02-11T12:47:28+0100",
+ "date": "2025-02-14T16:51:45+0100",
  "dirty": true,
  "error": null,
- "full-revisionid": "0ffd31288952f78be3b0c810eac1890759634d35",
- "version": "0.2.0"
+ "full-revisionid": "ff49de3af89e8739b3b3c5fa041055cf4bc9bc14",
+ "version": "0.3.0"
 }
 '''  # END VERSION_JSON
 

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/base/executor.py

@@ -6,12 +6,12 @@ from concurrent.futures import (
 from concurrent.futures import (
     Future,
 )
+from threading import Thread
 from typing import Callable, Optional, Union
 
 from executorlib.standalone.inputcheck import check_resource_dict
 from executorlib.standalone.queue import cancel_items_in_queue
 from executorlib.standalone.serialize import cloudpickle_register
-from executorlib.standalone.thread import RaisingThread
 
 
 class ExecutorBase(FutureExecutor):
@@ -29,7 +29,7 @@ class ExecutorBase(FutureExecutor):
         cloudpickle_register(ind=3)
         self._max_cores = max_cores
         self._future_queue: Optional[queue.Queue] = queue.Queue()
-        self._process: Optional[Union[RaisingThread, list[RaisingThread]]] = None
+        self._process: Optional[Union[Thread, list[Thread]]] = None
 
     @property
     def info(self) -> Optional[dict]:
@@ -40,13 +40,13 @@ class ExecutorBase(FutureExecutor):
             Optional[dict]: Information about the executor.
         """
         if self._process is not None and isinstance(self._process, list):
-            meta_data_dict = self._process[0].get_kwargs().copy()
+            meta_data_dict = self._process[0]._kwargs.copy()  # type: ignore
             if "future_queue" in meta_data_dict:
                 del meta_data_dict["future_queue"]
             meta_data_dict["max_workers"] = len(self._process)
             return meta_data_dict
         elif self._process is not None:
-            meta_data_dict = self._process.get_kwargs().copy()
+            meta_data_dict = self._process._kwargs.copy()  # type: ignore
             if "future_queue" in meta_data_dict:
                 del meta_data_dict["future_queue"]
             return meta_data_dict
@@ -138,13 +138,13 @@ class ExecutorBase(FutureExecutor):
             cancel_items_in_queue(que=self._future_queue)
         if self._process is not None and self._future_queue is not None:
             self._future_queue.put({"shutdown": True, "wait": wait})
-            if wait and isinstance(self._process, RaisingThread):
+            if wait and isinstance(self._process, Thread):
                 self._process.join()
                 self._future_queue.join()
         self._process = None
         self._future_queue = None
 
-    def _set_process(self, process: RaisingThread):
+    def _set_process(self, process: Thread):
         """
         Set the process for the executor.
 

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/cache/executor.py

@@ -1,4 +1,5 @@
 import os
+from threading import Thread
 from typing import Callable, Optional
 
 from executorlib.base.executor import ExecutorBase
@@ -15,7 +16,6 @@ from executorlib.standalone.inputcheck import (
     check_max_workers_and_cores,
     check_nested_flux_executor,
 )
-from executorlib.standalone.thread import RaisingThread
 
 try:
     from executorlib.cache.queue_spawner import execute_with_pysqa
@@ -64,7 +64,7 @@ class FileExecutor(ExecutorBase):
         cache_directory_path = os.path.abspath(cache_directory)
         os.makedirs(cache_directory_path, exist_ok=True)
         self._set_process(
-            RaisingThread(
+            Thread(
                 target=execute_tasks_h5,
                 kwargs={
                     "future_queue": self._future_queue,

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/cache/shared.py

@@ -115,8 +115,10 @@ def execute_tasks_h5(
                         ]
                     else:
                         if len(future_wait_key_lst) > 0:
-                            raise ValueError(
-                                "Future objects are not supported as input if disable_dependencies=True."
+                            task_dict["future"].set_exception(
+                                ValueError(
+                                    "Future objects are not supported as input if disable_dependencies=True."
+                                )
                             )
                         task_dependent_lst = []
                     process_dict[task_key] = execute_function(

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/interactive/executor.py

@@ -1,4 +1,5 @@
 from concurrent.futures import Future
+from threading import Thread
 from typing import Any, Callable, Optional
 
 from executorlib.base.executor import ExecutorBase
@@ -8,7 +9,6 @@ from executorlib.standalone.plot import (
     generate_nodes_and_edges,
     generate_task_hash,
 )
-from executorlib.standalone.thread import RaisingThread
 
 
 class ExecutorWithDependencies(ExecutorBase):
@@ -41,7 +41,7 @@ class ExecutorWithDependencies(ExecutorBase):
     ) -> None:
         super().__init__(max_cores=max_cores)
         self._set_process(
-            RaisingThread(
+            Thread(
                 target=execute_tasks_with_dependencies,
                 kwargs={
                     # Executor Arguments

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/interactive/shared.py

@@ -3,7 +3,9 @@ import os
 import queue
 import sys
 import time
-from concurrent.futures import Future
+from asyncio.exceptions import CancelledError
+from concurrent.futures import Future, TimeoutError
+from threading import Thread
 from time import sleep
 from typing import Any, Callable, Optional, Union
 
@@ -19,7 +21,6 @@ from executorlib.standalone.interactive.communication import (
 )
 from executorlib.standalone.interactive.spawner import BaseSpawner, MpiExecSpawner
 from executorlib.standalone.serialize import serialize_funct_h5
-from executorlib.standalone.thread import RaisingThread
 
 
 class ExecutorBroker(ExecutorBase):
@@ -88,7 +89,7 @@ class ExecutorBroker(ExecutorBase):
         self._process = None
         self._future_queue = None
 
-    def _set_process(self, process: list[RaisingThread]):  # type: ignore
+    def _set_process(self, process: list[Thread]):  # type: ignore
         """
         Set the process for the executor.
 
@@ -148,7 +149,7 @@ class InteractiveExecutor(ExecutorBroker):
         executor_kwargs["queue_join_on_shutdown"] = False
         self._set_process(
             process=[
-                RaisingThread(
+                Thread(
                     target=execute_parallel_tasks,
                     kwargs=executor_kwargs,
                 )
@@ -204,7 +205,7 @@ class InteractiveStepExecutor(ExecutorBase):
         executor_kwargs["max_cores"] = max_cores
         executor_kwargs["max_workers"] = max_workers
         self._set_process(
-            RaisingThread(
+            Thread(
                 target=execute_separate_tasks,
                 kwargs=executor_kwargs,
             )
@@ -361,15 +362,19 @@ def execute_tasks_with_dependencies(
             task_dict is not None and "fn" in task_dict and "future" in task_dict
         ):
             future_lst, ready_flag = _get_future_objects_from_input(task_dict=task_dict)
-            if len(future_lst) == 0 or ready_flag:
-                # No future objects are used in the input or all future objects are already done
-                task_dict["args"], task_dict["kwargs"] = _update_futures_in_input(
-                    args=task_dict["args"], kwargs=task_dict["kwargs"]
-                )
-                executor_queue.put(task_dict)
-            else:  # Otherwise add the function to the wait list
-                task_dict["future_lst"] = future_lst
-                wait_lst.append(task_dict)
+            exception_lst = _get_exception_lst(future_lst=future_lst)
+            if not _get_exception(future_obj=task_dict["future"]):
+                if len(exception_lst) > 0:
+                    task_dict["future"].set_exception(exception_lst[0])
+                elif len(future_lst) == 0 or ready_flag:
+                    # No future objects are used in the input or all future objects are already done
+                    task_dict["args"], task_dict["kwargs"] = _update_futures_in_input(
+                        args=task_dict["args"], kwargs=task_dict["kwargs"]
+                    )
+                    executor_queue.put(task_dict)
+                else:  # Otherwise add the function to the wait list
+                    task_dict["future_lst"] = future_lst
+                    wait_lst.append(task_dict)
             future_queue.task_done()
         elif len(wait_lst) > 0:
             number_waiting = len(wait_lst)
@@ -455,7 +460,10 @@ def _submit_waiting_task(wait_lst: list[dict], executor_queue: queue.Queue) -> l
     """
     wait_tmp_lst = []
     for task_wait_dict in wait_lst:
-        if all(future.done() for future in task_wait_dict["future_lst"]):
+        exception_lst = _get_exception_lst(future_lst=task_wait_dict["future_lst"])
+        if len(exception_lst) > 0:
+            task_wait_dict["future"].set_exception(exception_lst[0])
+        elif all(future.done() for future in task_wait_dict["future_lst"]):
             del task_wait_dict["future_lst"]
             task_wait_dict["args"], task_wait_dict["kwargs"] = _update_futures_in_input(
                 args=task_wait_dict["args"], kwargs=task_wait_dict["kwargs"]
@@ -582,7 +590,7 @@ def _submit_function_to_separate_process(
             "init_function": None,
         }
     )
-    process = RaisingThread(
+    process = Thread(
         target=execute_parallel_tasks,
         kwargs=task_kwargs,
     )
@@ -603,14 +611,13 @@ def _execute_task(
         future_queue (Queue): Queue for receiving new tasks.
     """
     f = task_dict.pop("future")
-    if f.set_running_or_notify_cancel():
+    if not f.done() and f.set_running_or_notify_cancel():
         try:
             f.set_result(interface.send_and_receive_dict(input_dict=task_dict))
         except Exception as thread_exception:
             interface.shutdown(wait=True)
             future_queue.task_done()
             f.set_exception(exception=thread_exception)
-            raise thread_exception
         else:
             future_queue.task_done()
 
@@ -663,3 +670,15 @@ def _execute_task_with_cache(
         future = task_dict["future"]
         future.set_result(result)
         future_queue.task_done()
+
+
+def _get_exception_lst(future_lst: list[Future]) -> list:
+    return [f.exception() for f in future_lst if _get_exception(future_obj=f)]
+
+
+def _get_exception(future_obj: Future) -> bool:
+    try:
+        excp = future_obj.exception(timeout=10**-10)
+        return excp is not None and not isinstance(excp, CancelledError)
+    except TimeoutError:
+        return False

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib/standalone/__init__.py

@@ -7,7 +7,6 @@ from executorlib.standalone.interactive.communication import (
     interface_shutdown,
 )
 from executorlib.standalone.interactive.spawner import MpiExecSpawner
-from executorlib.standalone.thread import RaisingThread
 
 __all__ = [
     "SocketInterface",
@@ -16,6 +15,5 @@ __all__ = [
     "interface_send",
     "interface_shutdown",
     "interface_receive",
-    "RaisingThread",
     "MpiExecSpawner",
 ]

{executorlib-0.2.0 → executorlib-0.3.0/executorlib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: executorlib
-Version: 0.2.0
+Version: 0.3.0
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Author-email: Jan Janssen <janssen@lanl.gov>
 License: BSD 3-Clause License

{executorlib-0.2.0 → executorlib-0.3.0}/executorlib.egg-info/SOURCES.txt

@@ -39,7 +39,6 @@ executorlib/standalone/inputcheck.py
 executorlib/standalone/plot.py
 executorlib/standalone/queue.py
 executorlib/standalone/serialize.py
-executorlib/standalone/thread.py
 executorlib/standalone/interactive/__init__.py
 executorlib/standalone/interactive/backend.py
 executorlib/standalone/interactive/communication.py
@@ -66,6 +65,5 @@ tests/test_shared_backend.py
 tests/test_shared_communication.py
 tests/test_shared_executorbase.py
 tests/test_shared_input_check.py
-tests/test_shared_thread.py
 tests/test_shell_executor.py
 tests/test_shell_interactive.py

{executorlib-0.2.0 → executorlib-0.3.0}/tests/test_cache_executor_serial.py

@@ -3,12 +3,12 @@ import os
 from queue import Queue
 import shutil
 import unittest
+from threading import Thread
 
 from executorlib.cache.subprocess_spawner import (
     execute_in_subprocess,
     terminate_subprocess,
 )
-from executorlib.standalone.thread import RaisingThread
 
 try:
     from executorlib.cache.executor import FileExecutor, create_file_executor
@@ -57,7 +57,8 @@ class TestCacheExecutorSerial(unittest.TestCase):
             with FileExecutor(
                 execute_function=execute_in_subprocess, disable_dependencies=True
             ) as exe:
-                exe.submit(my_funct, 1, b=exe.submit(my_funct, 1, b=2))
+                fs = exe.submit(my_funct, 1, b=exe.submit(my_funct, 1, b=2))
+                fs.result()
 
     def test_executor_working_directory(self):
         cwd = os.path.join(os.path.dirname(__file__), "executables")
@@ -81,7 +82,7 @@ class TestCacheExecutorSerial(unittest.TestCase):
         )
         cache_dir = os.path.abspath("cache")
         os.makedirs(cache_dir, exist_ok=True)
-        process = RaisingThread(
+        process = Thread(
             target=execute_tasks_h5,
             kwargs={
                 "future_queue": q,
@@ -122,7 +123,7 @@ class TestCacheExecutorSerial(unittest.TestCase):
         )
         cache_dir = os.path.abspath("cache")
         os.makedirs(cache_dir, exist_ok=True)
-        process = RaisingThread(
+        process = Thread(
             target=execute_tasks_h5,
             kwargs={
                 "future_queue": q,
@@ -163,7 +164,7 @@ class TestCacheExecutorSerial(unittest.TestCase):
         )
         cache_dir = os.path.abspath("cache")
         os.makedirs(cache_dir, exist_ok=True)
-        process = RaisingThread(
+        process = Thread(
             target=execute_tasks_h5,
             kwargs={
                 "future_queue": q,

executorlib-0.3.0/tests/test_dependencies_executor.py

@@ -0,0 +1,330 @@
+from concurrent.futures import Future
+import unittest
+import sys
+from time import sleep
+from queue import Queue
+from threading import Thread
+
+from executorlib import SingleNodeExecutor
+from executorlib.interfaces.single import create_single_node_executor
+from executorlib.interactive.shared import execute_tasks_with_dependencies
+from executorlib.standalone.serialize import cloudpickle_register
+
+
+try:
+    import pygraphviz
+
+    skip_graphviz_test = False
+except ImportError:
+    skip_graphviz_test = True
+
+
+def add_function(parameter_1, parameter_2):
+    sleep(0.2)
+    return parameter_1 + parameter_2
+
+
+def generate_tasks(length):
+    sleep(0.2)
+    return range(length)
+
+
+def calc_from_lst(lst, ind, parameter):
+    sleep(0.2)
+    return lst[ind] + parameter
+
+
+def merge(lst):
+    sleep(0.2)
+    return sum(lst)
+
+
+def return_input_dict(input_dict):
+    return input_dict
+
+
+def raise_error(parameter):
+    raise RuntimeError
+
+
+class TestExecutorWithDependencies(unittest.TestCase):
+    def test_executor(self):
+        with SingleNodeExecutor(max_cores=1) as exe:
+            cloudpickle_register(ind=1)
+            future_1 = exe.submit(add_function, 1, parameter_2=2)
+            future_2 = exe.submit(add_function, 1, parameter_2=future_1)
+            self.assertEqual(future_2.result(), 4)
+
+    def test_dependency_steps(self):
+        cloudpickle_register(ind=1)
+        fs1 = Future()
+        fs2 = Future()
+        q = Queue()
+        q.put(
+            {
+                "fn": add_function,
+                "args": (),
+                "kwargs": {"parameter_1": 1, "parameter_2": 2},
+                "future": fs1,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        q.put(
+            {
+                "fn": add_function,
+                "args": (),
+                "kwargs": {"parameter_1": 1, "parameter_2": fs1},
+                "future": fs2,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        executor = create_single_node_executor(
+            max_workers=1,
+            max_cores=2,
+            resource_dict={
+                "cores": 1,
+                "threads_per_core": 1,
+                "gpus_per_core": 0,
+                "cwd": None,
+                "openmpi_oversubscribe": False,
+                "slurm_cmd_args": [],
+            },
+        )
+        process = Thread(
+            target=execute_tasks_with_dependencies,
+            kwargs={
+                "future_queue": q,
+                "executor_queue": executor._future_queue,
+                "executor": executor,
+                "refresh_rate": 0.01,
+            },
+        )
+        process.start()
+        self.assertFalse(fs1.done())
+        self.assertFalse(fs2.done())
+        self.assertEqual(fs2.result(), 4)
+        self.assertTrue(fs1.done())
+        self.assertTrue(fs2.done())
+        q.put({"shutdown": True, "wait": True})
+
+    def test_dependency_steps_error(self):
+        cloudpickle_register(ind=1)
+        fs1 = Future()
+        fs2 = Future()
+        q = Queue()
+        q.put(
+            {
+                "fn": raise_error,
+                "args": (),
+                "kwargs": {"parameter": 0},
+                "future": fs1,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        q.put(
+            {
+                "fn": add_function,
+                "args": (),
+                "kwargs": {"parameter_1": 1, "parameter_2": fs1},
+                "future": fs2,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        executor = create_single_node_executor(
+            max_workers=1,
+            max_cores=2,
+            resource_dict={
+                "cores": 1,
+                "threads_per_core": 1,
+                "gpus_per_core": 0,
+                "cwd": None,
+                "openmpi_oversubscribe": False,
+                "slurm_cmd_args": [],
+            },
+        )
+        process = Thread(
+            target=execute_tasks_with_dependencies,
+            kwargs={
+                "future_queue": q,
+                "executor_queue": executor._future_queue,
+                "executor": executor,
+                "refresh_rate": 0.01,
+            },
+        )
+        process.start()
+        self.assertFalse(fs1.done())
+        self.assertFalse(fs2.done())
+        self.assertTrue(fs1.exception() is not None)
+        self.assertTrue(fs2.exception() is not None)
+        with self.assertRaises(RuntimeError):
+            fs2.result()
+        q.put({"shutdown": True, "wait": True})
+
+    def test_dependency_steps_error_before(self):
+        cloudpickle_register(ind=1)
+        fs1 = Future()
+        fs1.set_exception(RuntimeError())
+        fs2 = Future()
+        q = Queue()
+        q.put(
+            {
+                "fn": add_function,
+                "args": (),
+                "kwargs": {"parameter_1": 1, "parameter_2": 2},
+                "future": fs1,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        q.put(
+            {
+                "fn": add_function,
+                "args": (),
+                "kwargs": {"parameter_1": 1, "parameter_2": fs1},
+                "future": fs2,
+                "resource_dict": {"cores": 1},
+            }
+        )
+        executor = create_single_node_executor(
+            max_workers=1,
+            max_cores=2,
+            resource_dict={
+                "cores": 1,
+                "threads_per_core": 1,
+                "gpus_per_core": 0,
+                "cwd": None,
+                "openmpi_oversubscribe": False,
+                "slurm_cmd_args": [],
+            },
+        )
+        process = Thread(
+            target=execute_tasks_with_dependencies,
+            kwargs={
+                "future_queue": q,
+                "executor_queue": executor._future_queue,
+                "executor": executor,
+                "refresh_rate": 0.01,
+            },
+        )
+        process.start()
+        self.assertTrue(fs1.exception() is not None)
+        self.assertTrue(fs2.exception() is not None)
+        with self.assertRaises(RuntimeError):
+            fs2.result()
+        executor.shutdown(wait=True)
+        q.put({"shutdown": True, "wait": True})
+        q.join()
+        process.join()
+
+    def test_many_to_one(self):
+        length = 5
+        parameter = 1
+        with SingleNodeExecutor(max_cores=2) as exe:
+            cloudpickle_register(ind=1)
+            future_lst = exe.submit(
+                generate_tasks,
+                length=length,
+                resource_dict={"cores": 1},
+            )
+            lst = []
+            for i in range(length):
+                lst.append(
+                    exe.submit(
+                        calc_from_lst,
+                        lst=future_lst,
+                        ind=i,
+                        parameter=parameter,
+                        resource_dict={"cores": 1},
+                    )
+                )
+            future_sum = exe.submit(
+                merge,
+                lst=lst,
+                resource_dict={"cores": 1},
+            )
+            self.assertEqual(future_sum.result(), 15)
+
+    def test_future_input_dict(self):
+        with SingleNodeExecutor() as exe:
+            fs = exe.submit(
+                return_input_dict,
+                input_dict={"a": exe.submit(sum, [2, 2])},
+            )
+            self.assertEqual(fs.result()["a"], 4)
+
+
+class TestExecutorErrors(unittest.TestCase):
+    def test_block_allocation_false_one_worker(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=1, block_allocation=False) as exe:
+                cloudpickle_register(ind=1)
+                fs = exe.submit(raise_error, parameter=0)
+                fs.result()
+
+    def test_block_allocation_true_one_worker(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe:
+                cloudpickle_register(ind=1)
+                fs = exe.submit(raise_error, parameter=0)
+                fs.result()
+
+    def test_block_allocation_false_two_workers(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=2, block_allocation=False) as exe:
+                cloudpickle_register(ind=1)
+                fs = exe.submit(raise_error, parameter=0)
+                fs.result()
+
+    def test_block_allocation_true_two_workers(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe:
+                cloudpickle_register(ind=1)
+                fs = exe.submit(raise_error, parameter=0)
+                fs.result()
+
+    def test_block_allocation_false_one_worker_loop(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=1, block_allocation=False) as exe:
+                cloudpickle_register(ind=1)
+                lst = []
+                for i in range(1, 4):
+                    lst = exe.submit(
+                        raise_error,
+                        parameter=lst,
+                    )
+                lst.result()
+
+    def test_block_allocation_true_one_worker_loop(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe:
+                cloudpickle_register(ind=1)
+                lst = []
+                for i in range(1, 4):
+                    lst = exe.submit(
+                        raise_error,
+                        parameter=lst,
+                    )
+                lst.result()
+
+    def test_block_allocation_false_two_workers_loop(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=2, block_allocation=False) as exe:
+                cloudpickle_register(ind=1)
+                lst = []
+                for i in range(1, 4):
+                    lst = exe.submit(
+                        raise_error,
+                        parameter=lst,
+                    )
+                lst.result()
+
+    def test_block_allocation_true_two_workers_loop(self):
+        with self.assertRaises(RuntimeError):
+            with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe:
+                cloudpickle_register(ind=1)
+                lst = []
+                for i in range(1, 4):
+                    lst = exe.submit(
+                        raise_error,
+                        parameter=lst,
+                    )
+                lst.result()

{executorlib-0.2.0 → executorlib-0.3.0}/tests/test_local_executor.py

@@ -318,7 +318,8 @@ class TestFuturePool(unittest.TestCase):
                 executor_kwargs={"cores": 1},
                 spawner=MpiExecSpawner,
             ) as p:
-                p.submit(raise_error)
+                fs = p.submit(raise_error)
+                fs.result()
 
     def test_executor_exception_future(self):
         with self.assertRaises(RuntimeError):
@@ -424,6 +425,7 @@ class TestFuturePool(unittest.TestCase):
         f = Future()
         q = Queue()
         q.put({"fn": calc_array, "args": (), "kwargs": {}, "future": f})
+        q.put({"shutdown": True, "wait": True})
         cloudpickle_register(ind=1)
         with self.assertRaises(TypeError):
             execute_parallel_tasks(
@@ -432,12 +434,14 @@ class TestFuturePool(unittest.TestCase):
                 openmpi_oversubscribe=False,
                 spawner=MpiExecSpawner,
             )
+            f.result()
         q.join()
 
     def test_execute_task_failed_wrong_argument(self):
         f = Future()
         q = Queue()
         q.put({"fn": calc_array, "args": (), "kwargs": {"j": 4}, "future": f})
+        q.put({"shutdown": True, "wait": True})
         cloudpickle_register(ind=1)
         with self.assertRaises(TypeError):
             execute_parallel_tasks(
@@ -446,6 +450,7 @@ class TestFuturePool(unittest.TestCase):
                 openmpi_oversubscribe=False,
                 spawner=MpiExecSpawner,
             )
+            f.result()
         q.join()
 
     def test_execute_task(self):

{executorlib-0.2.0 → executorlib-0.3.0}/tests/test_shell_executor.py

@@ -53,6 +53,9 @@ class SubprocessExecutorTest(unittest.TestCase):
                 "future": f,
             }
         )
+        test_queue.put(
+            {"shutdown": True, "wait": True}
+        )
         cloudpickle_register(ind=1)
         with self.assertRaises(TypeError):
             execute_parallel_tasks(
@@ -61,6 +64,7 @@ class SubprocessExecutorTest(unittest.TestCase):
                 openmpi_oversubscribe=False,
                 spawner=MpiExecSpawner,
             )
+            f.result()
 
     def test_broken_executable(self):
         test_queue = queue.Queue()
@@ -73,6 +77,12 @@ class SubprocessExecutorTest(unittest.TestCase):
                 "future": f,
             }
         )
+        test_queue.put(
+            {
+                "shutdown": True,
+                "wait": True,
+            }
+        )
         cloudpickle_register(ind=1)
         with self.assertRaises(FileNotFoundError):
             execute_parallel_tasks(
@@ -81,6 +91,7 @@ class SubprocessExecutorTest(unittest.TestCase):
                 openmpi_oversubscribe=False,
                 spawner=MpiExecSpawner,
             )
+            f.result()
 
     def test_shell_static_executor_args(self):
         with SingleNodeExecutor(max_workers=1) as exe:

executorlib-0.2.0/executorlib/standalone/thread.py

@@ -1,42 +0,0 @@
-from threading import Thread
-
-
-class RaisingThread(Thread):
-    """
-    A subclass of Thread that allows catching exceptions raised in the thread.
-
-    Based on https://stackoverflow.com/questions/2829329/catch-a-threads-exception-in-the-caller-thread
-    """
-
-    def __init__(
-        self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None
-    ):
-        super().__init__(
-            group=group,
-            target=target,
-            name=name,
-            args=args,
-            kwargs=kwargs,
-            daemon=daemon,
-        )
-        self._exception = None
-
-    def get_kwargs(self):
-        return self._kwargs
-
-    def run(self) -> None:
-        """
-        Run the thread's target function and catch any exceptions raised.
-        """
-        try:
-            super().run()
-        except Exception as e:
-            self._exception = e
-
-    def join(self, timeout=None) -> None:
-        """
-        Wait for the thread to complete and re-raise any exceptions caught during execution.
-        """
-        super().join(timeout=timeout)
-        if self._exception:
-            raise self._exception

executorlib-0.2.0/tests/test_dependencies_executor.py

@@ -1,169 +0,0 @@
-from concurrent.futures import Future
-import unittest
-from time import sleep
-from queue import Queue
-
-from executorlib import SingleNodeExecutor
-from executorlib.interfaces.single import create_single_node_executor
-from executorlib.interactive.shared import execute_tasks_with_dependencies
-from executorlib.standalone.serialize import cloudpickle_register
-from executorlib.standalone.thread import RaisingThread
-
-
-try:
-    import pygraphviz
-
-    skip_graphviz_test = False
-except ImportError:
-    skip_graphviz_test = True
-
-
-def add_function(parameter_1, parameter_2):
-    sleep(0.2)
-    return parameter_1 + parameter_2
-
-
-def generate_tasks(length):
-    sleep(0.2)
-    return range(length)
-
-
-def calc_from_lst(lst, ind, parameter):
-    sleep(0.2)
-    return lst[ind] + parameter
-
-
-def merge(lst):
-    sleep(0.2)
-    return sum(lst)
-
-
-def return_input_dict(input_dict):
-    return input_dict
-
-
-def raise_error():
-    raise RuntimeError
-
-
-class TestExecutorWithDependencies(unittest.TestCase):
-    def test_executor(self):
-        with SingleNodeExecutor(max_cores=1) as exe:
-            cloudpickle_register(ind=1)
-            future_1 = exe.submit(add_function, 1, parameter_2=2)
-            future_2 = exe.submit(add_function, 1, parameter_2=future_1)
-            self.assertEqual(future_2.result(), 4)
-
-    def test_dependency_steps(self):
-        cloudpickle_register(ind=1)
-        fs1 = Future()
-        fs2 = Future()
-        q = Queue()
-        q.put(
-            {
-                "fn": add_function,
-                "args": (),
-                "kwargs": {"parameter_1": 1, "parameter_2": 2},
-                "future": fs1,
-                "resource_dict": {"cores": 1},
-            }
-        )
-        q.put(
-            {
-                "fn": add_function,
-                "args": (),
-                "kwargs": {"parameter_1": 1, "parameter_2": fs1},
-                "future": fs2,
-                "resource_dict": {"cores": 1},
-            }
-        )
-        executor = create_single_node_executor(
-            max_workers=1,
-            max_cores=2,
-            resource_dict={
-                "cores": 1,
-                "threads_per_core": 1,
-                "gpus_per_core": 0,
-                "cwd": None,
-                "openmpi_oversubscribe": False,
-                "slurm_cmd_args": [],
-            },
-        )
-        process = RaisingThread(
-            target=execute_tasks_with_dependencies,
-            kwargs={
-                "future_queue": q,
-                "executor_queue": executor._future_queue,
-                "executor": executor,
-                "refresh_rate": 0.01,
-            },
-        )
-        process.start()
-        self.assertFalse(fs1.done())
-        self.assertFalse(fs2.done())
-        self.assertEqual(fs2.result(), 4)
-        self.assertTrue(fs1.done())
-        self.assertTrue(fs2.done())
-        q.put({"shutdown": True, "wait": True})
-
-    def test_many_to_one(self):
-        length = 5
-        parameter = 1
-        with SingleNodeExecutor(max_cores=2) as exe:
-            cloudpickle_register(ind=1)
-            future_lst = exe.submit(
-                generate_tasks,
-                length=length,
-                resource_dict={"cores": 1},
-            )
-            lst = []
-            for i in range(length):
-                lst.append(
-                    exe.submit(
-                        calc_from_lst,
-                        lst=future_lst,
-                        ind=i,
-                        parameter=parameter,
-                        resource_dict={"cores": 1},
-                    )
-                )
-            future_sum = exe.submit(
-                merge,
-                lst=lst,
-                resource_dict={"cores": 1},
-            )
-            self.assertEqual(future_sum.result(), 15)
-
-    def test_future_input_dict(self):
-        with SingleNodeExecutor() as exe:
-            fs = exe.submit(
-                return_input_dict,
-                input_dict={"a": exe.submit(sum, [2, 2])},
-            )
-            self.assertEqual(fs.result()["a"], 4)
-
-
-class TestExecutorErrors(unittest.TestCase):
-    def test_block_allocation_false_one_worker(self):
-        with self.assertRaises(RuntimeError):
-            with SingleNodeExecutor(max_cores=1, block_allocation=False) as exe:
-                cloudpickle_register(ind=1)
-                _ = exe.submit(raise_error)
-
-    def test_block_allocation_true_one_worker(self):
-        with self.assertRaises(RuntimeError):
-            with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe:
-                cloudpickle_register(ind=1)
-                _ = exe.submit(raise_error)
-
-    def test_block_allocation_false_two_workers(self):
-        with self.assertRaises(RuntimeError):
-            with SingleNodeExecutor(max_cores=2, block_allocation=False) as exe:
-                cloudpickle_register(ind=1)
-                _ = exe.submit(raise_error)
-
-    def test_block_allocation_true_two_workers(self):
-        with self.assertRaises(RuntimeError):
-            with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe:
-                cloudpickle_register(ind=1)
-                _ = exe.submit(raise_error)

executorlib-0.2.0/tests/test_shared_thread.py

@@ -1,15 +0,0 @@
-import unittest
-
-from executorlib.standalone.thread import RaisingThread
-
-
-def raise_error():
-    raise ValueError
-
-
-class TestRaisingThread(unittest.TestCase):
-    def test_raising_thread(self):
-        with self.assertRaises(ValueError):
-            process = RaisingThread(target=raise_error)
-            process.start()
-            process.join()