executorlib 1.7.4__tar.gz → 1.8.1__tar.gz

{executorlib-1.7.4 → executorlib-1.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.7.4
+Version: 1.8.1
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -55,12 +55,12 @@ Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'all'
 Requires-Dist: mpi4py<=4.1.1,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.3.3; extra == 'all'
+Requires-Dist: pysqa==0.3.4; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.15.1,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.15.1,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.3.3; extra == 'cluster'
+Requires-Dist: pysqa==0.3.4; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'

{executorlib-1.7.4 → executorlib-1.8.1}/pyproject.toml

@@ -52,12 +52,12 @@ graphnotebook = [
 ]
 mpi = ["mpi4py>=3.1.4,<=4.1.1"]
 cluster = [
-    "pysqa==0.3.3",
+    "pysqa==0.3.4",
     "h5py>=3.6.0,<=3.15.1",
 ]
 all = [
     "mpi4py>=3.1.4,<=4.1.1",
-    "pysqa==0.3.3",
+    "pysqa==0.3.4",
     "h5py>=3.6.0,<=3.15.1",
     "pygraphviz>=1.10,<=1.14",
     "networkx>=2.8.8,<=3.6.1",

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.7.4'
-__version_tuple__ = version_tuple = (1, 7, 4)
+__version__ = version = '1.8.1'
+__version_tuple__ = version_tuple = (1, 8, 1)
 
 __commit_id__ = commit_id = None

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/executor/flux.py

@@ -9,6 +9,7 @@ from executorlib.standalone.inputcheck import (
     check_plot_dependency_graph,
     check_pmi,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.task_scheduler.interactive.blockallocation import (
@@ -67,6 +68,7 @@ class FluxJobExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -108,6 +110,7 @@ class FluxJobExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.FluxJobExecutor leverages either the message passing interface (MPI), the SLURM workload manager
@@ -156,6 +159,7 @@ class FluxJobExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -187,6 +191,7 @@ class FluxJobExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
@@ -212,6 +217,7 @@ class FluxJobExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -261,6 +267,7 @@ class FluxClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -300,6 +307,7 @@ class FluxClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.FluxClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -346,6 +354,7 @@ class FluxClusterExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -405,6 +414,7 @@ class FluxClusterExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         disable_dependencies=disable_dependencies,
+                        wait=wait,
                     )
                 )
         else:
@@ -445,6 +455,7 @@ def create_flux_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a flux executor
@@ -483,6 +494,7 @@ def create_flux_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -504,6 +516,7 @@ def create_flux_executor(
     check_command_line_argument_lst(
         command_line_argument_lst=resource_dict.get("slurm_cmd_args", [])
     )
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if "openmpi_oversubscribe" in resource_dict:
         del resource_dict["openmpi_oversubscribe"]
     if "slurm_cmd_args" in resource_dict:

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/executor/single.py

@@ -7,6 +7,7 @@ from executorlib.standalone.inputcheck import (
     check_init_function,
     check_plot_dependency_graph,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.standalone.interactive.spawner import MpiExecSpawner
@@ -60,6 +61,7 @@ class SingleNodeExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -97,6 +99,7 @@ class SingleNodeExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SingleNodeExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -142,6 +145,7 @@ class SingleNodeExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -169,6 +173,7 @@ class SingleNodeExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
@@ -190,6 +195,7 @@ class SingleNodeExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -232,6 +238,7 @@ class TestClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -269,6 +276,7 @@ class TestClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.api.TestClusterExecutor is designed to test the file based communication used in the
@@ -307,6 +315,7 @@ class TestClusterExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -346,6 +355,7 @@ class TestClusterExecutor(BaseExecutor):
                     init_function=init_function,
                     disable_dependencies=disable_dependencies,
                     execute_function=execute_in_subprocess,
+                    wait=wait,
                 )
             )
         else:
@@ -379,6 +389,7 @@ def create_single_node_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a single node executor
@@ -413,6 +424,7 @@ def create_single_node_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -429,6 +441,7 @@ def create_single_node_executor(
     check_command_line_argument_lst(
         command_line_argument_lst=resource_dict.get("slurm_cmd_args", [])
     )
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if "threads_per_core" in resource_dict:
         del resource_dict["threads_per_core"]
     if "gpus_per_core" in resource_dict:

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/executor/slurm.py

@@ -6,6 +6,7 @@ from executorlib.standalone.inputcheck import (
     check_log_obj_size,
     check_plot_dependency_graph,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.task_scheduler.interactive.blockallocation import (
@@ -65,6 +66,7 @@ class SlurmClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -104,6 +106,7 @@ class SlurmClusterExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SlurmClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -150,6 +153,7 @@ class SlurmClusterExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -210,6 +214,7 @@ class SlurmClusterExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         disable_dependencies=disable_dependencies,
+                        wait=wait,
                     )
                 )
         else:
@@ -281,6 +286,7 @@ class SlurmJobExecutor(BaseExecutor):
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
         export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -319,6 +325,7 @@ class SlurmJobExecutor(BaseExecutor):
         plot_dependency_graph_filename: Optional[str] = None,
         export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SlurmJobExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -368,6 +375,7 @@ class SlurmJobExecutor(BaseExecutor):
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
             export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -396,6 +404,7 @@ class SlurmJobExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
@@ -418,6 +427,7 @@ class SlurmJobExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -432,6 +442,7 @@ def create_slurm_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a SLURM executor
@@ -471,6 +482,7 @@ def create_slurm_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -483,6 +495,7 @@ def create_slurm_executor(
     resource_dict["log_obj_size"] = log_obj_size
     resource_dict["pmi_mode"] = pmi_mode
     check_init_function(block_allocation=block_allocation, init_function=init_function)
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if block_allocation:
         resource_dict["init_function"] = init_function
         max_workers = validate_number_of_cores(

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/standalone/hdf.py

@@ -11,6 +11,7 @@ group_dict = {
     "kwargs": "input_kwargs",
     "output": "output",
     "error": "error",
+    "resource_dict": "resource_dict",
     "runtime": "runtime",
     "queue_id": "queue_id",
     "error_log_file": "error_log_file",
@@ -61,6 +62,12 @@ def load(file_name: str) -> dict:
             data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
         else:
             data_dict["kwargs"] = {}
+        if "resource_dict" in hdf:
+            data_dict["resource_dict"] = cloudpickle.loads(
+                np.void(hdf["/resource_dict"])
+            )
+        else:
+            data_dict["resource_dict"] = {}
         if "error_log_file" in hdf:
             data_dict["error_log_file"] = cloudpickle.loads(
                 np.void(hdf["/error_log_file"])

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/standalone/inputcheck.py

@@ -17,6 +17,18 @@ def check_oversubscribe(oversubscribe: bool) -> None:
         )
 
 
+def check_wait_on_shutdown(
+    wait_on_shutdown: bool,
+) -> None:
+    """
+    Check if wait_on_shutdown is False and raise a ValueError if it is.
+    """
+    if not wait_on_shutdown:
+        raise ValueError(
+            "The wait_on_shutdown parameter is only supported for the executorlib.FluxClusterExecutor and executorlib.SlurmClusterExecutor."
+        )
+
+
 def check_command_line_argument_lst(command_line_argument_lst: list[str]) -> None:
     """
     Check if command_line_argument_lst is not empty and raise a ValueError if it is.

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/standalone/interactive/spawner.py

@@ -11,6 +11,7 @@ class BaseSpawner(ABC):
         self,
         cwd: Optional[str] = None,
         cores: int = 1,
+        worker_id: int = 0,
         openmpi_oversubscribe: bool = False,
     ):
         """
@@ -20,9 +21,11 @@ class BaseSpawner(ABC):
             cwd (str): The current working directory.
             cores (int, optional): The number of cores to use. Defaults to 1.
             openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
+            worker_id (int): The worker ID. Defaults to 0.
         """
         self._cwd = cwd
         self._cores = cores
+        self._worker_id = worker_id
         self._openmpi_oversubscribe = openmpi_oversubscribe
 
     @abstractmethod
@@ -69,6 +72,7 @@ class SubprocessSpawner(BaseSpawner):
         self,
         cwd: Optional[str] = None,
         cores: int = 1,
+        worker_id: int = 0,
         openmpi_oversubscribe: bool = False,
         threads_per_core: int = 1,
     ):
@@ -79,11 +83,13 @@ class SubprocessSpawner(BaseSpawner):
             cwd (str, optional): The current working directory. Defaults to None.
             cores (int, optional): The number of cores to use. Defaults to 1.
             threads_per_core (int, optional): The number of threads per core. Defaults to 1.
+            worker_id (int): The worker ID. Defaults to 0.
             openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
         """
         super().__init__(
             cwd=cwd,
             cores=cores,
+            worker_id=worker_id,
             openmpi_oversubscribe=openmpi_oversubscribe,
         )
         self._process: Optional[subprocess.Popen] = None
@@ -106,6 +112,7 @@ class SubprocessSpawner(BaseSpawner):
         """
         if self._cwd is not None:
             os.makedirs(self._cwd, exist_ok=True)
+        set_current_directory_in_environment()
         self._process = subprocess.Popen(
             args=self.generate_command(command_lst=command_lst),
             cwd=self._cwd,
@@ -189,3 +196,15 @@ def generate_mpiexec_command(
         if openmpi_oversubscribe:
             command_prepend_lst += ["--oversubscribe"]
         return command_prepend_lst
+
+
+def set_current_directory_in_environment():
+    """
+    Add the current directory to the PYTHONPATH to be able to access local Python modules.
+    """
+    environment = os.environ
+    current_path = os.getcwd()
+    if "PYTHONPATH" in environment and current_path not in environment["PYTHONPATH"]:
+        environment["PYTHONPATH"] = os.getcwd() + ":" + environment["PYTHONPATH"]
+    elif "PYTHONPATH" not in environment:
+        environment["PYTHONPATH"] = os.getcwd()

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/base.py

@@ -198,7 +198,9 @@ class TaskSchedulerBase(FutureExecutor):
         if cancel_futures and self._future_queue is not None:
             cancel_items_in_queue(que=self._future_queue)
         if self._process is not None and self._future_queue is not None:
-            self._future_queue.put({"shutdown": True, "wait": wait})
+            self._future_queue.put(
+                {"shutdown": True, "wait": wait, "cancel_futures": cancel_futures}
+            )
             if wait and isinstance(self._process, Thread):
                 self._process.join()
                 self._future_queue.join()

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/file/shared.py

@@ -57,6 +57,7 @@ def execute_tasks_h5(
     backend: Optional[str] = None,
     disable_dependencies: bool = False,
     pmi_mode: Optional[str] = None,
+    wait: bool = True,
 ) -> None:
     """
     Execute tasks stored in a queue using HDF5 files.
@@ -72,6 +73,7 @@ def execute_tasks_h5(
         backend (str, optional): name of the backend used to spawn tasks.
         disable_dependencies (boolean): Disable resolving future objects during the submission.
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         None
@@ -86,30 +88,35 @@ def execute_tasks_h5(
         with contextlib.suppress(queue.Empty):
             task_dict = future_queue.get_nowait()
         if task_dict is not None and "shutdown" in task_dict and task_dict["shutdown"]:
-            if task_dict["wait"]:
+            if task_dict["wait"] and wait:
                 while len(memory_dict) > 0:
-                    memory_dict = {
-                        key: _check_task_output(
-                            task_key=key,
-                            future_obj=value,
-                            cache_directory=cache_dir_dict[key],
-                        )
-                        for key, value in memory_dict.items()
-                        if not value.done()
-                    }
-            if (
-                terminate_function is not None
-                and terminate_function == terminate_subprocess
-            ):
-                for task in process_dict.values():
-                    terminate_function(task=task)
-            elif terminate_function is not None:
-                for queue_id in process_dict.values():
-                    terminate_function(
-                        queue_id=queue_id,
-                        config_directory=pysqa_config_directory,
+                    memory_dict = _refresh_memory_dict(
+                        memory_dict=memory_dict,
+                        cache_dir_dict=cache_dir_dict,
+                        process_dict=process_dict,
+                        terminate_function=terminate_function,
+                        pysqa_config_directory=pysqa_config_directory,
                         backend=backend,
                     )
+            if not task_dict["cancel_futures"] and wait:
+                _cancel_processes(
+                    process_dict=process_dict,
+                    terminate_function=terminate_function,
+                    pysqa_config_directory=pysqa_config_directory,
+                    backend=backend,
+                )
+            else:
+                memory_dict = _refresh_memory_dict(
+                    memory_dict=memory_dict,
+                    cache_dir_dict=cache_dir_dict,
+                    process_dict=process_dict,
+                    terminate_function=terminate_function,
+                    pysqa_config_directory=pysqa_config_directory,
+                    backend=backend,
+                )
+                for value in memory_dict.values():
+                    if not value.done():
+                        value.cancel()
             future_queue.task_done()
             future_queue.join()
             break
@@ -177,15 +184,14 @@ def execute_tasks_h5(
                 cache_dir_dict[task_key] = cache_directory
             future_queue.task_done()
         else:
-            memory_dict = {
-                key: _check_task_output(
-                    task_key=key,
-                    future_obj=value,
-                    cache_directory=cache_dir_dict[key],
-                )
-                for key, value in memory_dict.items()
-                if not value.done()
-            }
+            memory_dict = _refresh_memory_dict(
+                memory_dict=memory_dict,
+                cache_dir_dict=cache_dir_dict,
+                process_dict=process_dict,
+                terminate_function=terminate_function,
+                pysqa_config_directory=pysqa_config_directory,
+                backend=backend,
+            )
 
 
 def _check_task_output(
@@ -259,3 +265,72 @@ def _convert_args_and_kwargs(
         else:
             task_kwargs[key] = arg
     return task_args, task_kwargs, future_wait_key_lst
+
+
+def _refresh_memory_dict(
+    memory_dict: dict,
+    cache_dir_dict: dict,
+    process_dict: dict,
+    terminate_function: Optional[Callable] = None,
+    pysqa_config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+) -> dict:
+    """
+    Refresh memory dictionary
+
+    Args:
+        memory_dict (dict): dictionary with task keys and future objects
+        cache_dir_dict (dict): dictionary with task keys and cache directories
+        process_dict (dict): dictionary with task keys and process reference.
+        terminate_function (callable): The function to terminate the tasks.
+        pysqa_config_directory (str): path to the pysqa config directory (only for pysqa based backend).
+        backend (str): name of the backend used to spawn tasks.
+
+    Returns:
+        dict: Updated memory dictionary
+    """
+    cancelled_lst = [
+        key for key, value in memory_dict.items() if value.done() and value.cancelled()
+    ]
+    _cancel_processes(
+        process_dict={k: v for k, v in process_dict.items() if k in cancelled_lst},
+        terminate_function=terminate_function,
+        pysqa_config_directory=pysqa_config_directory,
+        backend=backend,
+    )
+    return {
+        key: _check_task_output(
+            task_key=key,
+            future_obj=value,
+            cache_directory=cache_dir_dict[key],
+        )
+        for key, value in memory_dict.items()
+        if not value.done()
+    }
+
+
+def _cancel_processes(
+    process_dict: dict,
+    terminate_function: Optional[Callable] = None,
+    pysqa_config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+):
+    """
+    Cancel processes
+
+    Args:
+        process_dict (dict): dictionary with task keys and process reference.
+        terminate_function (callable): The function to terminate the tasks.
+        pysqa_config_directory (str): path to the pysqa config directory (only for pysqa based backend).
+        backend (str): name of the backend used to spawn tasks.
+    """
+    if terminate_function is not None and terminate_function == terminate_subprocess:
+        for task in process_dict.values():
+            terminate_function(task=task)
+    elif terminate_function is not None and backend is not None:
+        for queue_id in process_dict.values():
+            terminate_function(
+                queue_id=queue_id,
+                config_directory=pysqa_config_directory,
+                backend=backend,
+            )

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/file/spawner_pysqa.py

@@ -5,6 +5,9 @@ from pysqa import QueueAdapter
 
 from executorlib.standalone.hdf import dump, get_queue_id
 from executorlib.standalone.inputcheck import check_file_exists
+from executorlib.standalone.interactive.spawner import (
+    set_current_directory_in_environment,
+)
 from executorlib.standalone.scheduler import pysqa_execute_command, terminate_with_pysqa
 
 
@@ -85,6 +88,7 @@ def execute_with_pysqa(
                 os.path.dirname(os.path.abspath(cwd))
             )
         submit_kwargs.update(resource_dict)
+        set_current_directory_in_environment()
         queue_id = qa.submit_job(**submit_kwargs)
         dump(file_name=file_name, data_dict={"queue_id": queue_id})
     return queue_id

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/file/spawner_subprocess.py

@@ -5,6 +5,9 @@ from typing import Optional
 
 from executorlib.standalone.hdf import dump
 from executorlib.standalone.inputcheck import check_file_exists
+from executorlib.standalone.interactive.spawner import (
+    set_current_directory_in_environment,
+)
 
 
 def execute_in_subprocess(
@@ -53,11 +56,12 @@ def execute_in_subprocess(
         )
     if backend is not None:
         raise ValueError("backend parameter is not supported for subprocess spawner.")
-    if resource_dict is None:
-        resource_dict = {}
-    cwd = resource_dict.get("cwd", cache_directory)
+    cwd = _get_working_directory(
+        cache_directory=cache_directory, resource_dict=resource_dict
+    )
     if cwd is not None:
         os.makedirs(cwd, exist_ok=True)
+    set_current_directory_in_environment()
     return subprocess.Popen(command, universal_newlines=True, cwd=cwd)
 
 
@@ -71,3 +75,14 @@ def terminate_subprocess(task):
     task.terminate()
     while task.poll() is None:
         time.sleep(0.1)
+
+
+def _get_working_directory(
+    cache_directory: Optional[str] = None, resource_dict: Optional[dict] = None
+):
+    if resource_dict is None:
+        resource_dict = {}
+    if "cwd" in resource_dict and resource_dict["cwd"] is not None:
+        return resource_dict["cwd"]
+    else:
+        return cache_directory

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/file/task_scheduler.py

@@ -35,6 +35,7 @@ class FileTaskScheduler(TaskSchedulerBase):
         backend: Optional[str] = None,
         disable_dependencies: bool = False,
         pmi_mode: Optional[str] = None,
+        wait: bool = True,
     ):
         """
         Initialize the FileExecutor.
@@ -50,6 +51,7 @@ class FileTaskScheduler(TaskSchedulerBase):
             backend (str, optional): name of the backend used to spawn tasks.
             disable_dependencies (boolean): Disable resolving future objects during the submission.
             pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
         """
         super().__init__(max_cores=None)
         default_resource_dict = {
@@ -73,6 +75,7 @@ class FileTaskScheduler(TaskSchedulerBase):
             "backend": backend,
             "disable_dependencies": disable_dependencies,
             "pmi_mode": pmi_mode,
+            "wait": wait,
         }
         self._set_process(
             Thread(
@@ -98,6 +101,7 @@ def create_file_executor(
     init_function: Optional[Callable] = None,
     disable_dependencies: bool = False,
     execute_function: Callable = execute_with_pysqa,
+    wait: bool = True,
 ):
     if block_allocation:
         raise ValueError(
@@ -128,4 +132,5 @@ def create_file_executor(
         execute_function=execute_function,
         terminate_function=terminate_function,
         pmi_mode=pmi_mode,
+        wait=wait,
     )

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/interactive/blockallocation.py

@@ -208,7 +208,7 @@ def _execute_multiple_tasks(
     queue_join_on_shutdown: bool = True,
     log_obj_size: bool = False,
     error_log_file: Optional[str] = None,
-    worker_id: Optional[int] = None,
+    worker_id: int = 0,
     stop_function: Optional[Callable] = None,
     restart_limit: int = 0,
     **kwargs,
@@ -244,7 +244,7 @@ def _execute_multiple_tasks(
         command_lst=get_interactive_execute_command(
             cores=cores,
         ),
-        connections=spawner(cores=cores, **kwargs),
+        connections=spawner(cores=cores, worker_id=worker_id, **kwargs),
         hostname_localhost=hostname_localhost,
         log_obj_size=log_obj_size,
         worker_id=worker_id,

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/interactive/dependency_plot.py

@@ -5,7 +5,6 @@ from concurrent.futures import Future
 from typing import Optional
 
 import cloudpickle
-import numpy as np
 
 from executorlib.standalone.select import FutureSelector
 
@@ -219,7 +218,11 @@ def plot_dependency_graph_function(
     graph = nx.DiGraph()
     for node in node_lst:
         if node["type"] == "input":
-            graph.add_node(node["id"], label=str(node["value"]), shape=node["shape"])
+            graph.add_node(
+                node["id"],
+                label=_short_object_name(node=node["value"]),
+                shape=node["shape"],
+            )
         else:
             graph.add_node(node["id"], label=str(node["name"]), shape=node["shape"])
     for edge in edge_lst:
@@ -245,6 +248,8 @@ def export_dependency_graph_function(
         edge_lst (list): List of edges.
         file_name (str): Name of the file to store the exported graph in.
     """
+    import numpy as np
+
     pwd_nodes_lst = []
     for n in node_lst:
         if n["type"] == "function":
@@ -305,3 +310,31 @@ def export_dependency_graph_function(
     }
     with open(file_name, "w") as f:
         json.dump(pwd_dict, f, indent=4)
+
+
+def _short_object_name(node):
+    node_value_str = str(node)
+    if isinstance(node, tuple):
+        short_name = str(tuple(_short_object_name(node=el) for el in node))
+    elif isinstance(node, list):
+        short_name = str([_short_object_name(node=el) for el in node])
+    elif isinstance(node, dict):
+        short_name = str(
+            {
+                _short_object_name(node=key): _short_object_name(node=value)
+                for key, value in node.items()
+            }
+        )
+    elif "object at" in node_value_str:
+        short_name = node_value_str[1:-1].split(maxsplit=1)[0].split(".")[-1] + "()"
+    elif "<function" in node_value_str:
+        short_name = node_value_str.split()[1] + "()"
+    elif "\n" in node_value_str:
+        short_name = str(type(node)).split("'")[1].split(".")[-1] + "()"
+    elif "(" in node_value_str and ")" in node_value_str:
+        short_name = node_value_str.split("(", maxsplit=1)[0] + "()"
+    elif len(node_value_str) > 20:
+        short_name = node_value_str[:21] + "..."
+    else:
+        short_name = node_value_str
+    return short_name

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/interactive/spawner_flux.py

@@ -5,7 +5,10 @@ from typing import Callable, Optional
 import flux
 import flux.job
 
-from executorlib.standalone.interactive.spawner import BaseSpawner
+from executorlib.standalone.interactive.spawner import (
+    BaseSpawner,
+    set_current_directory_in_environment,
+)
 
 
 def validate_max_workers(max_workers: int, cores: int, threads_per_core: int):
@@ -31,6 +34,7 @@ class FluxPythonSpawner(BaseSpawner):
         threads_per_core (int, optional): The number of threads per base. Defaults to 1.
         gpus_per_core (int, optional): The number of GPUs per base. Defaults to 0.
         num_nodes (int, optional): The number of compute nodes to use for executing the task. Defaults to None.
+        worker_id (int): The worker ID. Defaults to 0.
         exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults to
                           False.
         openmpi_oversubscribe (bool, optional): Whether to oversubscribe. Defaults to False.
@@ -49,6 +53,7 @@ class FluxPythonSpawner(BaseSpawner):
         threads_per_core: int = 1,
         gpus_per_core: int = 0,
         num_nodes: Optional[int] = None,
+        worker_id: int = 0,
         exclusive: bool = False,
         priority: Optional[int] = None,
         openmpi_oversubscribe: bool = False,
@@ -60,6 +65,7 @@ class FluxPythonSpawner(BaseSpawner):
         super().__init__(
             cwd=cwd,
             cores=cores,
+            worker_id=worker_id,
             openmpi_oversubscribe=openmpi_oversubscribe,
         )
         self._threads_per_core = threads_per_core
@@ -115,18 +121,20 @@ class FluxPythonSpawner(BaseSpawner):
                 num_nodes=self._num_nodes,
                 exclusive=self._exclusive,
             )
+        set_current_directory_in_environment()
         jobspec.environment = dict(os.environ)
         if self._pmi_mode is not None:
             jobspec.setattr_shell_option("pmi", self._pmi_mode)
         if self._cwd is not None:
             jobspec.cwd = self._cwd
             os.makedirs(self._cwd, exist_ok=True)
+        file_prefix = "flux_" + str(self._worker_id)
         if self._flux_log_files and self._cwd is not None:
-            jobspec.stderr = os.path.join(self._cwd, "flux.err")
-            jobspec.stdout = os.path.join(self._cwd, "flux.out")
+            jobspec.stderr = os.path.join(self._cwd, file_prefix + ".err")
+            jobspec.stdout = os.path.join(self._cwd, file_prefix + ".out")
         elif self._flux_log_files:
-            jobspec.stderr = os.path.abspath("flux.err")
-            jobspec.stdout = os.path.abspath("flux.out")
+            jobspec.stderr = os.path.abspath(file_prefix + ".err")
+            jobspec.stdout = os.path.abspath(file_prefix + ".out")
         if self._priority is not None:
             self._future = self._flux_executor.submit(
                 jobspec=jobspec, urgency=self._priority

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -6,7 +6,10 @@ from typing import Callable, Optional
 from pysqa import QueueAdapter
 
 from executorlib.standalone.inputcheck import validate_number_of_cores
-from executorlib.standalone.interactive.spawner import BaseSpawner
+from executorlib.standalone.interactive.spawner import (
+    BaseSpawner,
+    set_current_directory_in_environment,
+)
 from executorlib.standalone.scheduler import pysqa_execute_command, terminate_with_pysqa
 from executorlib.task_scheduler.interactive.blockallocation import (
     BlockAllocationTaskScheduler,
@@ -21,6 +24,7 @@ class PysqaSpawner(BaseSpawner):
         threads_per_core: int = 1,
         gpus_per_core: int = 0,
         num_nodes: Optional[int] = None,
+        worker_id: int = 0,
         exclusive: bool = False,
         openmpi_oversubscribe: bool = False,
         slurm_cmd_args: Optional[list[str]] = None,
@@ -38,6 +42,7 @@ class PysqaSpawner(BaseSpawner):
             threads_per_core (int): The number of threads per core. Defaults to 1.
             gpus_per_core (int): number of GPUs per worker - defaults to 0
             num_nodes (int, optional): The number of compute nodes to use for executing the task.  Defaults to None.
+            worker_id (int): The worker ID. Defaults to 0.
             exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults
                               to False.
             openmpi_oversubscribe (bool): Whether to oversubscribe the cores. Defaults to False.
@@ -49,6 +54,7 @@ class PysqaSpawner(BaseSpawner):
         super().__init__(
             cwd=cwd,
             cores=cores,
+            worker_id=worker_id,
             openmpi_oversubscribe=openmpi_oversubscribe,
         )
         self._threads_per_core = threads_per_core
@@ -180,6 +186,7 @@ class PysqaSpawner(BaseSpawner):
             working_directory = os.path.join(self._cwd, hash)
         else:
             working_directory = os.path.abspath(hash)
+        set_current_directory_in_environment()
         return queue_adapter.submit_job(
             command=" ".join(self.generate_command(command_lst=command_lst)),
             working_directory=working_directory,

{executorlib-1.7.4 → executorlib-1.8.1}/src/executorlib/task_scheduler/interactive/spawner_slurm.py

@@ -27,6 +27,7 @@ class SrunSpawner(SubprocessSpawner):
         threads_per_core: int = 1,
         gpus_per_core: int = 0,
         num_nodes: Optional[int] = None,
+        worker_id: int = 0,
         exclusive: bool = False,
         openmpi_oversubscribe: bool = False,
         slurm_cmd_args: Optional[list[str]] = None,
@@ -41,6 +42,7 @@ class SrunSpawner(SubprocessSpawner):
             threads_per_core (int, optional): The number of threads per core. Defaults to 1.
             gpus_per_core (int, optional): The number of GPUs per core. Defaults to 0.
             num_nodes (int, optional): The number of compute nodes to use for executing the task. Defaults to None.
+            worker_id (int): The worker ID. Defaults to 0.
             exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults to False.
             openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
             slurm_cmd_args (list[str], optional): Additional command line arguments. Defaults to [].
@@ -49,6 +51,7 @@ class SrunSpawner(SubprocessSpawner):
         super().__init__(
             cwd=cwd,
             cores=cores,
+            worker_id=worker_id,
             openmpi_oversubscribe=openmpi_oversubscribe,
             threads_per_core=threads_per_core,
         )