executorlib 1.7.3__tar.gz → 1.8.0__tar.gz

{executorlib-1.7.3 → executorlib-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.7.3
+Version: 1.8.0
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -51,22 +51,22 @@ Requires-Dist: cloudpickle<=3.1.2,>=2.0.0
 Requires-Dist: pyzmq<=27.1.0,>=25.0.0
 Provides-Extra: all
 Requires-Dist: h5py<=3.15.1,>=3.6.0; extra == 'all'
-Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'all'
+Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'all'
 Requires-Dist: mpi4py<=4.1.1,>=3.1.4; extra == 'all'
-Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'all'
+Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.3.3; extra == 'all'
+Requires-Dist: pysqa==0.3.4; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.15.1,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.15.1,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.3.3; extra == 'cluster'
+Requires-Dist: pysqa==0.3.4; extra == 'cluster'
 Provides-Extra: graph
-Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graph'
+Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
 Provides-Extra: graphnotebook
-Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'graphnotebook'
-Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graphnotebook'
+Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'graphnotebook'
+Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graphnotebook'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graphnotebook'
 Provides-Extra: mpi
 Requires-Dist: mpi4py<=4.1.1,>=3.1.4; extra == 'mpi'
@@ -208,6 +208,7 @@ as hierarchical job scheduler within the allocations.
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
   * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
+  * [Advanced Scheduling](https://executorlib.readthedocs.io/en/latest/1-single-node.html#advanced-scheduling)
   * [Testing and Debugging](https://executorlib.readthedocs.io/en/latest/1-single-node.html#testing-and-debugging)
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)

{executorlib-1.7.3 → executorlib-1.8.0}/README.md

@@ -134,6 +134,7 @@ as hierarchical job scheduler within the allocations.
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
   * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
+  * [Advanced Scheduling](https://executorlib.readthedocs.io/en/latest/1-single-node.html#advanced-scheduling)
   * [Testing and Debugging](https://executorlib.readthedocs.io/en/latest/1-single-node.html#testing-and-debugging)
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)

{executorlib-1.7.3 → executorlib-1.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-    "hatchling==1.27.0",
+    "hatchling>=1.27.0,<=1.28.0",
     "hatch-vcs==0.5.0",
     "cloudpickle>=2.0.0,<=3.1.2",
     "pyzmq>=25.0.0,<=27.1.0",
@@ -43,25 +43,25 @@ Repository = "https://github.com/pyiron/executorlib"
 cache = ["h5py>=3.6.0,<=3.15.1"]
 graph = [
     "pygraphviz>=1.10,<=1.14",
-    "networkx>=2.8.8,<=3.4.2",
+    "networkx>=2.8.8,<=3.6.1",
 ]
 graphnotebook = [
     "pygraphviz>=1.10,<=1.14",
-    "networkx>=2.8.8,<=3.4.2",
-    "ipython>=7.33.0,<=9.0.2",
+    "networkx>=2.8.8,<=3.6.1",
+    "ipython>=7.33.0,<=9.9.0",
 ]
 mpi = ["mpi4py>=3.1.4,<=4.1.1"]
 cluster = [
-    "pysqa==0.3.3",
+    "pysqa==0.3.4",
     "h5py>=3.6.0,<=3.15.1",
 ]
 all = [
     "mpi4py>=3.1.4,<=4.1.1",
-    "pysqa==0.3.3",
+    "pysqa==0.3.4",
     "h5py>=3.6.0,<=3.15.1",
     "pygraphviz>=1.10,<=1.14",
-    "networkx>=2.8.8,<=3.4.2",
-    "ipython>=7.33.0,<=9.0.2",
+    "networkx>=2.8.8,<=3.6.1",
+    "ipython>=7.33.0,<=9.9.0",
 ]
 
 [tool.ruff]

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.7.3'
-__version_tuple__ = version_tuple = (1, 7, 3)
+__version__ = version = '1.8.0'
+__version_tuple__ = version_tuple = (1, 8, 0)
 
 __commit_id__ = commit_id = None

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/executor/base.py

@@ -107,6 +107,43 @@ class BaseExecutor(FutureExecutor, ABC):
         else:
             raise RuntimeError("cannot schedule new futures after shutdown")
 
+    def map(
+        self,
+        fn: Callable,
+        *iterables,
+        timeout: Optional[float] = None,
+        chunksize: int = 1,
+    ):
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: The size of the chunks the iterable will be broken into
+                before being passed to a child process. This argument is only
+                used by ProcessPoolExecutor; it is ignored by
+                ThreadPoolExecutor.
+
+        Returns:
+            An iterator equivalent to: map(func, *iterables) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If fn(*args) raises for any values.
+        """
+        if self._is_active:
+            return self._task_scheduler.map(
+                *([fn] + list(iterables)),
+                timeout=timeout,
+                chunksize=chunksize,
+            )
+        else:
+            raise RuntimeError("cannot schedule new futures after shutdown")
+
     def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
         """
         Clean-up the resources associated with the Executor.

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/executor/flux.py

@@ -9,6 +9,7 @@ from executorlib.standalone.inputcheck import (
     check_plot_dependency_graph,
     check_pmi,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.task_scheduler.interactive.blockallocation import (
@@ -65,7 +66,9 @@ class FluxJobExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -105,7 +108,9 @@ class FluxJobExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.FluxJobExecutor leverages either the message passing interface (MPI), the SLURM workload manager
@@ -152,7 +157,9 @@ class FluxJobExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -184,11 +191,13 @@ class FluxJobExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
         else:
@@ -208,6 +217,7 @@ class FluxJobExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -255,7 +265,9 @@ class FluxClusterExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -293,7 +305,9 @@ class FluxClusterExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.FluxClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -338,7 +352,9 @@ class FluxClusterExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -398,6 +414,7 @@ class FluxClusterExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         disable_dependencies=disable_dependencies,
+                        wait=wait,
                     )
                 )
         else:
@@ -420,6 +437,7 @@ class FluxClusterExecutor(BaseExecutor):
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
 
@@ -437,6 +455,7 @@ def create_flux_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a flux executor
@@ -475,6 +494,7 @@ def create_flux_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -496,6 +516,7 @@ def create_flux_executor(
     check_command_line_argument_lst(
         command_line_argument_lst=resource_dict.get("slurm_cmd_args", [])
     )
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if "openmpi_oversubscribe" in resource_dict:
         del resource_dict["openmpi_oversubscribe"]
     if "slurm_cmd_args" in resource_dict:

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/executor/single.py

@@ -7,6 +7,7 @@ from executorlib.standalone.inputcheck import (
     check_init_function,
     check_plot_dependency_graph,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.standalone.interactive.spawner import MpiExecSpawner
@@ -58,7 +59,9 @@ class SingleNodeExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -94,7 +97,9 @@ class SingleNodeExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SingleNodeExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -138,7 +143,9 @@ class SingleNodeExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -166,11 +173,13 @@ class SingleNodeExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
         else:
@@ -186,6 +195,7 @@ class SingleNodeExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -226,7 +236,9 @@ class TestClusterExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -262,7 +274,9 @@ class TestClusterExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.api.TestClusterExecutor is designed to test the file based communication used in the
@@ -299,7 +313,9 @@ class TestClusterExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -339,6 +355,7 @@ class TestClusterExecutor(BaseExecutor):
                     init_function=init_function,
                     disable_dependencies=disable_dependencies,
                     execute_function=execute_in_subprocess,
+                    wait=wait,
                 )
             )
         else:
@@ -358,6 +375,7 @@ class TestClusterExecutor(BaseExecutor):
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
 
@@ -371,6 +389,7 @@ def create_single_node_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a single node executor
@@ -405,6 +424,7 @@ def create_single_node_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -421,6 +441,7 @@ def create_single_node_executor(
     check_command_line_argument_lst(
         command_line_argument_lst=resource_dict.get("slurm_cmd_args", [])
     )
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if "threads_per_core" in resource_dict:
         del resource_dict["threads_per_core"]
     if "gpus_per_core" in resource_dict:

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/executor/slurm.py

@@ -6,6 +6,7 @@ from executorlib.standalone.inputcheck import (
     check_log_obj_size,
     check_plot_dependency_graph,
     check_refresh_rate,
+    check_wait_on_shutdown,
     validate_number_of_cores,
 )
 from executorlib.task_scheduler.interactive.blockallocation import (
@@ -63,7 +64,9 @@ class SlurmClusterExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -101,7 +104,9 @@ class SlurmClusterExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SlurmClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -146,7 +151,9 @@ class SlurmClusterExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -207,6 +214,7 @@ class SlurmClusterExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         disable_dependencies=disable_dependencies,
+                        wait=wait,
                     )
                 )
         else:
@@ -225,6 +233,7 @@ class SlurmClusterExecutor(BaseExecutor):
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
 
@@ -275,7 +284,9 @@ class SlurmJobExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        export_workflow_filename (str): Name of the file to store the exported workflow graph in.
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Examples:
         ```
@@ -312,7 +323,9 @@ class SlurmJobExecutor(BaseExecutor):
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        export_workflow_filename: Optional[str] = None,
         log_obj_size: bool = False,
+        wait: bool = True,
     ):
         """
         The executorlib.SlurmJobExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -360,7 +373,9 @@ class SlurmJobExecutor(BaseExecutor):
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            export_workflow_filename (str): Name of the file to store the exported workflow graph in.
             log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+            wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
         """
         default_resource_dict: dict = {
@@ -389,11 +404,13 @@ class SlurmJobExecutor(BaseExecutor):
                         block_allocation=block_allocation,
                         init_function=init_function,
                         log_obj_size=log_obj_size,
+                        wait=wait,
                     ),
                     max_cores=max_cores,
                     refresh_rate=refresh_rate,
                     plot_dependency_graph=plot_dependency_graph,
                     plot_dependency_graph_filename=plot_dependency_graph_filename,
+                    export_workflow_filename=export_workflow_filename,
                 )
             )
         else:
@@ -410,6 +427,7 @@ class SlurmJobExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     log_obj_size=log_obj_size,
+                    wait=wait,
                 )
             )
 
@@ -424,6 +442,7 @@ def create_slurm_executor(
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
     log_obj_size: bool = False,
+    wait: bool = True,
 ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
     """
     Create a SLURM executor
@@ -463,6 +482,7 @@ def create_slurm_executor(
                                     of the individual function.
         init_function (None): optional function to preset arguments for functions which are submitted later
         log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
+        wait (bool): Whether to wait for the completion of all tasks before shutting down the executor.
 
     Returns:
         InteractiveStepExecutor/ InteractiveExecutor
@@ -475,6 +495,7 @@ def create_slurm_executor(
     resource_dict["log_obj_size"] = log_obj_size
     resource_dict["pmi_mode"] = pmi_mode
     check_init_function(block_allocation=block_allocation, init_function=init_function)
+    check_wait_on_shutdown(wait_on_shutdown=wait)
     if block_allocation:
         resource_dict["init_function"] = init_function
         max_workers = validate_number_of_cores(

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/standalone/hdf.py

@@ -11,6 +11,7 @@ group_dict = {
     "kwargs": "input_kwargs",
     "output": "output",
     "error": "error",
+    "resource_dict": "resource_dict",
     "runtime": "runtime",
     "queue_id": "queue_id",
     "error_log_file": "error_log_file",
@@ -61,6 +62,12 @@ def load(file_name: str) -> dict:
             data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
         else:
             data_dict["kwargs"] = {}
+        if "resource_dict" in hdf:
+            data_dict["resource_dict"] = cloudpickle.loads(
+                np.void(hdf["/resource_dict"])
+            )
+        else:
+            data_dict["resource_dict"] = {}
         if "error_log_file" in hdf:
             data_dict["error_log_file"] = cloudpickle.loads(
                 np.void(hdf["/error_log_file"])

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/standalone/inputcheck.py

@@ -17,6 +17,18 @@ def check_oversubscribe(oversubscribe: bool) -> None:
         )
 
 
+def check_wait_on_shutdown(
+    wait_on_shutdown: bool,
+) -> None:
+    """
+    Check if wait_on_shutdown is False and raise a ValueError if it is.
+    """
+    if not wait_on_shutdown:
+        raise ValueError(
+            "The wait_on_shutdown parameter is only supported for the executorlib.FluxClusterExecutor and executorlib.SlurmClusterExecutor."
+        )
+
+
 def check_command_line_argument_lst(command_line_argument_lst: list[str]) -> None:
     """
     Check if command_line_argument_lst is not empty and raise a ValueError if it is.

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/standalone/interactive/spawner.py

@@ -11,6 +11,7 @@ class BaseSpawner(ABC):
         self,
         cwd: Optional[str] = None,
         cores: int = 1,
+        worker_id: int = 0,
         openmpi_oversubscribe: bool = False,
     ):
         """
@@ -20,9 +21,11 @@ class BaseSpawner(ABC):
             cwd (str): The current working directory.
             cores (int, optional): The number of cores to use. Defaults to 1.
             openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
+            worker_id (int): The worker ID. Defaults to 0.
         """
         self._cwd = cwd
         self._cores = cores
+        self._worker_id = worker_id
         self._openmpi_oversubscribe = openmpi_oversubscribe
 
     @abstractmethod
@@ -69,6 +72,7 @@ class SubprocessSpawner(BaseSpawner):
         self,
         cwd: Optional[str] = None,
         cores: int = 1,
+        worker_id: int = 0,
         openmpi_oversubscribe: bool = False,
         threads_per_core: int = 1,
     ):
@@ -79,11 +83,13 @@ class SubprocessSpawner(BaseSpawner):
             cwd (str, optional): The current working directory. Defaults to None.
             cores (int, optional): The number of cores to use. Defaults to 1.
             threads_per_core (int, optional): The number of threads per core. Defaults to 1.
+            worker_id (int): The worker ID. Defaults to 0.
             openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
         """
         super().__init__(
             cwd=cwd,
             cores=cores,
+            worker_id=worker_id,
             openmpi_oversubscribe=openmpi_oversubscribe,
         )
         self._process: Optional[subprocess.Popen] = None

{executorlib-1.7.3 → executorlib-1.8.0}/src/executorlib/standalone/serialize.py

@@ -75,7 +75,7 @@ def serialize_funct(
                 "kwargs": fn_kwargs,
             }
         )
-        task_key = fn.__name__ + _get_hash(binary=binary_all)
+        task_key = _get_function_name(fn=fn) + _get_hash(binary=binary_all)
     data = {
         "fn": fn,
         "args": fn_args,
@@ -99,3 +99,10 @@ def _get_hash(binary: bytes) -> str:
     # Remove specification of jupyter kernel from hash to be deterministic
     binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)
     return str(hashlib.md5(binary_no_ipykernel).hexdigest())
+
+
+def _get_function_name(fn: Callable) -> str:
+    if hasattr(fn, "__name__"):
+        return fn.__name__
+    else:
+        return str(fn.__class__).split("'")[-2].split(".")[-1]