executorlib 1.6.0__tar.gz → 1.6.2__tar.gz

{executorlib-1.6.0 → executorlib-1.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.6.0
+Version: 1.6.2
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -48,19 +48,19 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Physics
 Requires-Python: <3.14,>3.9
 Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
-Requires-Dist: pyzmq<=27.0.0,>=25.0.0
+Requires-Dist: pyzmq<=27.0.2,>=25.0.0
 Provides-Extra: all
 Requires-Dist: h5py<=3.14.0,>=3.6.0; extra == 'all'
 Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'all'
 Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.2.7; extra == 'all'
+Requires-Dist: pysqa==0.3.1; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.14.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.14.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.2.7; extra == 'cluster'
+Requires-Dist: pysqa==0.3.1; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
@@ -208,6 +208,7 @@ as hierarchical job scheduler within the allocations.
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
   * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
+  * [Testing and Debugging](https://executorlib.readthedocs.io/en/latest/1-single-node.html#testing-and-debugging)
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)

{executorlib-1.6.0 → executorlib-1.6.2}/README.md

@@ -134,6 +134,7 @@ as hierarchical job scheduler within the allocations.
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
   * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
+  * [Testing and Debugging](https://executorlib.readthedocs.io/en/latest/1-single-node.html#testing-and-debugging)
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/__init__.py

@@ -12,6 +12,9 @@ Finally, the get_cache_data() function allows users to cache the content of thei
 pandas.DataFrame.
 """
 
+from typing import Optional
+
+import executorlib._version
 from executorlib.executor.base import BaseExecutor
 from executorlib.executor.flux import (
     FluxClusterExecutor,
@@ -22,12 +25,48 @@ from executorlib.executor.slurm import (
     SlurmClusterExecutor,
     SlurmJobExecutor,
 )
-from executorlib.standalone.cache import get_cache_data
 
-from . import _version
+
+def get_cache_data(cache_directory: str) -> list[dict]:
+    """
+    Collect all HDF5 files in the cache directory
+
+    Args:
+        cache_directory (str): The directory to store cache files.
+
+    Returns:
+        list[dict]: List of dictionaries each representing on of the HDF5 files in the cache directory.
+    """
+    from executorlib.standalone.hdf import get_cache_data
+
+    return get_cache_data(cache_directory=cache_directory)
+
+
+def terminate_tasks_in_cache(
+    cache_directory: str,
+    config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+):
+    """
+    Delete all jobs stored in the cache directory from the queuing system
+
+    Args:
+        cache_directory (str): The directory to store cache files.
+        config_directory (str, optional): path to the config directory.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+    """
+    from executorlib.task_scheduler.file.queue_spawner import terminate_tasks_in_cache
+
+    return terminate_tasks_in_cache(
+        cache_directory=cache_directory,
+        config_directory=config_directory,
+        backend=backend,
+    )
+
 
 __all__: list[str] = [
     "get_cache_data",
+    "terminate_tasks_in_cache",
     "BaseExecutor",
     "FluxJobExecutor",
     "FluxClusterExecutor",
@@ -36,11 +75,4 @@ __all__: list[str] = [
     "SlurmClusterExecutor",
 ]
 
-try:
-    from executorlib.task_scheduler.file.queue_spawner import terminate_tasks_in_cache
-
-    __all__ += ["terminate_tasks_in_cache"]
-except ImportError:
-    pass
-
-__version__ = _version.__version__
+__version__ = executorlib._version.__version__

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '1.6.0'
-__version_tuple__ = version_tuple = (1, 6, 0)
+__version__ = version = '1.6.2'
+__version_tuple__ = version_tuple = (1, 6, 2)
+
+__commit_id__ = commit_id = None

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/backend/interactive_parallel.py

@@ -43,7 +43,7 @@ def main() -> None:
             host=argument_dict["host"], port=argument_dict["zmqport"]
         )
 
-    memory = None
+    memory = {"executorlib_worker_id": int(argument_dict["worker_id"])}
 
     # required for flux interface - otherwise the current path is not included in the python path
     cwd = abspath(".")
@@ -97,7 +97,7 @@ def main() -> None:
             and "args" in input_dict
             and "kwargs" in input_dict
         ):
-            memory = call_funct(input_dict=input_dict, funct=None)
+            memory.update(call_funct(input_dict=input_dict, funct=None, memory=memory))
 
 
 if __name__ == "__main__":

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/backend/interactive_serial.py

@@ -29,7 +29,7 @@ def main(argument_lst: Optional[list[str]] = None):
         host=argument_dict["host"], port=argument_dict["zmqport"]
     )
 
-    memory = None
+    memory = {"executorlib_worker_id": int(argument_dict["worker_id"])}
 
     # required for flux interface - otherwise the current path is not included in the python path
     cwd = abspath(".")
@@ -72,7 +72,7 @@ def main(argument_lst: Optional[list[str]] = None):
             and "args" in input_dict
             and "kwargs" in input_dict
         ):
-            memory = call_funct(input_dict=input_dict, funct=None)
+            memory.update(call_funct(input_dict=input_dict, funct=None, memory=memory))
 
 
 if __name__ == "__main__":

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/executor/base.py

@@ -50,6 +50,23 @@ class BaseExecutor(FutureExecutor, ABC):
         """
         return self._task_scheduler.future_queue
 
+    def batched(
+        self,
+        iterable: list[Future],
+        n: int,
+    ) -> list[Future]:
+        """
+        Batch futures from the iterable into tuples of length n. The last batch may be shorter than n.
+
+        Args:
+            iterable (list): list of future objects to batch based on which future objects finish first
+            n (int): badge size
+
+        Returns:
+            list[Future]: list of future objects one for each batch
+        """
+        return self._task_scheduler.batched(iterable=iterable, n=n)
+
     def submit(  # type: ignore
         self,
         fn: Callable,
@@ -68,16 +85,16 @@ class BaseExecutor(FutureExecutor, ABC):
             fn (callable): function to submit for execution
             args: arguments for the submitted function
             kwargs: keyword arguments for the submitted function
-            resource_dict (dict): resource dictionary, which defines the resources used for the execution of the
-                                  function. Example resource dictionary: {
-                                      cores: 1,
-                                      threads_per_core: 1,
-                                      gpus_per_worker: 0,
-                                      oversubscribe: False,
-                                      cwd: None,
-                                      executor: None,
-                                      hostname_localhost: False,
-                                  }
+            resource_dict (dict): A dictionary of resources required by the task. With the following keys:
+                              - cores (int): number of MPI cores to be used for each function call
+                              - threads_per_core (int): number of OpenMP threads to be used for each function call
+                              - gpus_per_core (int): number of GPUs per worker - defaults to 0
+                              - cwd (str/None): current working directory where the parallel python task is executed
+                              - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and
+                                                              SLURM only) - default False
+                              - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              - error_log_file (str): Name of the error log file to use for storing exceptions raised
+                                                      by the Python functions submitted to the Executor.
 
         Returns:
             Future: A Future representing the given call.

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/executor/flux.py

@@ -43,8 +43,8 @@ class FluxJobExecutor(BaseExecutor):
                                                   compute notes. Defaults to False.
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
-        flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
         flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False.
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -93,8 +93,8 @@ class FluxJobExecutor(BaseExecutor):
         cache_directory: Optional[str] = None,
         max_cores: Optional[int] = None,
         resource_dict: Optional[dict] = None,
+        pmi_mode: Optional[str] = None,
         flux_executor=None,
-        flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
         flux_log_files: bool = False,
         hostname_localhost: Optional[bool] = None,
@@ -130,8 +130,8 @@ class FluxJobExecutor(BaseExecutor):
                                                       compute notes. Defaults to False.
                                   - error_log_file (str): Name of the error log file to use for storing exceptions
                                                           raised by the Python functions submitted to the Executor.
+            pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
             flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
-            flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
             flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
             flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False.
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -175,8 +175,8 @@ class FluxJobExecutor(BaseExecutor):
                         cache_directory=cache_directory,
                         max_cores=max_cores,
                         resource_dict=resource_dict,
+                        pmi_mode=pmi_mode,
                         flux_executor=flux_executor,
-                        flux_executor_pmi_mode=flux_executor_pmi_mode,
                         flux_executor_nesting=flux_executor_nesting,
                         flux_log_files=flux_log_files,
                         hostname_localhost=hostname_localhost,
@@ -199,8 +199,8 @@ class FluxJobExecutor(BaseExecutor):
                     cache_directory=cache_directory,
                     max_cores=max_cores,
                     resource_dict=resource_dict,
+                    pmi_mode=pmi_mode,
                     flux_executor=flux_executor,
-                    flux_executor_pmi_mode=flux_executor_pmi_mode,
                     flux_executor_nesting=flux_executor_nesting,
                     flux_log_files=flux_log_files,
                     hostname_localhost=hostname_localhost,
@@ -236,6 +236,7 @@ class FluxClusterExecutor(BaseExecutor):
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -283,6 +284,7 @@ class FluxClusterExecutor(BaseExecutor):
         max_cores: Optional[int] = None,
         resource_dict: Optional[dict] = None,
         pysqa_config_directory: Optional[str] = None,
+        pmi_mode: Optional[str] = None,
         hostname_localhost: Optional[bool] = None,
         block_allocation: bool = False,
         init_function: Optional[Callable] = None,
@@ -317,6 +319,7 @@ class FluxClusterExecutor(BaseExecutor):
                                   - error_log_file (str): Name of the error log file to use for storing exceptions
                                                           raised by the Python functions submitted to the Executor.
             pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
+            pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -366,7 +369,7 @@ class FluxClusterExecutor(BaseExecutor):
                     cache_directory=cache_directory,
                     resource_dict=resource_dict,
                     flux_executor=None,
-                    flux_executor_pmi_mode=None,
+                    pmi_mode=pmi_mode,
                     flux_executor_nesting=False,
                     flux_log_files=False,
                     pysqa_config_directory=pysqa_config_directory,
@@ -384,8 +387,8 @@ class FluxClusterExecutor(BaseExecutor):
                         cache_directory=cache_directory,
                         max_cores=max_cores,
                         resource_dict=resource_dict,
+                        pmi_mode=None,
                         flux_executor=None,
-                        flux_executor_pmi_mode=None,
                         flux_executor_nesting=False,
                         flux_log_files=False,
                         hostname_localhost=hostname_localhost,
@@ -405,8 +408,8 @@ def create_flux_executor(
     max_cores: Optional[int] = None,
     cache_directory: Optional[str] = None,
     resource_dict: Optional[dict] = None,
+    pmi_mode: Optional[str] = None,
     flux_executor=None,
-    flux_executor_pmi_mode: Optional[str] = None,
     flux_executor_nesting: bool = False,
     flux_log_files: bool = False,
     hostname_localhost: Optional[bool] = None,
@@ -434,8 +437,8 @@ def create_flux_executor(
                                                   compute notes. Defaults to False.
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
-        flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
         flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False.
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -467,7 +470,7 @@ def create_flux_executor(
     resource_dict["hostname_localhost"] = hostname_localhost
     resource_dict["log_obj_size"] = log_obj_size
     check_init_function(block_allocation=block_allocation, init_function=init_function)
-    check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode)
+    check_pmi(backend="flux_allocation", pmi=pmi_mode)
     check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False))
     check_command_line_argument_lst(
         command_line_argument_lst=resource_dict.get("slurm_cmd_args", [])
@@ -476,8 +479,8 @@ def create_flux_executor(
         del resource_dict["openmpi_oversubscribe"]
     if "slurm_cmd_args" in resource_dict:
         del resource_dict["slurm_cmd_args"]
+    resource_dict["pmi_mode"] = pmi_mode
     resource_dict["flux_executor"] = flux_executor
-    resource_dict["flux_executor_pmi_mode"] = flux_executor_pmi_mode
     resource_dict["flux_executor_nesting"] = flux_executor_nesting
     resource_dict["flux_log_files"] = flux_log_files
     if block_allocation:

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/executor/single.py

@@ -329,7 +329,7 @@ class TestClusterExecutor(BaseExecutor):
                     cache_directory=cache_directory,
                     resource_dict=resource_dict,
                     flux_executor=None,
-                    flux_executor_pmi_mode=None,
+                    pmi_mode=None,
                     flux_executor_nesting=False,
                     flux_log_files=False,
                     pysqa_config_directory=None,

{executorlib-1.6.0 → executorlib-1.6.2}/executorlib/executor/slurm.py

@@ -44,6 +44,7 @@ class SlurmClusterExecutor(BaseExecutor):
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -91,6 +92,7 @@ class SlurmClusterExecutor(BaseExecutor):
         max_cores: Optional[int] = None,
         resource_dict: Optional[dict] = None,
         pysqa_config_directory: Optional[str] = None,
+        pmi_mode: Optional[str] = None,
         hostname_localhost: Optional[bool] = None,
         block_allocation: bool = False,
         init_function: Optional[Callable] = None,
@@ -125,6 +127,7 @@ class SlurmClusterExecutor(BaseExecutor):
                                   - error_log_file (str): Name of the error log file to use for storing exceptions
                                                           raised by the Python functions submitted to the Executor.
             pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
+            pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -173,8 +176,8 @@ class SlurmClusterExecutor(BaseExecutor):
                     max_cores=max_cores,
                     cache_directory=cache_directory,
                     resource_dict=resource_dict,
+                    pmi_mode=pmi_mode,
                     flux_executor=None,
-                    flux_executor_pmi_mode=None,
                     flux_executor_nesting=False,
                     flux_log_files=False,
                     pysqa_config_directory=pysqa_config_directory,
@@ -232,6 +235,7 @@ class SlurmJobExecutor(BaseExecutor):
                                                   compute notes. Defaults to False.
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -278,6 +282,7 @@ class SlurmJobExecutor(BaseExecutor):
         cache_directory: Optional[str] = None,
         max_cores: Optional[int] = None,
         resource_dict: Optional[dict] = None,
+        pmi_mode: Optional[str] = None,
         hostname_localhost: Optional[bool] = None,
         block_allocation: bool = False,
         init_function: Optional[Callable] = None,
@@ -315,6 +320,7 @@ class SlurmJobExecutor(BaseExecutor):
                                                       compute notes. Defaults to False.
                                   - error_log_file (str): Name of the error log file to use for storing exceptions
                                                           raised by the Python functions submitted to the Executor.
+            pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -356,6 +362,7 @@ class SlurmJobExecutor(BaseExecutor):
                         cache_directory=cache_directory,
                         max_cores=max_cores,
                         resource_dict=resource_dict,
+                        pmi_mode=pmi_mode,
                         hostname_localhost=hostname_localhost,
                         block_allocation=block_allocation,
                         init_function=init_function,
@@ -376,6 +383,7 @@ class SlurmJobExecutor(BaseExecutor):
                     cache_directory=cache_directory,
                     max_cores=max_cores,
                     resource_dict=resource_dict,
+                    pmi_mode=pmi_mode,
                     hostname_localhost=hostname_localhost,
                     block_allocation=block_allocation,
                     init_function=init_function,
@@ -389,6 +397,7 @@ def create_slurm_executor(
     max_cores: Optional[int] = None,
     cache_directory: Optional[str] = None,
     resource_dict: Optional[dict] = None,
+    pmi_mode: Optional[str] = None,
     hostname_localhost: Optional[bool] = None,
     block_allocation: bool = False,
     init_function: Optional[Callable] = None,
@@ -418,6 +427,7 @@ def create_slurm_executor(
                                                   compute notes. Defaults to False.
                               - error_log_file (str): Name of the error log file to use for storing exceptions raised
                                                       by the Python functions submitted to the Executor.
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                   context of an HPC cluster this essential to be able to communicate to an
                                   Executor running on a different compute node within the same allocation. And
@@ -441,6 +451,7 @@ def create_slurm_executor(
     resource_dict["cache_directory"] = cache_directory
     resource_dict["hostname_localhost"] = hostname_localhost
     resource_dict["log_obj_size"] = log_obj_size
+    resource_dict["pmi_mode"] = pmi_mode
     check_init_function(block_allocation=block_allocation, init_function=init_function)
     if block_allocation:
         resource_dict["init_function"] = init_function

executorlib-1.6.2/executorlib/standalone/batched.py

@@ -0,0 +1,27 @@
+from concurrent.futures import Future
+
+
+def batched_futures(lst: list[Future], skip_lst: list[list], n: int) -> list[list]:
+    """
+    Batch n completed future objects. If the number of completed futures is smaller than n and the end of the batch is
+    not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_lst
+    then they are returned as batch.
+
+    Args:
+        lst (list): list of all future objects
+        skip_lst (list): list of previous batches of future objects
+        n (int): batch size
+
+    Returns:
+        list: results of the batched futures
+    """
+    skipped_elements_lst = [item for items in skip_lst for item in items]
+
+    done_lst = []
+    n_expected = min(n, len(lst) - len(skipped_elements_lst))
+    for v in lst:
+        if v.done() and v.result() not in skipped_elements_lst:
+            done_lst.append(v.result())
+        if len(done_lst) == n_expected:
+            return done_lst
+    return []

executorlib-1.6.2/executorlib/standalone/command.py

@@ -0,0 +1,114 @@
+import importlib.util
+import os
+import sys
+from typing import Optional
+
+
+def get_command_path(executable: str) -> str:
+    """
+    Get path of the backend executable script
+
+    Args:
+        executable (str): Name of the backend executable script, either mpiexec.py or serial.py
+
+    Returns:
+        str: absolute path to the executable script
+    """
+    return os.path.abspath(os.path.join(__file__, "..", "..", "backend", executable))
+
+
+def get_cache_execute_command(
+    file_name: str,
+    cores: int = 1,
+    backend: Optional[str] = None,
+    exclusive: bool = False,
+    openmpi_oversubscribe: bool = False,
+    pmi_mode: Optional[str] = None,
+) -> list:
+    """
+    Get command to call backend as a list of two strings
+
+    Args:
+        file_name (str): The name of the file.
+        cores (int, optional): Number of cores used to execute the task. Defaults to 1.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+        exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults to False.
+        openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
+        pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
+
+    Returns:
+        list[str]: List of strings containing the python executable path and the backend script to execute
+    """
+    command_lst = [sys.executable]
+    if cores > 1 and importlib.util.find_spec("mpi4py") is not None:
+        if backend is None:
+            command_lst = (
+                ["mpiexec", "-n", str(cores)]
+                + command_lst
+                + [get_command_path(executable="cache_parallel.py"), file_name]
+            )
+        elif backend == "slurm":
+            command_prepend = ["srun", "-n", str(cores)]
+            if pmi_mode is not None:
+                command_prepend += ["--mpi=" + pmi_mode]
+            if openmpi_oversubscribe:
+                command_prepend += ["--oversubscribe"]
+            if exclusive:
+                command_prepend += ["--exact"]
+            command_lst = (
+                command_prepend
+                + command_lst
+                + [get_command_path(executable="cache_parallel.py"), file_name]
+            )
+        elif backend == "flux":
+            flux_command = ["flux", "run"]
+            if pmi_mode is not None:
+                flux_command += ["-o", "pmi=" + pmi_mode]
+            if openmpi_oversubscribe:
+                raise ValueError(
+                    "The option openmpi_oversubscribe is not available with the flux backend."
+                )
+            if exclusive:
+                raise ValueError(
+                    "The option exclusive is not available with the flux backend."
+                )
+            command_lst = (
+                flux_command
+                + ["-n", str(cores)]
+                + command_lst
+                + [get_command_path(executable="cache_parallel.py"), file_name]
+            )
+        else:
+            raise ValueError(f"backend should be None, slurm or flux, not {backend}")
+    elif cores > 1:
+        raise ImportError(
+            "mpi4py is required for parallel calculations. Please install mpi4py."
+        )
+    else:
+        command_lst += [get_command_path(executable="cache_serial.py"), file_name]
+    return command_lst
+
+
+def get_interactive_execute_command(
+    cores: int,
+) -> list:
+    """
+    Get command to call backend as a list of two strings
+
+    Args:
+        cores (int): Number of cores used to execute the task, if it is greater than one use interactive_parallel.py
+                     else interactive_serial.py
+
+    Returns:
+        list[str]: List of strings containing the python executable path and the backend script to execute
+    """
+    command_lst = [sys.executable]
+    if cores > 1 and importlib.util.find_spec("mpi4py") is not None:
+        command_lst += [get_command_path(executable="interactive_parallel.py")]
+    elif cores > 1:
+        raise ImportError(
+            "mpi4py is required for parallel calculations. Please install mpi4py."
+        )
+    else:
+        command_lst += [get_command_path(executable="interactive_serial.py")]
+    return command_lst