executorlib 0.0.1__tar.gz

executorlib-0.0.1/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Jan Janssen
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

executorlib-0.0.1/MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE

executorlib-0.0.1/PKG-INFO

@@ -0,0 +1,193 @@
+Metadata-Version: 2.1
+Name: executorlib
+Version: 0.0.1
+Summary: Scale serial and MPI-parallel python functions over hundreds of compute nodes all from within a jupyter notebook or serial python process.
+Author-email: Jan Janssen <janssen@lanl.gov>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2022, Jan Janssen
+        All rights reserved.
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        * Redistributions of source code must retain the above copyright notice, this
+          list of conditions and the following disclaimer.
+        
+        * Redistributions in binary form must reproduce the above copyright notice,
+          this list of conditions and the following disclaimer in the documentation
+          and/or other materials provided with the distribution.
+        
+        * Neither the name of the copyright holder nor the names of its
+          contributors may be used to endorse or promote products derived from
+          this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, https://github.com/pyiron/executorlib
+Project-URL: Documentation, https://executorlib.readthedocs.io
+Project-URL: Repository, https://github.com/pyiron/executorlib
+Keywords: pyiron
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Topic :: Scientific/Engineering :: Physics
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: <3.13,>=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: cloudpickle<=3.0.0,>=2.0.0
+Requires-Dist: pyzmq<=26.0.3,>=25.0.0
+Provides-Extra: conda
+Requires-Dist: conda_subprocess<=0.0.4,>=0.0.3; extra == "conda"
+Provides-Extra: mpi
+Requires-Dist: mpi4py<=3.1.6,>=3.1.4; extra == "mpi"
+Provides-Extra: hdf
+Requires-Dist: h5py<=3.11.0,>=3.6.0; extra == "hdf"
+Requires-Dist: h5io<=0.2.3,>=0.2.1; extra == "hdf"
+Provides-Extra: graph
+Requires-Dist: pygraphviz<=1.13,>=1.10; extra == "graph"
+Requires-Dist: matplotlib<=3.9.1,>=3.5.3; extra == "graph"
+Requires-Dist: networkx<=3.3,>=2.8.8; extra == "graph"
+Requires-Dist: ipython<=8.26.0,>=7.33.0; extra == "graph"
+
+# executorlib
+[![Unittests](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml)
+[![Coverage Status](https://coveralls.io/repos/github/pyiron/executorlib/badge.svg?branch=main)](https://coveralls.io/github/pyiron/executorlib?branch=main)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD?labpath=notebooks%2Fexamples.ipynb)
+
+## Challenges
+In high performance computing (HPC) the Python programming language is commonly used as high-level language to
+orchestrate the coupling of scientific applications. Still the efficient usage of highly parallel HPC clusters remains
+challenging, in primarily three aspects:
+
+* **Communication**: Distributing python function calls over hundreds of compute node and gathering the results on a
+  shared file system is technically possible, but highly inefficient. A socket-based communication approach is 
+  preferable.
+* **Resource Management**: Assigning Python functions to GPUs or executing Python functions on multiple CPUs using the 
+  message passing interface (MPI) requires major modifications to the python workflow.
+* **Integration**: Existing workflow libraries implement a secondary the job management on the Python level rather than
+  leveraging the existing infrastructure provided by the job scheduler of the HPC.
+
+### executorlib is ...
+In a given HPC allocation the `executorlib` library addresses these challenges by extending the Executor interface
+of the standard Python library to support the resource assignment in the HPC context. Computing resources can either be
+assigned on a per function call basis or as a block allocation on a per Executor basis. The `executorlib` library
+is built on top of the [flux-framework](https://flux-framework.org) to enable fine-grained resource assignment. In
+addition, [Simple Linux Utility for Resource Management (SLURM)](https://slurm.schedmd.com) is supported as alternative
+queuing system and for workstation installations `executorlib` can be installed without a job scheduler.
+
+### executorlib is not ...
+The executorlib library is not designed to request an allocation from the job scheduler of an HPC. Instead within a given
+allocation from the job scheduler the `executorlib` library can be employed to distribute a series of python
+function calls over the available computing resources to achieve maximum computing resource utilization.
+
+## Example
+The following examples illustrates how `executorlib` can be used to distribute a series of MPI parallel function calls 
+within a queuing system allocation. `example.py`:
+```python
+import flux.job
+from executorlib import Executor
+
+def calc(i):
+    from mpi4py import MPI
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+with flux.job.FluxExecutor() as flux_exe:
+    with Executor(max_cores=2, cores_per_worker=2, executor=flux_exe) as exe:
+        fs = exe.submit(calc, 3)
+        print(fs.result())
+```
+This example can be executed using:
+```
+python example.py
+```
+Which returns:
+```
+>>> [(0, 2, 0), (0, 2, 1)], [(1, 2, 0), (1, 2, 1)]
+```
+The important part in this example is that [mpi4py](https://mpi4py.readthedocs.io) is only used in the `calc()`
+function, not in the python script, consequently it is not necessary to call the script with `mpiexec` but instead
+a call with the regular python interpreter is sufficient. This highlights how `executorlib` allows the users to
+parallelize one function at a time and not having to convert their whole workflow to use [mpi4py](https://mpi4py.readthedocs.io).
+The same code can also be executed inside a jupyter notebook directly which enables an interactive development process.
+
+The interface of the standard [concurrent.futures.Executor](https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures)
+is extended by adding the option `cores_per_worker=2` to assign multiple MPI ranks to each function call. To create two 
+workers the maximum number of cores can be increased to `max_cores=4`. In this case each worker receives two cores
+resulting in a total of four CPU cores being utilized.
+
+After submitting the function `calc()` with the corresponding parameter to the executor `exe.submit(calc, 0)`
+a python [`concurrent.futures.Future`](https://docs.python.org/3/library/concurrent.futures.html#future-objects) is
+returned. Consequently, the `executorlib.Executor` can be used as a drop-in replacement for the
+[`concurrent.futures.Executor`](https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures)
+which allows the user to add parallelism to their workflow one function at a time.
+
+## Disclaimer
+While we try to develop a stable and reliable software library, the development remains a opensource project under the
+BSD 3-Clause License without any warranties::
+```
+BSD 3-Clause License
+
+Copyright (c) 2022, Jan Janssen
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```
+
+# Documentation
+* [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
+  * [Compatible Job Schedulers](https://executorlib.readthedocs.io/en/latest/installation.html#compatible-job-schedulers)
+  * [executorlib with Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#executorlib-with-flux-framework)
+  * [Test Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#test-flux-framework)
+  * [Without Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#without-flux-framework)
+* [Examples](https://executorlib.readthedocs.io/en/latest/examples.html)
+  * [Compatibility](https://executorlib.readthedocs.io/en/latest/examples.html#compatibility)
+  * [Resource Assignment](https://executorlib.readthedocs.io/en/latest/examples.html#resource-assignment)
+  * [Data Handling](https://executorlib.readthedocs.io/en/latest/examples.html#data-handling)
+  * [Up-Scaling](https://executorlib.readthedocs.io/en/latest/examples.html#up-scaling)
+  * [Coupled Functions](https://executorlib.readthedocs.io/en/latest/examples.html#coupled-functions)
+  * [SLURM Job Scheduler](https://executorlib.readthedocs.io/en/latest/examples.html#slurm-job-scheduler) 
+  * [Workstation Support](https://executorlib.readthedocs.io/en/latest/examples.html#workstation-support)
+* [Development](https://executorlib.readthedocs.io/en/latest/development.html)
+  * [Contributions](https://executorlib.readthedocs.io/en/latest/development.html#contributions)
+  * [License](https://executorlib.readthedocs.io/en/latest/development.html#license)
+  * [Integration](https://executorlib.readthedocs.io/en/latest/development.html#integration)

executorlib-0.0.1/README.md

@@ -0,0 +1,127 @@
+# executorlib
+[![Unittests](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml)
+[![Coverage Status](https://coveralls.io/repos/github/pyiron/executorlib/badge.svg?branch=main)](https://coveralls.io/github/pyiron/executorlib?branch=main)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD?labpath=notebooks%2Fexamples.ipynb)
+
+## Challenges
+In high performance computing (HPC) the Python programming language is commonly used as high-level language to
+orchestrate the coupling of scientific applications. Still the efficient usage of highly parallel HPC clusters remains
+challenging, in primarily three aspects:
+
+* **Communication**: Distributing python function calls over hundreds of compute node and gathering the results on a
+  shared file system is technically possible, but highly inefficient. A socket-based communication approach is 
+  preferable.
+* **Resource Management**: Assigning Python functions to GPUs or executing Python functions on multiple CPUs using the 
+  message passing interface (MPI) requires major modifications to the python workflow.
+* **Integration**: Existing workflow libraries implement a secondary the job management on the Python level rather than
+  leveraging the existing infrastructure provided by the job scheduler of the HPC.
+
+### executorlib is ...
+In a given HPC allocation the `executorlib` library addresses these challenges by extending the Executor interface
+of the standard Python library to support the resource assignment in the HPC context. Computing resources can either be
+assigned on a per function call basis or as a block allocation on a per Executor basis. The `executorlib` library
+is built on top of the [flux-framework](https://flux-framework.org) to enable fine-grained resource assignment. In
+addition, [Simple Linux Utility for Resource Management (SLURM)](https://slurm.schedmd.com) is supported as alternative
+queuing system and for workstation installations `executorlib` can be installed without a job scheduler.
+
+### executorlib is not ...
+The executorlib library is not designed to request an allocation from the job scheduler of an HPC. Instead within a given
+allocation from the job scheduler the `executorlib` library can be employed to distribute a series of python
+function calls over the available computing resources to achieve maximum computing resource utilization.
+
+## Example
+The following examples illustrates how `executorlib` can be used to distribute a series of MPI parallel function calls 
+within a queuing system allocation. `example.py`:
+```python
+import flux.job
+from executorlib import Executor
+
+def calc(i):
+    from mpi4py import MPI
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+with flux.job.FluxExecutor() as flux_exe:
+    with Executor(max_cores=2, cores_per_worker=2, executor=flux_exe) as exe:
+        fs = exe.submit(calc, 3)
+        print(fs.result())
+```
+This example can be executed using:
+```
+python example.py
+```
+Which returns:
+```
+>>> [(0, 2, 0), (0, 2, 1)], [(1, 2, 0), (1, 2, 1)]
+```
+The important part in this example is that [mpi4py](https://mpi4py.readthedocs.io) is only used in the `calc()`
+function, not in the python script, consequently it is not necessary to call the script with `mpiexec` but instead
+a call with the regular python interpreter is sufficient. This highlights how `executorlib` allows the users to
+parallelize one function at a time and not having to convert their whole workflow to use [mpi4py](https://mpi4py.readthedocs.io).
+The same code can also be executed inside a jupyter notebook directly which enables an interactive development process.
+
+The interface of the standard [concurrent.futures.Executor](https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures)
+is extended by adding the option `cores_per_worker=2` to assign multiple MPI ranks to each function call. To create two 
+workers the maximum number of cores can be increased to `max_cores=4`. In this case each worker receives two cores
+resulting in a total of four CPU cores being utilized.
+
+After submitting the function `calc()` with the corresponding parameter to the executor `exe.submit(calc, 0)`
+a python [`concurrent.futures.Future`](https://docs.python.org/3/library/concurrent.futures.html#future-objects) is
+returned. Consequently, the `executorlib.Executor` can be used as a drop-in replacement for the
+[`concurrent.futures.Executor`](https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures)
+which allows the user to add parallelism to their workflow one function at a time.
+
+## Disclaimer
+While we try to develop a stable and reliable software library, the development remains a opensource project under the
+BSD 3-Clause License without any warranties::
+```
+BSD 3-Clause License
+
+Copyright (c) 2022, Jan Janssen
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```
+
+# Documentation
+* [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
+  * [Compatible Job Schedulers](https://executorlib.readthedocs.io/en/latest/installation.html#compatible-job-schedulers)
+  * [executorlib with Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#executorlib-with-flux-framework)
+  * [Test Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#test-flux-framework)
+  * [Without Flux Framework](https://executorlib.readthedocs.io/en/latest/installation.html#without-flux-framework)
+* [Examples](https://executorlib.readthedocs.io/en/latest/examples.html)
+  * [Compatibility](https://executorlib.readthedocs.io/en/latest/examples.html#compatibility)
+  * [Resource Assignment](https://executorlib.readthedocs.io/en/latest/examples.html#resource-assignment)
+  * [Data Handling](https://executorlib.readthedocs.io/en/latest/examples.html#data-handling)
+  * [Up-Scaling](https://executorlib.readthedocs.io/en/latest/examples.html#up-scaling)
+  * [Coupled Functions](https://executorlib.readthedocs.io/en/latest/examples.html#coupled-functions)
+  * [SLURM Job Scheduler](https://executorlib.readthedocs.io/en/latest/examples.html#slurm-job-scheduler) 
+  * [Workstation Support](https://executorlib.readthedocs.io/en/latest/examples.html#workstation-support)
+* [Development](https://executorlib.readthedocs.io/en/latest/development.html)
+  * [Contributions](https://executorlib.readthedocs.io/en/latest/development.html#contributions)
+  * [License](https://executorlib.readthedocs.io/en/latest/development.html#license)
+  * [Integration](https://executorlib.readthedocs.io/en/latest/development.html#integration)

executorlib-0.0.1/executorlib/__init__.py

@@ -0,0 +1,225 @@
+from typing import Optional
+
+from executorlib.interactive import create_executor
+from executorlib.interactive.dependencies import ExecutorWithDependencies
+from executorlib.shared.inputcheck import (
+    check_plot_dependency_graph as _check_plot_dependency_graph,
+)
+from executorlib.shared.inputcheck import (
+    check_refresh_rate as _check_refresh_rate,
+)
+from executorlib.shell.executor import SubprocessExecutor
+from executorlib.shell.interactive import ShellExecutor
+
+from ._version import get_versions
+
+__version__ = get_versions()["version"]
+__all__ = [
+    SubprocessExecutor,
+    ShellExecutor,
+]
+
+
+try:
+    from executorlib.cache.executor import FileExecutor
+
+    __all__ += [FileExecutor]
+except ImportError:
+    pass
+
+
+class Executor:
+    """
+    The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
+    preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
+    the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not
+    require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly
+    in an interactive Jupyter notebook.
+
+    Args:
+        max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of
+                           cores which can be used in parallel - just like the max_cores parameter. Using max_cores is
+                           recommended, as computers have a limited number of compute cores.
+        max_cores (int): defines the number cores which can be used in parallel
+        cores_per_worker (int): number of MPI cores to be used for each function call
+        threads_per_core (int): number of OpenMP threads to be used for each function call
+        gpus_per_worker (int): number of GPUs per worker - defaults to 0
+        oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
+        cwd (str/None): current working directory where the parallel python task is executed
+        conda_environment_name (str): name of the conda environment to initialize
+        conda_environment_path (str): path of the conda environment to initialize
+        executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
+        hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
+                                      context of an HPC cluster this essential to be able to communicate to an
+                                      Executor running on a different compute node within the same allocation. And
+                                      in principle any computer should be able to resolve that their own hostname
+                                      points to the same address as localhost. Still MacOS >= 12 seems to disable
+                                      this look up for security reasons. So on MacOS it is required to set this
+                                      option to true
+        backend (str): Switch between the different backends "flux", "local" or "slurm". Alternatively, when "auto"
+                       is selected (the default) the available backend is determined automatically.
+        block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource
+                                    requirements, executorlib supports block allocation. In this case all resources have
+                                    to be defined on the executor, rather than during the submission of the individual
+                                    function.
+        init_function (None): optional function to preset arguments for functions which are submitted later
+        command_line_argument_lst (list): Additional command line arguments for the srun call (SLURM only)
+        pmi (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
+        disable_dependencies (boolean): Disable resolving future objects during the submission.
+        refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
+        plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
+                                      debugging purposes and to get an overview of the specified dependencies.
+
+    Examples:
+        ```
+        >>> import numpy as np
+        >>> from executorlib import Executor
+        >>>
+        >>> def calc(i, j, k):
+        >>>     from mpi4py import MPI
+        >>>     size = MPI.COMM_WORLD.Get_size()
+        >>>     rank = MPI.COMM_WORLD.Get_rank()
+        >>>     return np.array([i, j, k]), size, rank
+        >>>
+        >>> def init_k():
+        >>>     return {"k": 3}
+        >>>
+        >>> with Executor(cores=2, init_function=init_k) as p:
+        >>>     fs = p.submit(calc, 2, j=4)
+        >>>     print(fs.result())
+        [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)]
+        ```
+    """
+
+    def __init__(
+        self,
+        max_workers: int = 1,
+        max_cores: int = 1,
+        cores_per_worker: int = 1,
+        threads_per_core: int = 1,
+        gpus_per_worker: int = 0,
+        oversubscribe: bool = False,
+        cwd: Optional[str] = None,
+        conda_environment_name: Optional[str] = None,
+        conda_environment_path: Optional[str] = None,
+        executor=None,
+        hostname_localhost: bool = False,
+        backend: str = "auto",
+        block_allocation: bool = True,
+        init_function: Optional[callable] = None,
+        command_line_argument_lst: list[str] = [],
+        pmi: Optional[str] = None,
+        disable_dependencies: bool = False,
+        refresh_rate: float = 0.01,
+        plot_dependency_graph: bool = False,
+    ):
+        # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
+        pass
+
+    def __new__(
+        cls,
+        max_workers: int = 1,
+        max_cores: int = 1,
+        cores_per_worker: int = 1,
+        threads_per_core: int = 1,
+        gpus_per_worker: int = 0,
+        oversubscribe: bool = False,
+        cwd: Optional[str] = None,
+        conda_environment_name: Optional[str] = None,
+        conda_environment_path: Optional[str] = None,
+        executor=None,
+        hostname_localhost: bool = False,
+        backend: str = "auto",
+        block_allocation: bool = False,
+        init_function: Optional[callable] = None,
+        command_line_argument_lst: list[str] = [],
+        pmi: Optional[str] = None,
+        disable_dependencies: bool = False,
+        refresh_rate: float = 0.01,
+        plot_dependency_graph: bool = False,
+    ):
+        """
+        Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
+        executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The
+        executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used
+        for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be
+        installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor
+        requires the SLURM workload manager to be installed on the system.
+
+        Args:
+            max_workers (int): for backwards compatibility with the standard library, max_workers also defines the
+                               number of cores which can be used in parallel - just like the max_cores parameter. Using
+                               max_cores is recommended, as computers have a limited number of compute cores.
+            max_cores (int): defines the number cores which can be used in parallel
+            cores_per_worker (int): number of MPI cores to be used for each function call
+            threads_per_core (int): number of OpenMP threads to be used for each function call
+            gpus_per_worker (int): number of GPUs per worker - defaults to 0
+            oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
+            cwd (str/None): current working directory where the parallel python task is executed
+            conda_environment_name (str): name of the conda environment to initialize
+            conda_environment_path (str): path of the conda environment to initialize
+            executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
+            hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
+                                      context of an HPC cluster this essential to be able to communicate to an
+                                      Executor running on a different compute node within the same allocation. And
+                                      in principle any computer should be able to resolve that their own hostname
+                                      points to the same address as localhost. Still MacOS >= 12 seems to disable
+                                      this look up for security reasons. So on MacOS it is required to set this
+                                      option to true
+            backend (str): Switch between the different backends "flux", "local" or "slurm". Alternatively, when "auto"
+                           is selected (the default) the available backend is determined automatically.
+            block_allocation (boolean): To accelerate the submission of a series of python functions with the same
+                                        resource requirements, executorlib supports block allocation. In this case all
+                                        resources have to be defined on the executor, rather than during the submission
+                                        of the individual function.
+            init_function (None): optional function to preset arguments for functions which are submitted later
+            command_line_argument_lst (list): Additional command line arguments for the srun call (SLURM only)
+            pmi (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
+            disable_dependencies (boolean): Disable resolving future objects during the submission.
+            refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
+            plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
+                                          debugging purposes and to get an overview of the specified dependencies.
+
+        """
+        if not disable_dependencies:
+            return ExecutorWithDependencies(
+                max_workers=max_workers,
+                max_cores=max_cores,
+                cores_per_worker=cores_per_worker,
+                threads_per_core=threads_per_core,
+                gpus_per_worker=gpus_per_worker,
+                oversubscribe=oversubscribe,
+                cwd=cwd,
+                conda_environment_name=conda_environment_name,
+                conda_environment_path=conda_environment_path,
+                executor=executor,
+                hostname_localhost=hostname_localhost,
+                backend=backend,
+                block_allocation=block_allocation,
+                init_function=init_function,
+                command_line_argument_lst=command_line_argument_lst,
+                pmi=pmi,
+                refresh_rate=refresh_rate,
+                plot_dependency_graph=plot_dependency_graph,
+            )
+        else:
+            _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
+            _check_refresh_rate(refresh_rate=refresh_rate)
+            return create_executor(
+                max_workers=max_workers,
+                max_cores=max_cores,
+                cores_per_worker=cores_per_worker,
+                threads_per_core=threads_per_core,
+                gpus_per_worker=gpus_per_worker,
+                oversubscribe=oversubscribe,
+                cwd=cwd,
+                conda_environment_name=conda_environment_name,
+                conda_environment_path=conda_environment_path,
+                executor=executor,
+                hostname_localhost=hostname_localhost,
+                backend=backend,
+                block_allocation=block_allocation,
+                init_function=init_function,
+                command_line_argument_lst=command_line_argument_lst,
+                pmi=pmi,
+            )

executorlib-0.0.1/executorlib/_version.py

@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.29) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+{
+ "date": "2024-07-14T21:06:50+0200",
+ "dirty": true,
+ "error": null,
+ "full-revisionid": "0a84450d76b7081b62a9d948bac56f905d013b5d",
+ "version": "0.0.1"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)

executorlib-0.0.1/executorlib/backend/cache_parallel.py

@@ -0,0 +1,40 @@
+import pickle
+import sys
+
+import cloudpickle
+
+from executorlib.cache.shared import backend_load_file, backend_write_file
+
+
+def main():
+    from mpi4py import MPI
+
+    MPI.pickle.__init__(
+        cloudpickle.dumps,
+        cloudpickle.loads,
+        pickle.HIGHEST_PROTOCOL,
+    )
+    mpi_rank_zero = MPI.COMM_WORLD.Get_rank() == 0
+    mpi_size_larger_one = MPI.COMM_WORLD.Get_size() > 1
+    file_name = sys.argv[1]
+
+    if mpi_rank_zero:
+        apply_dict = backend_load_file(file_name=file_name)
+    else:
+        apply_dict = None
+    apply_dict = MPI.COMM_WORLD.bcast(apply_dict, root=0)
+    output = apply_dict["fn"].__call__(*apply_dict["args"], **apply_dict["kwargs"])
+    if mpi_size_larger_one:
+        result = MPI.COMM_WORLD.gather(output, root=0)
+    else:
+        result = output
+    if mpi_rank_zero:
+        backend_write_file(
+            file_name=file_name,
+            output=result,
+        )
+    MPI.COMM_WORLD.Barrier()
+
+
+if __name__ == "__main__":
+    main()

executorlib-0.0.1/executorlib/backend/cache_serial.py

@@ -0,0 +1,6 @@
+import sys
+
+from executorlib.cache.shared import execute_task_in_file
+
+if __name__ == "__main__":
+    execute_task_in_file(file_name=sys.argv[1])