executorlib 0.0.4__tar.gz → 0.0.5__tar.gz

executorlib-0.0.5/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.1
+Name: executorlib
+Version: 0.0.5
+Summary: Scale serial and MPI-parallel python functions over hundreds of compute nodes all from within a jupyter notebook or serial python process.
+Author-email: Jan Janssen <janssen@lanl.gov>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2022, Jan Janssen
+        All rights reserved.
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        * Redistributions of source code must retain the above copyright notice, this
+          list of conditions and the following disclaimer.
+        
+        * Redistributions in binary form must reproduce the above copyright notice,
+          this list of conditions and the following disclaimer in the documentation
+          and/or other materials provided with the distribution.
+        
+        * Neither the name of the copyright holder nor the names of its
+          contributors may be used to endorse or promote products derived from
+          this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, https://github.com/pyiron/executorlib
+Project-URL: Documentation, https://executorlib.readthedocs.io
+Project-URL: Repository, https://github.com/pyiron/executorlib
+Keywords: pyiron
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Topic :: Scientific/Engineering :: Physics
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: <3.14,>=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: cloudpickle<=3.1.0,>=2.0.0
+Requires-Dist: pyzmq<=26.2.0,>=25.0.0
+Provides-Extra: cache
+Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "cache"
+Provides-Extra: graph
+Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graph"
+Requires-Dist: matplotlib<=3.9.2,>=3.5.3; extra == "graph"
+Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graph"
+Requires-Dist: ipython<=8.29.0,>=7.33.0; extra == "graph"
+Provides-Extra: mpi
+Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "mpi"
+Provides-Extra: submission
+Requires-Dist: pysqa==0.2.2; extra == "submission"
+Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "submission"
+Provides-Extra: all
+Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "all"
+Requires-Dist: pysqa==0.2.2; extra == "all"
+Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "all"
+Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "all"
+Requires-Dist: matplotlib<=3.9.2,>=3.5.3; extra == "all"
+Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "all"
+Requires-Dist: ipython<=8.29.0,>=7.33.0; extra == "all"
+
+# executorlib
+[![Unittests](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml)
+[![Coverage Status](https://coveralls.io/repos/github/pyiron/executorlib/badge.svg?branch=main)](https://coveralls.io/github/pyiron/executorlib?branch=main)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD?labpath=notebooks%2Fexamples.ipynb)
+
+Up-scale python functions for high performance computing (HPC) with executorlib. 
+
+## Key Features
+* **Up-scale your Python functions beyond a single computer.** - executorlib extends the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
+  from the Python standard library and combines it with job schedulers for high performance computing (HPC) including 
+  the [Simple Linux Utility for Resource Management (SLURM)](https://slurm.schedmd.com) and [flux](http://flux-framework.org). 
+  With this combination executorlib allows users to distribute their Python functions over multiple compute nodes.
+* **Parallelize your Python program one function at a time** - executorlib allows users to assign dedicated computing
+  resources like CPU cores, threads or GPUs to one Python function call at a time. So you can accelerate your Python 
+  code function by function.
+* **Permanent caching of intermediate results to accelerate rapid prototyping** - To accelerate the development of 
+  machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for 
+  iterative development in interactive environments like jupyter notebooks.
+
+## Examples
+The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
+with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the 
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) for parallel 
+execution of Python functions on a single computer. executorlib extends this functionality to distribute Python 
+functions over multiple computers within a high performance computing (HPC) cluster. This can be either achieved by 
+submitting each function as individual job to the HPC job scheduler - [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html) - 
+or by requesting a compute allocation of multiple nodes and then distribute the Python functions within this - allocation -
+[HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html). Finally, to accelerate the 
+development process executorlib also provides a - [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html) - 
+to use the executorlib functionality on a single workstation for testing. Starting with the [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html) 
+set by setting the backend parameter to local - `backend="local"`:
+```python
+from executorlib import Executor
+
+
+with Executor(backend="local") as exe:
+    future_lst = [exe.submit(sum, [i, i]) for i in range(1, 5)]
+    print([f.result() for f in future_lst])
+```
+In the same way executorlib can also execute Python functions which use additional computing resources, like multiple 
+CPU cores, CPU threads or GPUs. For example if the Python function internally uses the Message Passing Interface (MPI) 
+via the [mpi4py](https://mpi4py.readthedocs.io) Python libary: 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="local") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+The additional `resource_dict` parameter defines the computing resources allocated to the execution of the submitted 
+Python function. In addition to the compute cores `cores`, the resource dictionary can also define the threads per core
+as `threads_per_core`, the GPUs per core as `gpus_per_core`, the working directory with `cwd`, the option to use the
+OpenMPI oversubscribe feature with `openmpi_oversubscribe` and finally for the [Simple Linux Utility for Resource 
+Management (SLURM)](https://slurm.schedmd.com) queuing system the option to provide additional command line arguments 
+with the `slurm_cmd_args` parameter - [resource dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary
+This flexibility to assign computing resources on a per-function-call basis simplifies the up-scaling of Python programs.
+Only the part of the Python functions which benefit from parallel execution are implemented as MPI parallel Python 
+funtions, while the rest of the program remains serial. 
+
+The same function can be submitted to the [SLURM](https://slurm.schedmd.com) queuing by just changing the `backend` 
+parameter to `slurm_submission`. The rest of the example remains the same, which highlights how executorlib accelerates
+the rapid prototyping and up-scaling of HPC Python programs. 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="slurm_submission") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+In this case the [Python simple queuing system adapter (pysqa)](https://pysqa.readthedocs.io) is used to submit the 
+`calc()` function to the [SLURM](https://slurm.schedmd.com) job scheduler and request an allocation with two CPU cores 
+for the execution of the function - [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html). In the background the [sbatch](https://slurm.schedmd.com/sbatch.html) 
+command is used to request the allocation to execute the Python function. 
+
+Within a given [SLURM](https://slurm.schedmd.com) allocation executorlib can also be used to assign a subset of the 
+available computing resources to execute a given Python function. In terms of the [SLURM](https://slurm.schedmd.com) 
+commands, this functionality internally uses the [srun](https://slurm.schedmd.com/srun.html) command to receive a subset
+of the resources of a given queuing system allocation. 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="slurm_allocation") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+In addition, to support for [SLURM](https://slurm.schedmd.com) executorlib also provides support for the hierarchical 
+[flux](http://flux-framework.org) job scheduler. The [flux](http://flux-framework.org) job scheduler is developed at 
+[Larwence Livermore National Laboratory](https://computing.llnl.gov/projects/flux-building-framework-resource-management)
+to address the needs for the up-coming generation of Exascale computers. Still even on traditional HPC clusters the 
+hierarchical approach of the [flux](http://flux-framework.org) is beneficial to distribute hundreds of tasks within a
+given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primary job scheduler of your HPC, it is 
+recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm-with-flux) 
+as hierarchical job scheduler within the allocations. 
+
+## Documentation
+* [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
+  * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
+  * [MPI Support](https://executorlib.readthedocs.io/en/latest/installation.html#mpi-support)
+  * [Caching](https://executorlib.readthedocs.io/en/latest/installation.html#caching)
+  * [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-submission-mode)
+  * [HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-allocation-mode)
+  * [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
+  * [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
+* [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html)
+  * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-local.html#basic-functionality)
+  * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-local.html#parallel-functions)
+  * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-local.html#performance-optimization)
+* [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html)
+  * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html#slurm)
+  * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html#flux)
+* [HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html)
+  * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm)
+  * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm-with-flux)
+  * [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#flux)
+* [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
+  * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
+  * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
+  * [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
+  * [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
+  * [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
+  * [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
+* [Developer](https://executorlib.readthedocs.io/en/latest/4-developer.html)
+  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
+  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
+  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
+  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
+* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)

executorlib-0.0.5/README.md

@@ -0,0 +1,154 @@
+# executorlib
+[![Unittests](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/unittest-openmpi.yml)
+[![Coverage Status](https://coveralls.io/repos/github/pyiron/executorlib/badge.svg?branch=main)](https://coveralls.io/github/pyiron/executorlib?branch=main)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD?labpath=notebooks%2Fexamples.ipynb)
+
+Up-scale python functions for high performance computing (HPC) with executorlib. 
+
+## Key Features
+* **Up-scale your Python functions beyond a single computer.** - executorlib extends the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
+  from the Python standard library and combines it with job schedulers for high performance computing (HPC) including 
+  the [Simple Linux Utility for Resource Management (SLURM)](https://slurm.schedmd.com) and [flux](http://flux-framework.org). 
+  With this combination executorlib allows users to distribute their Python functions over multiple compute nodes.
+* **Parallelize your Python program one function at a time** - executorlib allows users to assign dedicated computing
+  resources like CPU cores, threads or GPUs to one Python function call at a time. So you can accelerate your Python 
+  code function by function.
+* **Permanent caching of intermediate results to accelerate rapid prototyping** - To accelerate the development of 
+  machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for 
+  iterative development in interactive environments like jupyter notebooks.
+
+## Examples
+The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
+with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the 
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) for parallel 
+execution of Python functions on a single computer. executorlib extends this functionality to distribute Python 
+functions over multiple computers within a high performance computing (HPC) cluster. This can be either achieved by 
+submitting each function as individual job to the HPC job scheduler - [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html) - 
+or by requesting a compute allocation of multiple nodes and then distribute the Python functions within this - allocation -
+[HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html). Finally, to accelerate the 
+development process executorlib also provides a - [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html) - 
+to use the executorlib functionality on a single workstation for testing. Starting with the [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html) 
+set by setting the backend parameter to local - `backend="local"`:
+```python
+from executorlib import Executor
+
+
+with Executor(backend="local") as exe:
+    future_lst = [exe.submit(sum, [i, i]) for i in range(1, 5)]
+    print([f.result() for f in future_lst])
+```
+In the same way executorlib can also execute Python functions which use additional computing resources, like multiple 
+CPU cores, CPU threads or GPUs. For example if the Python function internally uses the Message Passing Interface (MPI) 
+via the [mpi4py](https://mpi4py.readthedocs.io) Python libary: 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="local") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+The additional `resource_dict` parameter defines the computing resources allocated to the execution of the submitted 
+Python function. In addition to the compute cores `cores`, the resource dictionary can also define the threads per core
+as `threads_per_core`, the GPUs per core as `gpus_per_core`, the working directory with `cwd`, the option to use the
+OpenMPI oversubscribe feature with `openmpi_oversubscribe` and finally for the [Simple Linux Utility for Resource 
+Management (SLURM)](https://slurm.schedmd.com) queuing system the option to provide additional command line arguments 
+with the `slurm_cmd_args` parameter - [resource dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary
+This flexibility to assign computing resources on a per-function-call basis simplifies the up-scaling of Python programs.
+Only the part of the Python functions which benefit from parallel execution are implemented as MPI parallel Python 
+funtions, while the rest of the program remains serial. 
+
+The same function can be submitted to the [SLURM](https://slurm.schedmd.com) queuing by just changing the `backend` 
+parameter to `slurm_submission`. The rest of the example remains the same, which highlights how executorlib accelerates
+the rapid prototyping and up-scaling of HPC Python programs. 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="slurm_submission") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+In this case the [Python simple queuing system adapter (pysqa)](https://pysqa.readthedocs.io) is used to submit the 
+`calc()` function to the [SLURM](https://slurm.schedmd.com) job scheduler and request an allocation with two CPU cores 
+for the execution of the function - [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html). In the background the [sbatch](https://slurm.schedmd.com/sbatch.html) 
+command is used to request the allocation to execute the Python function. 
+
+Within a given [SLURM](https://slurm.schedmd.com) allocation executorlib can also be used to assign a subset of the 
+available computing resources to execute a given Python function. In terms of the [SLURM](https://slurm.schedmd.com) 
+commands, this functionality internally uses the [srun](https://slurm.schedmd.com/srun.html) command to receive a subset
+of the resources of a given queuing system allocation. 
+```python
+from executorlib import Executor
+
+
+def calc(i):
+    from mpi4py import MPI
+
+    size = MPI.COMM_WORLD.Get_size()
+    rank = MPI.COMM_WORLD.Get_rank()
+    return i, size, rank
+
+
+with Executor(backend="slurm_allocation") as exe:
+    fs = exe.submit(calc, 3, resource_dict={"cores": 2})
+    print(fs.result())
+```
+In addition, to support for [SLURM](https://slurm.schedmd.com) executorlib also provides support for the hierarchical 
+[flux](http://flux-framework.org) job scheduler. The [flux](http://flux-framework.org) job scheduler is developed at 
+[Larwence Livermore National Laboratory](https://computing.llnl.gov/projects/flux-building-framework-resource-management)
+to address the needs for the up-coming generation of Exascale computers. Still even on traditional HPC clusters the 
+hierarchical approach of the [flux](http://flux-framework.org) is beneficial to distribute hundreds of tasks within a
+given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primary job scheduler of your HPC, it is 
+recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm-with-flux) 
+as hierarchical job scheduler within the allocations. 
+
+## Documentation
+* [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
+  * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
+  * [MPI Support](https://executorlib.readthedocs.io/en/latest/installation.html#mpi-support)
+  * [Caching](https://executorlib.readthedocs.io/en/latest/installation.html#caching)
+  * [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-submission-mode)
+  * [HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-allocation-mode)
+  * [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
+  * [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
+* [Local Mode](https://executorlib.readthedocs.io/en/latest/1-local.html)
+  * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-local.html#basic-functionality)
+  * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-local.html#parallel-functions)
+  * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-local.html#performance-optimization)
+* [HPC Submission Mode](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html)
+  * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html#slurm)
+  * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-submission.html#flux)
+* [HPC Allocation Mode](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html)
+  * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm)
+  * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#slurm-with-flux)
+  * [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-allocation.html#flux)
+* [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
+  * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
+  * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
+  * [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
+  * [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
+  * [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
+  * [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
+* [Developer](https://executorlib.readthedocs.io/en/latest/4-developer.html)
+  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
+  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
+  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
+  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
+* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)

{executorlib-0.0.4 → executorlib-0.0.5}/executorlib/__init__.py

@@ -1,31 +1,22 @@
 from typing import Optional
 
-from executorlib.interactive import create_executor
-from executorlib.interactive.dependencies import ExecutorWithDependencies
-from executorlib.shared.inputcheck import (
+from executorlib._version import get_versions as _get_versions
+from executorlib.interactive.executor import (
+    ExecutorWithDependencies as _ExecutorWithDependencies,
+)
+from executorlib.interactive.executor import create_executor as _create_executor
+from executorlib.standalone.inputcheck import (
     check_plot_dependency_graph as _check_plot_dependency_graph,
 )
-from executorlib.shared.inputcheck import (
+from executorlib.standalone.inputcheck import (
+    check_pysqa_config_directory as _check_pysqa_config_directory,
+)
+from executorlib.standalone.inputcheck import (
     check_refresh_rate as _check_refresh_rate,
 )
-from executorlib.shell.executor import SubprocessExecutor
-from executorlib.shell.interactive import ShellExecutor
-
-from ._version import get_versions
-
-__version__ = get_versions()["version"]
-__all__ = [
-    SubprocessExecutor,
-    ShellExecutor,
-]
-
 
-try:
-    from executorlib.cache.executor import FileExecutor
-
-    __all__ += [FileExecutor]
-except ImportError:
-    pass
+__version__ = _get_versions()["version"]
+__all__ = []
 
 
 class Executor:
@@ -41,16 +32,20 @@ class Executor:
                            cores which can be used in parallel - just like the max_cores parameter. Using max_cores is
                            recommended, as computers have a limited number of compute cores.
         backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local".
+        cache_directory (str, optional): The directory to store cache files. Defaults to "cache".
         max_cores (int): defines the number cores which can be used in parallel
-        cores_per_worker (int): number of MPI cores to be used for each function call
-        threads_per_core (int): number of OpenMP threads to be used for each function call
-        gpus_per_worker (int): number of GPUs per worker - defaults to 0
-        cwd (str/None): current working directory where the parallel python task is executed
-        openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
-        slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+        resource_dict (dict): A dictionary of resources required by the task. With the following keys:
+                              - cores_per_worker (int): number of MPI cores to be used for each function call
+                              - threads_per_core (int): number of OpenMP threads to be used for each function call
+                              - gpus_per_worker (int): number of GPUs per worker - defaults to 0
+                              - cwd (str/None): current working directory where the parallel python task is executed
+                              - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and
+                                                              SLURM only) - default False
+                              - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
+        pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -91,20 +86,17 @@ class Executor:
 
     def __init__(
         self,
-        max_workers: int = 1,
+        max_workers: Optional[int] = None,
         backend: str = "local",
-        max_cores: int = 1,
-        cores_per_worker: int = 1,
-        threads_per_core: int = 1,
-        gpus_per_worker: int = 0,
-        cwd: Optional[str] = None,
-        openmpi_oversubscribe: bool = False,
-        slurm_cmd_args: list[str] = [],
+        cache_directory: Optional[str] = None,
+        max_cores: Optional[int] = None,
+        resource_dict: Optional[dict] = None,
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        hostname_localhost: bool = False,
-        block_allocation: bool = True,
+        pysqa_config_directory: Optional[str] = None,
+        hostname_localhost: Optional[bool] = None,
+        block_allocation: bool = False,
         init_function: Optional[callable] = None,
         disable_dependencies: bool = False,
         refresh_rate: float = 0.01,
@@ -115,20 +107,17 @@ class Executor:
 
     def __new__(
         cls,
-        max_workers: int = 1,
+        max_workers: Optional[int] = None,
         backend: str = "local",
-        max_cores: int = 1,
-        cores_per_worker: int = 1,
-        threads_per_core: int = 1,
-        gpus_per_worker: int = 0,
-        cwd: Optional[str] = None,
-        openmpi_oversubscribe: bool = False,
-        slurm_cmd_args: list[str] = [],
+        cache_directory: Optional[str] = None,
+        max_cores: Optional[int] = None,
+        resource_dict: Optional[dict] = None,
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        hostname_localhost: bool = False,
-        block_allocation: bool = True,
+        pysqa_config_directory: Optional[str] = None,
+        hostname_localhost: Optional[bool] = None,
+        block_allocation: bool = False,
         init_function: Optional[callable] = None,
         disable_dependencies: bool = False,
         refresh_rate: float = 0.01,
@@ -147,16 +136,21 @@ class Executor:
                                number of cores which can be used in parallel - just like the max_cores parameter. Using
                                max_cores is recommended, as computers have a limited number of compute cores.
             backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local".
+            cache_directory (str, optional): The directory to store cache files. Defaults to "cache".
             max_cores (int): defines the number cores which can be used in parallel
-            cores_per_worker (int): number of MPI cores to be used for each function call
-            threads_per_core (int): number of OpenMP threads to be used for each function call
-            gpus_per_worker (int): number of GPUs per worker - defaults to 0
-            openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
-            slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
-            cwd (str/None): current working directory where the parallel python task is executed
+            resource_dict (dict): A dictionary of resources required by the task. With the following keys:
+                                  - cores (int): number of MPI cores to be used for each function call
+                                  - threads_per_core (int): number of OpenMP threads to be used for each function call
+                                  - gpus_per_core (int): number of GPUs per worker - defaults to 0
+                                  - cwd (str/None): current working directory where the parallel python task is executed
+                                  - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI
+                                                                  and SLURM only) - default False
+                                  - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM
+                                                           only)
             flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
             flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
             flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
+            pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -175,17 +169,45 @@ class Executor:
                                           debugging purposes and to get an overview of the specified dependencies.
 
         """
-        if not disable_dependencies:
-            return ExecutorWithDependencies(
+        default_resource_dict = {
+            "cores": 1,
+            "threads_per_core": 1,
+            "gpus_per_core": 0,
+            "cwd": None,
+            "openmpi_oversubscribe": False,
+            "slurm_cmd_args": [],
+        }
+        if resource_dict is None:
+            resource_dict = {}
+        resource_dict.update(
+            {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
+        )
+        if "_submission" in backend and not plot_dependency_graph:
+            from executorlib.cache.executor import create_file_executor
+
+            return create_file_executor(
+                max_workers=max_workers,
+                backend=backend,
+                max_cores=max_cores,
+                cache_directory=cache_directory,
+                resource_dict=resource_dict,
+                flux_executor=flux_executor,
+                flux_executor_pmi_mode=flux_executor_pmi_mode,
+                flux_executor_nesting=flux_executor_nesting,
+                pysqa_config_directory=pysqa_config_directory,
+                hostname_localhost=hostname_localhost,
+                block_allocation=block_allocation,
+                init_function=init_function,
+                disable_dependencies=disable_dependencies,
+            )
+        elif not disable_dependencies:
+            _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory)
+            return _ExecutorWithDependencies(
                 max_workers=max_workers,
                 backend=backend,
+                cache_directory=cache_directory,
                 max_cores=max_cores,
-                cores_per_worker=cores_per_worker,
-                threads_per_core=threads_per_core,
-                gpus_per_worker=gpus_per_worker,
-                cwd=cwd,
-                openmpi_oversubscribe=openmpi_oversubscribe,
-                slurm_cmd_args=slurm_cmd_args,
+                resource_dict=resource_dict,
                 flux_executor=flux_executor,
                 flux_executor_pmi_mode=flux_executor_pmi_mode,
                 flux_executor_nesting=flux_executor_nesting,
@@ -196,18 +218,15 @@ class Executor:
                 plot_dependency_graph=plot_dependency_graph,
             )
         else:
+            _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory)
             _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
             _check_refresh_rate(refresh_rate=refresh_rate)
-            return create_executor(
+            return _create_executor(
                 max_workers=max_workers,
                 backend=backend,
+                cache_directory=cache_directory,
                 max_cores=max_cores,
-                cores_per_worker=cores_per_worker,
-                threads_per_core=threads_per_core,
-                gpus_per_worker=gpus_per_worker,
-                cwd=cwd,
-                openmpi_oversubscribe=openmpi_oversubscribe,
-                slurm_cmd_args=slurm_cmd_args,
+                resource_dict=resource_dict,
                 flux_executor=flux_executor,
                 flux_executor_pmi_mode=flux_executor_pmi_mode,
                 flux_executor_nesting=flux_executor_nesting,

{executorlib-0.0.4 → executorlib-0.0.5}/executorlib/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-10-14T11:43:21+0200",
+ "date": "2024-11-20T13:20:24+0100",
  "dirty": true,
  "error": null,
- "full-revisionid": "7f3065ee67503167f6d86a654e6238c5ae361674",
- "version": "0.0.4"
+ "full-revisionid": "eb8cb29e2ebbc6930da51c5511b873ea5bf92f69",
+ "version": "0.0.5"
 }
 '''  # END VERSION_JSON
 

{executorlib-0.0.4 → executorlib-0.0.5}/executorlib/backend/cache_parallel.py

@@ -3,7 +3,7 @@ import sys
 
 import cloudpickle
 
-from executorlib.cache.shared import backend_load_file, backend_write_file
+from executorlib.cache.backend import backend_load_file, backend_write_file
 
 
 def main() -> None: