executorlib 1.2.0__tar.gz → 1.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. executorlib-1.2.1/.gitignore +5 -0
  2. {executorlib-1.2.0/executorlib.egg-info → executorlib-1.2.1}/PKG-INFO +28 -28
  3. {executorlib-1.2.0 → executorlib-1.2.1}/README.md +1 -0
  4. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/__init__.py +3 -2
  5. executorlib-1.2.1/executorlib/_version.py +21 -0
  6. {executorlib-1.2.0 → executorlib-1.2.1}/pyproject.toml +26 -12
  7. executorlib-1.2.0/MANIFEST.in +0 -1
  8. executorlib-1.2.0/PKG-INFO +0 -239
  9. executorlib-1.2.0/executorlib/_version.py +0 -21
  10. executorlib-1.2.0/executorlib.egg-info/SOURCES.txt +0 -77
  11. executorlib-1.2.0/executorlib.egg-info/dependency_links.txt +0 -1
  12. executorlib-1.2.0/executorlib.egg-info/requires.txt +0 -29
  13. executorlib-1.2.0/executorlib.egg-info/top_level.txt +0 -1
  14. executorlib-1.2.0/setup.cfg +0 -4
  15. executorlib-1.2.0/setup.py +0 -8
  16. executorlib-1.2.0/tests/test_backend_interactive_serial.py +0 -85
  17. executorlib-1.2.0/tests/test_base_executor_queue.py +0 -23
  18. executorlib-1.2.0/tests/test_cache_backend_execute.py +0 -146
  19. executorlib-1.2.0/tests/test_cache_fileexecutor_mpi.py +0 -44
  20. executorlib-1.2.0/tests/test_cache_fileexecutor_serial.py +0 -207
  21. executorlib-1.2.0/tests/test_fluxclusterexecutor.py +0 -48
  22. executorlib-1.2.0/tests/test_fluxjobexecutor.py +0 -175
  23. executorlib-1.2.0/tests/test_fluxjobexecutor_plot.py +0 -179
  24. executorlib-1.2.0/tests/test_fluxpythonspawner.py +0 -160
  25. executorlib-1.2.0/tests/test_integration_pyiron_workflow.py +0 -237
  26. executorlib-1.2.0/tests/test_interactive_dependencies.py +0 -143
  27. executorlib-1.2.0/tests/test_interactive_slurmspawner.py +0 -61
  28. executorlib-1.2.0/tests/test_mpiexecspawner.py +0 -543
  29. executorlib-1.2.0/tests/test_singlenodeexecutor_cache.py +0 -48
  30. executorlib-1.2.0/tests/test_singlenodeexecutor_dependencies.py +0 -363
  31. executorlib-1.2.0/tests/test_singlenodeexecutor_mpi.py +0 -138
  32. executorlib-1.2.0/tests/test_singlenodeexecutor_noblock.py +0 -77
  33. executorlib-1.2.0/tests/test_singlenodeexecutor_plot_dependency.py +0 -291
  34. executorlib-1.2.0/tests/test_singlenodeexecutor_resize.py +0 -80
  35. executorlib-1.2.0/tests/test_singlenodeexecutor_shell_executor.py +0 -160
  36. executorlib-1.2.0/tests/test_singlenodeexecutor_shell_interactive.py +0 -129
  37. executorlib-1.2.0/tests/test_standalone_hdf.py +0 -124
  38. executorlib-1.2.0/tests/test_standalone_inputcheck.py +0 -121
  39. executorlib-1.2.0/tests/test_standalone_interactive_arguments.py +0 -69
  40. executorlib-1.2.0/tests/test_standalone_interactive_backend.py +0 -118
  41. executorlib-1.2.0/tests/test_standalone_interactive_communication.py +0 -103
  42. {executorlib-1.2.0 → executorlib-1.2.1}/LICENSE +0 -0
  43. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/api.py +0 -0
  44. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/backend/__init__.py +0 -0
  45. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/backend/cache_parallel.py +0 -0
  46. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/backend/cache_serial.py +0 -0
  47. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/backend/interactive_parallel.py +0 -0
  48. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/backend/interactive_serial.py +0 -0
  49. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/executor/__init__.py +0 -0
  50. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/executor/base.py +0 -0
  51. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/executor/flux.py +0 -0
  52. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/executor/single.py +0 -0
  53. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/executor/slurm.py +0 -0
  54. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/__init__.py +0 -0
  55. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/cache.py +0 -0
  56. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/command.py +0 -0
  57. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/inputcheck.py +0 -0
  58. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/interactive/__init__.py +0 -0
  59. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/interactive/arguments.py +0 -0
  60. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/interactive/backend.py +0 -0
  61. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/interactive/communication.py +0 -0
  62. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/interactive/spawner.py +0 -0
  63. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/plot.py +0 -0
  64. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/queue.py +0 -0
  65. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/standalone/serialize.py +0 -0
  66. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/__init__.py +0 -0
  67. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/base.py +0 -0
  68. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/__init__.py +0 -0
  69. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/backend.py +0 -0
  70. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/hdf.py +0 -0
  71. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/queue_spawner.py +0 -0
  72. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/shared.py +0 -0
  73. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/subprocess_spawner.py +0 -0
  74. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/file/task_scheduler.py +0 -0
  75. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/__init__.py +0 -0
  76. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/blockallocation.py +0 -0
  77. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/dependency.py +0 -0
  78. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/fluxspawner.py +0 -0
  79. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/onetoone.py +0 -0
  80. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/shared.py +0 -0
  81. {executorlib-1.2.0 → executorlib-1.2.1}/executorlib/task_scheduler/interactive/slurmspawner.py +0 -0
@@ -0,0 +1,5 @@
1
+ *.pyc
2
+ .DS_Store
3
+ .coverage
4
+ .idea/
5
+ .vscode/
@@ -1,7 +1,10 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: executorlib
3
- Version: 1.2.0
3
+ Version: 1.2.1
4
4
  Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
5
+ Project-URL: Homepage, https://github.com/pyiron/executorlib
6
+ Project-URL: Documentation, https://executorlib.readthedocs.io
7
+ Project-URL: Repository, https://github.com/pyiron/executorlib
5
8
  Author-email: Jan Janssen <janssen@lanl.gov>
6
9
  License: BSD 3-Clause License
7
10
 
@@ -32,47 +35,43 @@ License: BSD 3-Clause License
32
35
  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33
36
  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34
37
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35
-
36
- Project-URL: Homepage, https://github.com/pyiron/executorlib
37
- Project-URL: Documentation, https://executorlib.readthedocs.io
38
- Project-URL: Repository, https://github.com/pyiron/executorlib
39
- Keywords: high performance computing,hpc,task scheduler,slurm,flux-framework,executor
38
+ License-File: LICENSE
39
+ Keywords: executor,flux-framework,high performance computing,hpc,slurm,task scheduler
40
40
  Classifier: Development Status :: 5 - Production/Stable
41
- Classifier: Topic :: Scientific/Engineering :: Physics
42
- Classifier: License :: OSI Approved :: BSD License
43
41
  Classifier: Intended Audience :: Science/Research
42
+ Classifier: License :: OSI Approved :: BSD License
44
43
  Classifier: Operating System :: OS Independent
45
44
  Classifier: Programming Language :: Python :: 3.9
46
45
  Classifier: Programming Language :: Python :: 3.10
47
46
  Classifier: Programming Language :: Python :: 3.11
48
47
  Classifier: Programming Language :: Python :: 3.12
49
48
  Classifier: Programming Language :: Python :: 3.13
49
+ Classifier: Topic :: Scientific/Engineering :: Physics
50
50
  Requires-Python: <3.14,>=3.9
51
- Description-Content-Type: text/markdown
52
- License-File: LICENSE
53
51
  Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
54
52
  Requires-Dist: pyzmq<=26.4.0,>=25.0.0
53
+ Provides-Extra: all
54
+ Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == 'all'
55
+ Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'all'
56
+ Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == 'all'
57
+ Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'all'
58
+ Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
59
+ Requires-Dist: pysqa==0.2.4; extra == 'all'
55
60
  Provides-Extra: cache
56
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cache"
61
+ Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == 'cache'
62
+ Provides-Extra: cluster
63
+ Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == 'cluster'
64
+ Requires-Dist: pysqa==0.2.4; extra == 'cluster'
57
65
  Provides-Extra: graph
58
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graph"
59
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graph"
66
+ Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graph'
67
+ Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
60
68
  Provides-Extra: graphnotebook
61
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graphnotebook"
62
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graphnotebook"
63
- Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == "graphnotebook"
69
+ Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'graphnotebook'
70
+ Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graphnotebook'
71
+ Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graphnotebook'
64
72
  Provides-Extra: mpi
65
- Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "mpi"
66
- Provides-Extra: cluster
67
- Requires-Dist: pysqa==0.2.4; extra == "cluster"
68
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cluster"
69
- Provides-Extra: all
70
- Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "all"
71
- Requires-Dist: pysqa==0.2.4; extra == "all"
72
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "all"
73
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "all"
74
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "all"
75
- Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == "all"
73
+ Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == 'mpi'
74
+ Description-Content-Type: text/markdown
76
75
 
77
76
  # executorlib
78
77
  [![Pipeline](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml)
@@ -233,6 +232,7 @@ as hierarchical job scheduler within the allocations.
233
232
  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
234
233
  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
235
234
  * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
235
+ * [Test Environment](https://executorlib.readthedocs.io/en/latest/4-developer.html#test-environment)
236
236
  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
237
237
  * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
238
238
  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
@@ -157,6 +157,7 @@ as hierarchical job scheduler within the allocations.
157
157
  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
158
158
  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
159
159
  * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
160
+ * [Test Environment](https://executorlib.readthedocs.io/en/latest/4-developer.html#test-environment)
160
161
  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
161
162
  * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
162
163
  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
@@ -1,4 +1,3 @@
1
- from executorlib._version import get_versions as _get_versions
2
1
  from executorlib.executor.flux import (
3
2
  FluxClusterExecutor,
4
3
  FluxJobExecutor,
@@ -10,6 +9,8 @@ from executorlib.executor.slurm import (
10
9
  )
11
10
  from executorlib.standalone.cache import get_cache_data
12
11
 
12
+ from . import _version
13
+
13
14
  __all__: list[str] = [
14
15
  "get_cache_data",
15
16
  "FluxJobExecutor",
@@ -19,4 +20,4 @@ __all__: list[str] = [
19
20
  "SlurmClusterExecutor",
20
21
  ]
21
22
 
22
- __version__ = _get_versions()["version"]
23
+ __version__ = _version.__version__
@@ -0,0 +1,21 @@
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
6
+ TYPE_CHECKING = False
7
+ if TYPE_CHECKING:
8
+ from typing import Tuple
9
+ from typing import Union
10
+
11
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
12
+ else:
13
+ VERSION_TUPLE = object
14
+
15
+ version: str
16
+ __version__: str
17
+ __version_tuple__: VERSION_TUPLE
18
+ version_tuple: VERSION_TUPLE
19
+
20
+ __version__ = version = '1.2.1'
21
+ __version_tuple__ = version_tuple = (1, 2, 1)
@@ -1,6 +1,11 @@
1
1
  [build-system]
2
- requires = ["cloudpickle", "pyzmq", "setuptools", "versioneer[toml]==0.29"]
3
- build-backend = "setuptools.build_meta"
2
+ requires = [
3
+ "hatchling==1.27.0",
4
+ "hatch-vcs==0.4.0",
5
+ "cloudpickle>=2.0.0,<=3.1.1",
6
+ "pyzmq>=25.0.0,<=26.4.0",
7
+ ]
8
+ build-backend = "hatchling.build"
4
9
 
5
10
  [project]
6
11
  name = "executorlib"
@@ -95,15 +100,24 @@ ignore = [
95
100
  "PLR0912",
96
101
  ]
97
102
 
98
- [tool.setuptools.packages.find]
99
- include = ["executorlib*"]
103
+ [tool.hatch.build]
104
+ include = [
105
+ "executorlib"
106
+ ]
107
+
108
+ [tool.hatch.build.hooks.vcs]
109
+ version-file = "executorlib/_version.py"
110
+
111
+ [tool.hatch.build.targets.sdist]
112
+ include = [
113
+ "executorlib"
114
+ ]
100
115
 
101
- [tool.setuptools.dynamic]
102
- version = {attr = "executorlib.__version__"}
116
+ [tool.hatch.build.targets.wheel]
117
+ packages = [
118
+ "executorlib"
119
+ ]
103
120
 
104
- [tool.versioneer]
105
- VCS = "git"
106
- style = "pep440-pre"
107
- versionfile_source = "executorlib/_version.py"
108
- parentdir_prefix = "executorlib"
109
- tag_prefix = "executorlib-"
121
+ [tool.hatch.version]
122
+ source = "vcs"
123
+ path = "executorlib/_version.py"
@@ -1 +0,0 @@
1
- include LICENSE
@@ -1,239 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: executorlib
3
- Version: 1.2.0
4
- Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
5
- Author-email: Jan Janssen <janssen@lanl.gov>
6
- License: BSD 3-Clause License
7
-
8
- Copyright (c) 2022, Jan Janssen
9
- All rights reserved.
10
-
11
- Redistribution and use in source and binary forms, with or without
12
- modification, are permitted provided that the following conditions are met:
13
-
14
- * Redistributions of source code must retain the above copyright notice, this
15
- list of conditions and the following disclaimer.
16
-
17
- * Redistributions in binary form must reproduce the above copyright notice,
18
- this list of conditions and the following disclaimer in the documentation
19
- and/or other materials provided with the distribution.
20
-
21
- * Neither the name of the copyright holder nor the names of its
22
- contributors may be used to endorse or promote products derived from
23
- this software without specific prior written permission.
24
-
25
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
26
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
28
- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
29
- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30
- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
31
- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
32
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33
- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35
-
36
- Project-URL: Homepage, https://github.com/pyiron/executorlib
37
- Project-URL: Documentation, https://executorlib.readthedocs.io
38
- Project-URL: Repository, https://github.com/pyiron/executorlib
39
- Keywords: high performance computing,hpc,task scheduler,slurm,flux-framework,executor
40
- Classifier: Development Status :: 5 - Production/Stable
41
- Classifier: Topic :: Scientific/Engineering :: Physics
42
- Classifier: License :: OSI Approved :: BSD License
43
- Classifier: Intended Audience :: Science/Research
44
- Classifier: Operating System :: OS Independent
45
- Classifier: Programming Language :: Python :: 3.9
46
- Classifier: Programming Language :: Python :: 3.10
47
- Classifier: Programming Language :: Python :: 3.11
48
- Classifier: Programming Language :: Python :: 3.12
49
- Classifier: Programming Language :: Python :: 3.13
50
- Requires-Python: <3.14,>=3.9
51
- Description-Content-Type: text/markdown
52
- License-File: LICENSE
53
- Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
54
- Requires-Dist: pyzmq<=26.4.0,>=25.0.0
55
- Provides-Extra: cache
56
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cache"
57
- Provides-Extra: graph
58
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graph"
59
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graph"
60
- Provides-Extra: graphnotebook
61
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graphnotebook"
62
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graphnotebook"
63
- Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == "graphnotebook"
64
- Provides-Extra: mpi
65
- Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "mpi"
66
- Provides-Extra: cluster
67
- Requires-Dist: pysqa==0.2.4; extra == "cluster"
68
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cluster"
69
- Provides-Extra: all
70
- Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "all"
71
- Requires-Dist: pysqa==0.2.4; extra == "all"
72
- Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "all"
73
- Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "all"
74
- Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "all"
75
- Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == "all"
76
-
77
- # executorlib
78
- [![Pipeline](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml)
79
- [![codecov](https://codecov.io/gh/pyiron/executorlib/graph/badge.svg?token=KFIO3R08H3)](https://codecov.io/gh/pyiron/executorlib)
80
- [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD)
81
- [![status](https://joss.theoj.org/papers/7a4bced702e5ea10e4630bc4d5dbaad2/status.svg)](https://joss.theoj.org/papers/7a4bced702e5ea10e4630bc4d5dbaad2)
82
- ![GitHub Repo stars](https://img.shields.io/github/stars/pyiron/executorlib)
83
-
84
- Up-scale python functions for high performance computing (HPC) with executorlib.
85
-
86
- ## Key Features
87
- * **Up-scale your Python functions beyond a single computer.** - executorlib extends the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
88
- from the Python standard library and combines it with job schedulers for high performance computing (HPC) including
89
- the [Simple Linux Utility for Resource Management (SLURM)](https://slurm.schedmd.com) and [flux](http://flux-framework.org).
90
- With this combination executorlib allows users to distribute their Python functions over multiple compute nodes.
91
- * **Parallelize your Python program one function at a time** - executorlib allows users to assign dedicated computing
92
- resources like CPU cores, threads or GPUs to one Python function call at a time. So you can accelerate your Python
93
- code function by function.
94
- * **Permanent caching of intermediate results to accelerate rapid prototyping** - To accelerate the development of
95
- machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for
96
- iterative development in interactive environments like jupyter notebooks.
97
-
98
- ## Examples
99
- The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
100
- with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the
101
- [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) for parallel
102
- execution of Python functions on a single computer. executorlib extends this functionality to distribute Python
103
- functions over multiple computers within a high performance computing (HPC) cluster. This can be either achieved by
104
- submitting each function as individual job to the HPC job scheduler with an [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html) -
105
- or by requesting a job from the HPC cluster and then distribute the Python functions within this job with an
106
- [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html). Finally, to accelerate the
107
- development process executorlib also provides a [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html) -
108
- to use the executorlib functionality on a laptop, workstation or single compute node for testing. Starting with the
109
- [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html):
110
- ```python
111
- from executorlib import SingleNodeExecutor
112
-
113
-
114
- with SingleNodeExecutor() as exe:
115
- future_lst = [exe.submit(sum, [i, i]) for i in range(1, 5)]
116
- print([f.result() for f in future_lst])
117
- ```
118
- In the same way executorlib can also execute Python functions which use additional computing resources, like multiple
119
- CPU cores, CPU threads or GPUs. For example if the Python function internally uses the Message Passing Interface (MPI)
120
- via the [mpi4py](https://mpi4py.readthedocs.io) Python libary:
121
- ```python
122
- from executorlib import SingleNodeExecutor
123
-
124
-
125
- def calc(i):
126
- from mpi4py import MPI
127
-
128
- size = MPI.COMM_WORLD.Get_size()
129
- rank = MPI.COMM_WORLD.Get_rank()
130
- return i, size, rank
131
-
132
-
133
- with SingleNodeExecutor() as exe:
134
- fs = exe.submit(calc, 3, resource_dict={"cores": 2})
135
- print(fs.result())
136
- ```
137
- The additional `resource_dict` parameter defines the computing resources allocated to the execution of the submitted
138
- Python function. In addition to the compute cores `cores`, the resource dictionary can also define the threads per core
139
- as `threads_per_core`, the GPUs per core as `gpus_per_core`, the working directory with `cwd`, the option to use the
140
- OpenMPI oversubscribe feature with `openmpi_oversubscribe` and finally for the [Simple Linux Utility for Resource
141
- Management (SLURM)](https://slurm.schedmd.com) queuing system the option to provide additional command line arguments
142
- with the `slurm_cmd_args` parameter - [resource dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
143
- This flexibility to assign computing resources on a per-function-call basis simplifies the up-scaling of Python programs.
144
- Only the part of the Python functions which benefit from parallel execution are implemented as MPI parallel Python
145
- funtions, while the rest of the program remains serial.
146
-
147
- The same function can be submitted to the [SLURM](https://slurm.schedmd.com) job scheduler by replacing the
148
- `SingleNodeExecutor` with the `SlurmClusterExecutor`. The rest of the example remains the same, which highlights how
149
- executorlib accelerates the rapid prototyping and up-scaling of HPC Python programs.
150
- ```python
151
- from executorlib import SlurmClusterExecutor
152
-
153
-
154
- def calc(i):
155
- from mpi4py import MPI
156
-
157
- size = MPI.COMM_WORLD.Get_size()
158
- rank = MPI.COMM_WORLD.Get_rank()
159
- return i, size, rank
160
-
161
-
162
- with SlurmClusterExecutor() as exe:
163
- fs = exe.submit(calc, 3, resource_dict={"cores": 2})
164
- print(fs.result())
165
- ```
166
- In this case the [Python simple queuing system adapter (pysqa)](https://pysqa.readthedocs.io) is used to submit the
167
- `calc()` function to the [SLURM](https://slurm.schedmd.com) job scheduler and request an allocation with two CPU cores
168
- for the execution of the function - [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html). In the background the [sbatch](https://slurm.schedmd.com/sbatch.html)
169
- command is used to request the allocation to execute the Python function.
170
-
171
- Within a given [SLURM](https://slurm.schedmd.com) job executorlib can also be used to assign a subset of the
172
- available computing resources to execute a given Python function. In terms of the [SLURM](https://slurm.schedmd.com)
173
- commands, this functionality internally uses the [srun](https://slurm.schedmd.com/srun.html) command to receive a subset
174
- of the resources of a given queuing system allocation.
175
- ```python
176
- from executorlib import SlurmJobExecutor
177
-
178
-
179
- def calc(i):
180
- from mpi4py import MPI
181
-
182
- size = MPI.COMM_WORLD.Get_size()
183
- rank = MPI.COMM_WORLD.Get_rank()
184
- return i, size, rank
185
-
186
-
187
- with SlurmJobExecutor() as exe:
188
- fs = exe.submit(calc, 3, resource_dict={"cores": 2})
189
- print(fs.result())
190
- ```
191
- In addition, to support for [SLURM](https://slurm.schedmd.com) executorlib also provides support for the hierarchical
192
- [flux](http://flux-framework.org) job scheduler. The [flux](http://flux-framework.org) job scheduler is developed at
193
- [Larwence Livermore National Laboratory](https://computing.llnl.gov/projects/flux-building-framework-resource-management)
194
- to address the needs for the up-coming generation of Exascale computers. Still even on traditional HPC clusters the
195
- hierarchical approach of the [flux](http://flux-framework.org) is beneficial to distribute hundreds of tasks within a
196
- given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primary job scheduler of your HPC, it is
197
- recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
198
- as hierarchical job scheduler within the allocations.
199
-
200
- ## Documentation
201
- * [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
202
- * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
203
- * [MPI Support](https://executorlib.readthedocs.io/en/latest/installation.html#mpi-support)
204
- * [Caching](https://executorlib.readthedocs.io/en/latest/installation.html#caching)
205
- * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-cluster-executor)
206
- * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-job-executor)
207
- * [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
208
- * [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
209
- * [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html)
210
- * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
211
- * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
212
- * [Performance Optimization](https://executorlib.readthedocs.io/en/latest/1-single-node.html#performance-optimization)
213
- * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
214
- * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
215
- * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)
216
- * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html)
217
- * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
218
- * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
219
- * [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux)
220
- * [Application](https://executorlib.readthedocs.io/en/latest/application.html)
221
- * [GPAW](https://executorlib.readthedocs.io/en/latest/5-1-gpaw.html)
222
- * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/5-2-quantum-espresso.html)
223
- * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
224
- * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
225
- * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
226
- * [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
227
- * [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
228
- * [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
229
- * [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
230
- * [Support & Contribution](https://executorlib.readthedocs.io/en/latest/4-developer.html)
231
- * [Issues](https://executorlib.readthedocs.io/en/latest/4-developer.html#issues)
232
- * [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
233
- * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
234
- * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
235
- * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
236
- * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
237
- * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
238
- * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
239
- * [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
@@ -1,21 +0,0 @@
1
-
2
- # This file was generated by 'versioneer.py' (0.29) from
3
- # revision-control system data, or from the parent directory name of an
4
- # unpacked source archive. Distribution tarballs contain a pre-generated copy
5
- # of this file.
6
-
7
- import json
8
-
9
- version_json = '''
10
- {
11
- "date": "2025-05-08T09:35:44+0200",
12
- "dirty": true,
13
- "error": null,
14
- "full-revisionid": "bac1a92bf7322312783148300a78d9084e985f1c",
15
- "version": "1.2.0"
16
- }
17
- ''' # END VERSION_JSON
18
-
19
-
20
- def get_versions():
21
- return json.loads(version_json)
@@ -1,77 +0,0 @@
1
- LICENSE
2
- MANIFEST.in
3
- README.md
4
- pyproject.toml
5
- setup.py
6
- executorlib/__init__.py
7
- executorlib/_version.py
8
- executorlib/api.py
9
- executorlib.egg-info/PKG-INFO
10
- executorlib.egg-info/SOURCES.txt
11
- executorlib.egg-info/dependency_links.txt
12
- executorlib.egg-info/requires.txt
13
- executorlib.egg-info/top_level.txt
14
- executorlib/backend/__init__.py
15
- executorlib/backend/cache_parallel.py
16
- executorlib/backend/cache_serial.py
17
- executorlib/backend/interactive_parallel.py
18
- executorlib/backend/interactive_serial.py
19
- executorlib/executor/__init__.py
20
- executorlib/executor/base.py
21
- executorlib/executor/flux.py
22
- executorlib/executor/single.py
23
- executorlib/executor/slurm.py
24
- executorlib/standalone/__init__.py
25
- executorlib/standalone/cache.py
26
- executorlib/standalone/command.py
27
- executorlib/standalone/inputcheck.py
28
- executorlib/standalone/plot.py
29
- executorlib/standalone/queue.py
30
- executorlib/standalone/serialize.py
31
- executorlib/standalone/interactive/__init__.py
32
- executorlib/standalone/interactive/arguments.py
33
- executorlib/standalone/interactive/backend.py
34
- executorlib/standalone/interactive/communication.py
35
- executorlib/standalone/interactive/spawner.py
36
- executorlib/task_scheduler/__init__.py
37
- executorlib/task_scheduler/base.py
38
- executorlib/task_scheduler/file/__init__.py
39
- executorlib/task_scheduler/file/backend.py
40
- executorlib/task_scheduler/file/hdf.py
41
- executorlib/task_scheduler/file/queue_spawner.py
42
- executorlib/task_scheduler/file/shared.py
43
- executorlib/task_scheduler/file/subprocess_spawner.py
44
- executorlib/task_scheduler/file/task_scheduler.py
45
- executorlib/task_scheduler/interactive/__init__.py
46
- executorlib/task_scheduler/interactive/blockallocation.py
47
- executorlib/task_scheduler/interactive/dependency.py
48
- executorlib/task_scheduler/interactive/fluxspawner.py
49
- executorlib/task_scheduler/interactive/onetoone.py
50
- executorlib/task_scheduler/interactive/shared.py
51
- executorlib/task_scheduler/interactive/slurmspawner.py
52
- tests/test_backend_interactive_serial.py
53
- tests/test_base_executor_queue.py
54
- tests/test_cache_backend_execute.py
55
- tests/test_cache_fileexecutor_mpi.py
56
- tests/test_cache_fileexecutor_serial.py
57
- tests/test_fluxclusterexecutor.py
58
- tests/test_fluxjobexecutor.py
59
- tests/test_fluxjobexecutor_plot.py
60
- tests/test_fluxpythonspawner.py
61
- tests/test_integration_pyiron_workflow.py
62
- tests/test_interactive_dependencies.py
63
- tests/test_interactive_slurmspawner.py
64
- tests/test_mpiexecspawner.py
65
- tests/test_singlenodeexecutor_cache.py
66
- tests/test_singlenodeexecutor_dependencies.py
67
- tests/test_singlenodeexecutor_mpi.py
68
- tests/test_singlenodeexecutor_noblock.py
69
- tests/test_singlenodeexecutor_plot_dependency.py
70
- tests/test_singlenodeexecutor_resize.py
71
- tests/test_singlenodeexecutor_shell_executor.py
72
- tests/test_singlenodeexecutor_shell_interactive.py
73
- tests/test_standalone_hdf.py
74
- tests/test_standalone_inputcheck.py
75
- tests/test_standalone_interactive_arguments.py
76
- tests/test_standalone_interactive_backend.py
77
- tests/test_standalone_interactive_communication.py
@@ -1,29 +0,0 @@
1
- cloudpickle<=3.1.1,>=2.0.0
2
- pyzmq<=26.4.0,>=25.0.0
3
-
4
- [all]
5
- mpi4py<=4.0.1,>=3.1.4
6
- pysqa==0.2.4
7
- h5py<=3.13.0,>=3.6.0
8
- pygraphviz<=1.14,>=1.10
9
- networkx<=3.4.2,>=2.8.8
10
- ipython<=9.0.2,>=7.33.0
11
-
12
- [cache]
13
- h5py<=3.13.0,>=3.6.0
14
-
15
- [cluster]
16
- pysqa==0.2.4
17
- h5py<=3.13.0,>=3.6.0
18
-
19
- [graph]
20
- pygraphviz<=1.14,>=1.10
21
- networkx<=3.4.2,>=2.8.8
22
-
23
- [graphnotebook]
24
- pygraphviz<=1.14,>=1.10
25
- networkx<=3.4.2,>=2.8.8
26
- ipython<=9.0.2,>=7.33.0
27
-
28
- [mpi]
29
- mpi4py<=4.0.1,>=3.1.4
@@ -1 +0,0 @@
1
- executorlib
@@ -1,4 +0,0 @@
1
- [egg_info]
2
- tag_build =
3
- tag_date = 0
4
-
@@ -1,8 +0,0 @@
1
- from setuptools import setup
2
-
3
- import versioneer
4
-
5
- setup(
6
- version=versioneer.get_version(),
7
- cmdclass=versioneer.get_cmdclass(),
8
- )