executorlib 1.9.2__tar.gz → 1.9.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- executorlib-1.9.4/.gitignore +218 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/PKG-INFO +52 -23
- {executorlib-1.9.2 → executorlib-1.9.4}/README.md +41 -12
- {executorlib-1.9.2 → executorlib-1.9.4}/pyproject.toml +11 -11
- executorlib-1.9.4/src/executorlib/_version.py +24 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/executor/base.py +12 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/executor/flux.py +9 -9
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/executor/single.py +8 -8
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/executor/slurm.py +8 -8
- executorlib-1.9.4/src/executorlib/standalone/batched.py +39 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/command.py +5 -5
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/hdf.py +11 -5
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/inputcheck.py +54 -1
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/interactive/arguments.py +15 -4
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/interactive/communication.py +6 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/select.py +30 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/serialize.py +12 -0
- executorlib-1.9.4/src/executorlib/standalone/validate.py +117 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/base.py +25 -1
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/backend.py +14 -7
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/shared.py +97 -3
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/spawner_pysqa.py +10 -3
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/dependency.py +44 -22
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/spawner_flux.py +5 -5
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/spawner_pysqa.py +4 -4
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/spawner_slurm.py +4 -4
- executorlib-1.9.2/.gitignore +0 -5
- executorlib-1.9.2/src/executorlib/_version.py +0 -34
- executorlib-1.9.2/src/executorlib/standalone/batched.py +0 -27
- executorlib-1.9.2/src/executorlib/standalone/validate.py +0 -64
- {executorlib-1.9.2 → executorlib-1.9.4}/LICENSE +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/api.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/backend/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/backend/cache_parallel.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/backend/cache_serial.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/backend/interactive_parallel.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/backend/interactive_serial.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/executor/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/error.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/interactive/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/interactive/backend.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/interactive/spawner.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/queue.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/standalone/scheduler.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/spawner_subprocess.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/file/task_scheduler.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/__init__.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/blockallocation.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/dependency_plot.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/onetoone.py +0 -0
- {executorlib-1.9.2 → executorlib-1.9.4}/src/executorlib/task_scheduler/interactive/shared.py +0 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
# .vscode/
|
|
203
|
+
# Temporary file for partial code execution
|
|
204
|
+
tempCodeRunnerFile.py
|
|
205
|
+
|
|
206
|
+
# Ruff stuff:
|
|
207
|
+
.ruff_cache/
|
|
208
|
+
|
|
209
|
+
# PyPI configuration file
|
|
210
|
+
.pypirc
|
|
211
|
+
|
|
212
|
+
# Marimo
|
|
213
|
+
marimo/_static/
|
|
214
|
+
marimo/_lsp/
|
|
215
|
+
__marimo__/
|
|
216
|
+
|
|
217
|
+
# Streamlit
|
|
218
|
+
.streamlit/secrets.toml
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: executorlib
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.4
|
|
4
4
|
Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
5
5
|
Project-URL: Homepage, https://github.com/pyiron/executorlib
|
|
6
6
|
Project-URL: Documentation, https://executorlib.readthedocs.io
|
|
@@ -36,7 +36,7 @@ License: BSD 3-Clause License
|
|
|
36
36
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
37
37
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
38
38
|
License-File: LICENSE
|
|
39
|
-
Keywords: executor,flux-framework,high performance computing,hpc,slurm,task scheduler
|
|
39
|
+
Keywords: concurrent.futures,distributed computing,executor,flux-framework,gpu,high performance computing,hpc,mpi4py,parallel computing,python,scientific computing,slurm,task scheduler
|
|
40
40
|
Classifier: Development Status :: 5 - Production/Stable
|
|
41
41
|
Classifier: Intended Audience :: Science/Research
|
|
42
42
|
Classifier: License :: OSI Approved :: BSD License
|
|
@@ -50,18 +50,18 @@ Requires-Python: <3.14,>3.9
|
|
|
50
50
|
Requires-Dist: cloudpickle<=3.1.2,>=2.0.0
|
|
51
51
|
Requires-Dist: pyzmq<=27.1.0,>=25.0.0
|
|
52
52
|
Provides-Extra: all
|
|
53
|
-
Requires-Dist: h5py<=3.
|
|
53
|
+
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'all'
|
|
54
54
|
Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'all'
|
|
55
|
-
Requires-Dist: mpi4py<=4.1.
|
|
55
|
+
Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
|
|
56
56
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
|
|
57
|
-
Requires-Dist: pydantic<=2.
|
|
57
|
+
Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'all'
|
|
58
58
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
|
|
59
|
-
Requires-Dist: pysqa==0.
|
|
59
|
+
Requires-Dist: pysqa==0.4.2; extra == 'all'
|
|
60
60
|
Provides-Extra: cache
|
|
61
|
-
Requires-Dist: h5py<=3.
|
|
61
|
+
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
|
|
62
62
|
Provides-Extra: cluster
|
|
63
|
-
Requires-Dist: h5py<=3.
|
|
64
|
-
Requires-Dist: pysqa==0.
|
|
63
|
+
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
|
|
64
|
+
Requires-Dist: pysqa==0.4.2; extra == 'cluster'
|
|
65
65
|
Provides-Extra: graph
|
|
66
66
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
|
|
67
67
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
|
|
@@ -70,9 +70,9 @@ Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'graphnotebook'
|
|
|
70
70
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graphnotebook'
|
|
71
71
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graphnotebook'
|
|
72
72
|
Provides-Extra: mpi
|
|
73
|
-
Requires-Dist: mpi4py<=4.1.
|
|
73
|
+
Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'mpi'
|
|
74
74
|
Provides-Extra: validation
|
|
75
|
-
Requires-Dist: pydantic<=2.
|
|
75
|
+
Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'validation'
|
|
76
76
|
Description-Content-Type: text/markdown
|
|
77
77
|
|
|
78
78
|
# executorlib
|
|
@@ -96,6 +96,26 @@ Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
|
96
96
|
machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for
|
|
97
97
|
iterative development in interactive environments like jupyter notebooks.
|
|
98
98
|
|
|
99
|
+
## Why executorlib?
|
|
100
|
+
executorlib is the lightest path to take *existing* Python functions and scale them across high performance computing
|
|
101
|
+
(HPC) nodes — with per-function-call resource control and native [SLURM](https://slurm.schedmd.com) and
|
|
102
|
+
[flux](http://flux-framework.org) integration — without rewriting your code into a new paradigm. It extends the standard
|
|
103
|
+
library [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects) you already
|
|
104
|
+
know, rather than asking you to adopt a new data, actor, or workflow model.
|
|
105
|
+
|
|
106
|
+
| | executorlib | [Futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
|
|
107
|
+
|---|---|---|---|---|---|
|
|
108
|
+
| Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️ | ❌ |
|
|
109
|
+
| Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |
|
|
110
|
+
| Native HPC scheduler (SLURM/flux) | ✅ | ❌ | ⚠️ | ✅ | ⚠️ |
|
|
111
|
+
| MPI-parallel functions | ✅ | ❌ | ⚠️ | ⚠️ | ⚠️ |
|
|
112
|
+
| Caching of results | ✅ | ❌ | ⚠️ | ✅ | ❌ |
|
|
113
|
+
| Setup / learning overhead | Low | Very low | Medium | Medium | Medium |
|
|
114
|
+
|
|
115
|
+
✅ first-class · ⚠️ possible via add-on/config · ❌ not supported. See the full
|
|
116
|
+
[comparison: when to use which](https://executorlib.readthedocs.io/en/latest/comparison.html) for honest guidance on
|
|
117
|
+
when another tool is the better fit.
|
|
118
|
+
|
|
99
119
|
## Examples
|
|
100
120
|
The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
|
|
101
121
|
with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the
|
|
@@ -207,6 +227,13 @@ as hierarchical job scheduler within the allocations.
|
|
|
207
227
|
* [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-job-executor)
|
|
208
228
|
* [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
|
|
209
229
|
* [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
|
|
230
|
+
* [Comparison](https://executorlib.readthedocs.io/en/latest/comparison.html#comparison)
|
|
231
|
+
* [At a glace](https://executorlib.readthedocs.io/en/latest/comparison.html#at-a-glance)
|
|
232
|
+
* [Concurrent futures](https://executorlib.readthedocs.io/en/latest/comparison.html#concurrent-futures-the-python-standard-library)
|
|
233
|
+
* [Dask](https://executorlib.readthedocs.io/en/latest/comparison.html#dask)
|
|
234
|
+
* [Parsl](https://executorlib.readthedocs.io/en/latest/comparison.html#parsl)
|
|
235
|
+
* [Ray](https://executorlib.readthedocs.io/en/latest/comparison.html#ray)
|
|
236
|
+
* [Choose executorlib when](https://executorlib.readthedocs.io/en/latest/comparison.html#choose-executorlib-when)
|
|
210
237
|
* [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html)
|
|
211
238
|
* [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
|
|
212
239
|
* [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
|
|
@@ -221,23 +248,25 @@ as hierarchical job scheduler within the allocations.
|
|
|
221
248
|
* [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
|
|
222
249
|
* [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux)
|
|
223
250
|
* [Application](https://executorlib.readthedocs.io/en/latest/application.html)
|
|
224
|
-
* [GPAW](https://executorlib.readthedocs.io/en/latest/
|
|
225
|
-
* [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/
|
|
251
|
+
* [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
|
|
252
|
+
* [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
|
|
226
253
|
* [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
|
|
227
254
|
* [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
|
|
228
255
|
* [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
|
|
229
256
|
* [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
|
|
257
|
+
* [Test Coverage for Integration Tests](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#test-coverage-for-integration-tests)
|
|
230
258
|
* [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
|
|
231
259
|
* [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
|
|
232
260
|
* [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
|
|
233
|
-
* [Support & Contribution](https://executorlib.readthedocs.io/en/latest/
|
|
234
|
-
* [Issues](https://executorlib.readthedocs.io/en/latest/
|
|
235
|
-
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/
|
|
236
|
-
* [License](https://executorlib.readthedocs.io/en/latest/
|
|
237
|
-
* [Modules](https://executorlib.readthedocs.io/en/latest/
|
|
238
|
-
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/
|
|
239
|
-
* [
|
|
240
|
-
* [
|
|
241
|
-
* [
|
|
242
|
-
* [External
|
|
261
|
+
* [Support & Contribution](https://executorlib.readthedocs.io/en/latest/5-developer.html)
|
|
262
|
+
* [Issues](https://executorlib.readthedocs.io/en/latest/5-developer.html#issues)
|
|
263
|
+
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/5-developer.html#pull-requests)
|
|
264
|
+
* [License](https://executorlib.readthedocs.io/en/latest/5-developer.html#license)
|
|
265
|
+
* [Modules](https://executorlib.readthedocs.io/en/latest/5-developer.html#modules)
|
|
266
|
+
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/5-developer.html#interface-class-hierarchy)
|
|
267
|
+
* [Execution Flow](https://executorlib.readthedocs.io/en/latest/5-developer.html#execution-flow)
|
|
268
|
+
* [Test Environment](https://executorlib.readthedocs.io/en/latest/5-developer.html#test-environment)
|
|
269
|
+
* [Communication](https://executorlib.readthedocs.io/en/latest/5-developer.html#communication)
|
|
270
|
+
* [External Libraries](https://executorlib.readthedocs.io/en/latest/5-developer.html#external-libraries)
|
|
271
|
+
* [External Executables](https://executorlib.readthedocs.io/en/latest/5-developer.html#external-executables)
|
|
243
272
|
* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
|
|
@@ -19,6 +19,26 @@ Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
|
19
19
|
machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for
|
|
20
20
|
iterative development in interactive environments like jupyter notebooks.
|
|
21
21
|
|
|
22
|
+
## Why executorlib?
|
|
23
|
+
executorlib is the lightest path to take *existing* Python functions and scale them across high performance computing
|
|
24
|
+
(HPC) nodes — with per-function-call resource control and native [SLURM](https://slurm.schedmd.com) and
|
|
25
|
+
[flux](http://flux-framework.org) integration — without rewriting your code into a new paradigm. It extends the standard
|
|
26
|
+
library [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects) you already
|
|
27
|
+
know, rather than asking you to adopt a new data, actor, or workflow model.
|
|
28
|
+
|
|
29
|
+
| | executorlib | [Futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
|
|
30
|
+
|---|---|---|---|---|---|
|
|
31
|
+
| Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️ | ❌ |
|
|
32
|
+
| Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |
|
|
33
|
+
| Native HPC scheduler (SLURM/flux) | ✅ | ❌ | ⚠️ | ✅ | ⚠️ |
|
|
34
|
+
| MPI-parallel functions | ✅ | ❌ | ⚠️ | ⚠️ | ⚠️ |
|
|
35
|
+
| Caching of results | ✅ | ❌ | ⚠️ | ✅ | ❌ |
|
|
36
|
+
| Setup / learning overhead | Low | Very low | Medium | Medium | Medium |
|
|
37
|
+
|
|
38
|
+
✅ first-class · ⚠️ possible via add-on/config · ❌ not supported. See the full
|
|
39
|
+
[comparison: when to use which](https://executorlib.readthedocs.io/en/latest/comparison.html) for honest guidance on
|
|
40
|
+
when another tool is the better fit.
|
|
41
|
+
|
|
22
42
|
## Examples
|
|
23
43
|
The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
|
|
24
44
|
with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the
|
|
@@ -130,6 +150,13 @@ as hierarchical job scheduler within the allocations.
|
|
|
130
150
|
* [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-job-executor)
|
|
131
151
|
* [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
|
|
132
152
|
* [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
|
|
153
|
+
* [Comparison](https://executorlib.readthedocs.io/en/latest/comparison.html#comparison)
|
|
154
|
+
* [At a glace](https://executorlib.readthedocs.io/en/latest/comparison.html#at-a-glance)
|
|
155
|
+
* [Concurrent futures](https://executorlib.readthedocs.io/en/latest/comparison.html#concurrent-futures-the-python-standard-library)
|
|
156
|
+
* [Dask](https://executorlib.readthedocs.io/en/latest/comparison.html#dask)
|
|
157
|
+
* [Parsl](https://executorlib.readthedocs.io/en/latest/comparison.html#parsl)
|
|
158
|
+
* [Ray](https://executorlib.readthedocs.io/en/latest/comparison.html#ray)
|
|
159
|
+
* [Choose executorlib when](https://executorlib.readthedocs.io/en/latest/comparison.html#choose-executorlib-when)
|
|
133
160
|
* [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html)
|
|
134
161
|
* [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
|
|
135
162
|
* [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
|
|
@@ -144,23 +171,25 @@ as hierarchical job scheduler within the allocations.
|
|
|
144
171
|
* [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
|
|
145
172
|
* [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux)
|
|
146
173
|
* [Application](https://executorlib.readthedocs.io/en/latest/application.html)
|
|
147
|
-
* [GPAW](https://executorlib.readthedocs.io/en/latest/
|
|
148
|
-
* [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/
|
|
174
|
+
* [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
|
|
175
|
+
* [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
|
|
149
176
|
* [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
|
|
150
177
|
* [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
|
|
151
178
|
* [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
|
|
152
179
|
* [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
|
|
180
|
+
* [Test Coverage for Integration Tests](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#test-coverage-for-integration-tests)
|
|
153
181
|
* [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
|
|
154
182
|
* [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
|
|
155
183
|
* [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
|
|
156
|
-
* [Support & Contribution](https://executorlib.readthedocs.io/en/latest/
|
|
157
|
-
* [Issues](https://executorlib.readthedocs.io/en/latest/
|
|
158
|
-
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/
|
|
159
|
-
* [License](https://executorlib.readthedocs.io/en/latest/
|
|
160
|
-
* [Modules](https://executorlib.readthedocs.io/en/latest/
|
|
161
|
-
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/
|
|
162
|
-
* [
|
|
163
|
-
* [
|
|
164
|
-
* [
|
|
165
|
-
* [External
|
|
184
|
+
* [Support & Contribution](https://executorlib.readthedocs.io/en/latest/5-developer.html)
|
|
185
|
+
* [Issues](https://executorlib.readthedocs.io/en/latest/5-developer.html#issues)
|
|
186
|
+
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/5-developer.html#pull-requests)
|
|
187
|
+
* [License](https://executorlib.readthedocs.io/en/latest/5-developer.html#license)
|
|
188
|
+
* [Modules](https://executorlib.readthedocs.io/en/latest/5-developer.html#modules)
|
|
189
|
+
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/5-developer.html#interface-class-hierarchy)
|
|
190
|
+
* [Execution Flow](https://executorlib.readthedocs.io/en/latest/5-developer.html#execution-flow)
|
|
191
|
+
* [Test Environment](https://executorlib.readthedocs.io/en/latest/5-developer.html#test-environment)
|
|
192
|
+
* [Communication](https://executorlib.readthedocs.io/en/latest/5-developer.html#communication)
|
|
193
|
+
* [External Libraries](https://executorlib.readthedocs.io/en/latest/5-developer.html#external-libraries)
|
|
194
|
+
* [External Executables](https://executorlib.readthedocs.io/en/latest/5-developer.html#external-executables)
|
|
166
195
|
* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
requires = [
|
|
3
|
-
"hatchling>=1.27.0,<=1.
|
|
3
|
+
"hatchling>=1.27.0,<=1.30.1",
|
|
4
4
|
"hatch-vcs==0.5.0",
|
|
5
5
|
"cloudpickle>=2.0.0,<=3.1.2",
|
|
6
6
|
"pyzmq>=25.0.0,<=27.1.0",
|
|
@@ -15,7 +15,7 @@ authors = [
|
|
|
15
15
|
]
|
|
16
16
|
readme = "README.md"
|
|
17
17
|
license = { file = "LICENSE" }
|
|
18
|
-
keywords = ["high performance computing", "hpc", "task scheduler", "slurm", "flux-framework", "executor"]
|
|
18
|
+
keywords = ["high performance computing", "hpc", "task scheduler", "slurm", "flux-framework", "executor", "python", "parallel computing", "distributed computing", "scientific computing", "concurrent.futures", "mpi4py", "gpu"]
|
|
19
19
|
requires-python = ">3.9, <3.14"
|
|
20
20
|
classifiers = [
|
|
21
21
|
"Development Status :: 5 - Production/Stable",
|
|
@@ -40,7 +40,7 @@ Documentation = "https://executorlib.readthedocs.io"
|
|
|
40
40
|
Repository = "https://github.com/pyiron/executorlib"
|
|
41
41
|
|
|
42
42
|
[project.optional-dependencies]
|
|
43
|
-
cache = ["h5py>=3.6.0,<=3.
|
|
43
|
+
cache = ["h5py>=3.6.0,<=3.16.0"]
|
|
44
44
|
graph = [
|
|
45
45
|
"pygraphviz>=1.10,<=1.14",
|
|
46
46
|
"networkx>=2.8.8,<=3.6.1",
|
|
@@ -50,19 +50,19 @@ graphnotebook = [
|
|
|
50
50
|
"networkx>=2.8.8,<=3.6.1",
|
|
51
51
|
"ipython>=7.33.0,<=9.9.0",
|
|
52
52
|
]
|
|
53
|
-
mpi = ["mpi4py>=3.1.4,<=4.1.
|
|
53
|
+
mpi = ["mpi4py>=3.1.4,<=4.1.2"]
|
|
54
54
|
cluster = [
|
|
55
|
-
"pysqa==0.
|
|
56
|
-
"h5py>=3.6.0,<=3.
|
|
55
|
+
"pysqa==0.4.2",
|
|
56
|
+
"h5py>=3.6.0,<=3.16.0",
|
|
57
57
|
]
|
|
58
58
|
validation = [
|
|
59
|
-
"pydantic>=2.5.3,<=2.
|
|
59
|
+
"pydantic>=2.5.3,<=2.13.4",
|
|
60
60
|
]
|
|
61
61
|
all = [
|
|
62
|
-
"mpi4py>=3.1.4,<=4.1.
|
|
63
|
-
"pysqa==0.
|
|
64
|
-
"h5py>=3.6.0,<=3.
|
|
65
|
-
"pydantic>=2.5.3,<=2.
|
|
62
|
+
"mpi4py>=3.1.4,<=4.1.2",
|
|
63
|
+
"pysqa==0.4.2",
|
|
64
|
+
"h5py>=3.6.0,<=3.16.0",
|
|
65
|
+
"pydantic>=2.5.3,<=2.13.4",
|
|
66
66
|
"pygraphviz>=1.10,<=1.14",
|
|
67
67
|
"networkx>=2.8.8,<=3.6.1",
|
|
68
68
|
"ipython>=7.33.0,<=9.9.0",
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '1.9.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 9, 4)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
@@ -25,10 +25,22 @@ class BaseExecutor(FutureExecutor, ABC):
|
|
|
25
25
|
|
|
26
26
|
@property
|
|
27
27
|
def max_workers(self) -> Optional[int]:
|
|
28
|
+
"""
|
|
29
|
+
Return the number of parallel workers configured for this executor.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Optional[int]: The maximum number of parallel workers, or None if unconstrained.
|
|
33
|
+
"""
|
|
28
34
|
return self._task_scheduler.max_workers
|
|
29
35
|
|
|
30
36
|
@max_workers.setter
|
|
31
37
|
def max_workers(self, max_workers: int):
|
|
38
|
+
"""
|
|
39
|
+
Set the number of parallel workers on the underlying task scheduler.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
max_workers (int): New maximum number of parallel workers.
|
|
43
|
+
"""
|
|
32
44
|
self._task_scheduler.max_workers = max_workers
|
|
33
45
|
|
|
34
46
|
@property
|
|
@@ -51,8 +51,8 @@ class FluxJobExecutor(BaseExecutor):
|
|
|
51
51
|
do not allow other tasks to use the same compute node.
|
|
52
52
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
53
53
|
multiple tasks in one file.
|
|
54
|
-
*
|
|
55
|
-
|
|
54
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
55
|
+
allowed to take in seconds.
|
|
56
56
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
57
57
|
influence the scheduling.
|
|
58
58
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -154,7 +154,7 @@ class FluxJobExecutor(BaseExecutor):
|
|
|
154
154
|
- do not allow other tasks to use the same compute node.
|
|
155
155
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
156
156
|
multiple tasks in one file.
|
|
157
|
-
*
|
|
157
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
158
158
|
function is allowed to take in seconds.
|
|
159
159
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
160
160
|
influence the scheduling.
|
|
@@ -283,8 +283,8 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
283
283
|
do not allow other tasks to use the same compute node.
|
|
284
284
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
285
285
|
multiple tasks in one file.
|
|
286
|
-
*
|
|
287
|
-
|
|
286
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
287
|
+
allowed to take in seconds.
|
|
288
288
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
289
289
|
influence the scheduling.
|
|
290
290
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -380,7 +380,7 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
380
380
|
- do not allow other tasks to use the same compute node.
|
|
381
381
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
382
382
|
multiple tasks in one file.
|
|
383
|
-
*
|
|
383
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
384
384
|
function is allowed to take in seconds.
|
|
385
385
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
386
386
|
influence the scheduling.
|
|
@@ -417,7 +417,7 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
417
417
|
"cwd": None,
|
|
418
418
|
"openmpi_oversubscribe": openmpi_oversubscribe,
|
|
419
419
|
"slurm_cmd_args": [],
|
|
420
|
-
"
|
|
420
|
+
"run_time_max": None,
|
|
421
421
|
}
|
|
422
422
|
if resource_dict is None:
|
|
423
423
|
resource_dict = {}
|
|
@@ -540,8 +540,8 @@ def create_flux_executor(
|
|
|
540
540
|
do not allow other tasks to use the same compute node.
|
|
541
541
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
542
542
|
multiple tasks in one file.
|
|
543
|
-
*
|
|
544
|
-
|
|
543
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
544
|
+
allowed to take in seconds.
|
|
545
545
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
546
546
|
influence the scheduling.
|
|
547
547
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -50,8 +50,8 @@ class SingleNodeExecutor(BaseExecutor):
|
|
|
50
50
|
do not allow other tasks to use the same compute node.
|
|
51
51
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
52
52
|
multiple tasks in one file.
|
|
53
|
-
*
|
|
54
|
-
|
|
53
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
54
|
+
allowed to take in seconds.
|
|
55
55
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
56
56
|
influence the scheduling.
|
|
57
57
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -145,7 +145,7 @@ class SingleNodeExecutor(BaseExecutor):
|
|
|
145
145
|
- do not allow other tasks to use the same compute node.
|
|
146
146
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
147
147
|
multiple tasks in one file.
|
|
148
|
-
*
|
|
148
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
149
149
|
function is allowed to take in seconds.
|
|
150
150
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
151
151
|
influence the scheduling.
|
|
@@ -259,8 +259,8 @@ class TestClusterExecutor(BaseExecutor):
|
|
|
259
259
|
do not allow other tasks to use the same compute node.
|
|
260
260
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
261
261
|
multiple tasks in one file.
|
|
262
|
-
*
|
|
263
|
-
|
|
262
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
263
|
+
allowed to take in seconds.
|
|
264
264
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
265
265
|
influence the scheduling.
|
|
266
266
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -350,7 +350,7 @@ class TestClusterExecutor(BaseExecutor):
|
|
|
350
350
|
- do not allow other tasks to use the same compute node.
|
|
351
351
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
352
352
|
multiple tasks in one file.
|
|
353
|
-
*
|
|
353
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
354
354
|
function is allowed to take in seconds.
|
|
355
355
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
356
356
|
influence the scheduling.
|
|
@@ -479,8 +479,8 @@ def create_single_node_executor(
|
|
|
479
479
|
do not allow other tasks to use the same compute node.
|
|
480
480
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
481
481
|
multiple tasks in one file.
|
|
482
|
-
*
|
|
483
|
-
|
|
482
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
483
|
+
allowed to take in seconds.
|
|
484
484
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
485
485
|
influence the scheduling.
|
|
486
486
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|