executorlib 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {executorlib-1.0.0/executorlib.egg-info → executorlib-1.1.0}/PKG-INFO +4 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/README.md +2 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/__init__.py +5 -10
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/_version.py +3 -3
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/cache_parallel.py +4 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/cache_serial.py +1 -1
- executorlib-1.1.0/executorlib/executor/base.py +118 -0
- {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/flux.py +76 -104
- {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/single.py +33 -43
- {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/slurm.py +71 -90
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/__init__.py +13 -5
- executorlib-1.1.0/executorlib/standalone/cache.py +42 -0
- executorlib-1.0.0/executorlib/base/executor.py → executorlib-1.1.0/executorlib/task_scheduler/base.py +2 -1
- {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/backend.py +2 -2
- {executorlib-1.0.0/executorlib/standalone → executorlib-1.1.0/executorlib/task_scheduler/file}/hdf.py +10 -27
- {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/queue_spawner.py +1 -1
- {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/shared.py +1 -1
- executorlib-1.0.0/executorlib/cache/executor.py → executorlib-1.1.0/executorlib/task_scheduler/file/task_scheduler.py +9 -9
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/blockallocation.py +7 -6
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/dependency.py +7 -5
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/onetoone.py +6 -6
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/shared.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0/executorlib.egg-info}/PKG-INFO +4 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/SOURCES.txt +22 -20
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/requires.txt +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/pyproject.toml +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_base_executor_queue.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_backend_execute.py +3 -3
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_fileexecutor_mpi.py +3 -3
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_fileexecutor_serial.py +8 -8
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxjobexecutor.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxjobexecutor_plot.py +17 -17
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxpythonspawner.py +8 -8
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_interactive_dependencies.py +5 -5
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_interactive_slurmspawner.py +2 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_mpiexecspawner.py +28 -28
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_cache.py +2 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_dependencies.py +2 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_plot_dependency.py +28 -28
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_resize.py +2 -2
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_shell_executor.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_shell_interactive.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_hdf.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_backend.py +1 -1
- {executorlib-1.0.0 → executorlib-1.1.0}/LICENSE +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/MANIFEST.in +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/__init__.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/interactive_parallel.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/interactive_serial.py +0 -0
- {executorlib-1.0.0/executorlib/base → executorlib-1.1.0/executorlib/executor}/__init__.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/command.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/inputcheck.py +0 -0
- {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/standalone/interactive}/__init__.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/arguments.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/backend.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/communication.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/spawner.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/plot.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/queue.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/serialize.py +0 -0
- {executorlib-1.0.0/executorlib/interactive → executorlib-1.1.0/executorlib/task_scheduler}/__init__.py +0 -0
- {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/task_scheduler/file}/__init__.py +0 -0
- {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/subprocess_spawner.py +0 -0
- {executorlib-1.0.0/executorlib/standalone → executorlib-1.1.0/executorlib/task_scheduler}/interactive/__init__.py +0 -0
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/fluxspawner.py +0 -0
- {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/slurmspawner.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/dependency_links.txt +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/top_level.txt +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/setup.cfg +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/setup.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_backend_interactive_serial.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxclusterexecutor.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_integration_pyiron_workflow.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_mpi.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_noblock.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_inputcheck.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_arguments.py +0 -0
- {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_communication.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: executorlib
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
5
5
|
Author-email: Jan Janssen <janssen@lanl.gov>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -51,7 +51,7 @@ Requires-Python: <3.14,>=3.9
|
|
|
51
51
|
Description-Content-Type: text/markdown
|
|
52
52
|
License-File: LICENSE
|
|
53
53
|
Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
|
|
54
|
-
Requires-Dist: pyzmq<=26.
|
|
54
|
+
Requires-Dist: pyzmq<=26.4.0,>=25.0.0
|
|
55
55
|
Provides-Extra: cache
|
|
56
56
|
Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cache"
|
|
57
57
|
Provides-Extra: graph
|
|
@@ -232,6 +232,8 @@ as hierarchical job scheduler within the allocations.
|
|
|
232
232
|
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
|
|
233
233
|
* [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
|
|
234
234
|
* [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
|
|
235
|
+
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
|
|
235
236
|
* [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
|
|
237
|
+
* [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
|
|
236
238
|
* [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
|
|
237
239
|
* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
|
|
@@ -156,6 +156,8 @@ as hierarchical job scheduler within the allocations.
|
|
|
156
156
|
* [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
|
|
157
157
|
* [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
|
|
158
158
|
* [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
|
|
159
|
+
* [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
|
|
159
160
|
* [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
|
|
161
|
+
* [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
|
|
160
162
|
* [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
|
|
161
163
|
* [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
from executorlib._version import get_versions as _get_versions
|
|
2
|
-
from executorlib.
|
|
2
|
+
from executorlib.executor.flux import (
|
|
3
3
|
FluxClusterExecutor,
|
|
4
4
|
FluxJobExecutor,
|
|
5
5
|
)
|
|
6
|
-
from executorlib.
|
|
7
|
-
from executorlib.
|
|
6
|
+
from executorlib.executor.single import SingleNodeExecutor
|
|
7
|
+
from executorlib.executor.slurm import (
|
|
8
8
|
SlurmClusterExecutor,
|
|
9
9
|
SlurmJobExecutor,
|
|
10
10
|
)
|
|
11
|
+
from executorlib.standalone.cache import get_cache_data
|
|
11
12
|
|
|
12
13
|
__all__: list[str] = [
|
|
14
|
+
"get_cache_data",
|
|
13
15
|
"FluxJobExecutor",
|
|
14
16
|
"FluxClusterExecutor",
|
|
15
17
|
"SingleNodeExecutor",
|
|
@@ -17,11 +19,4 @@ __all__: list[str] = [
|
|
|
17
19
|
"SlurmClusterExecutor",
|
|
18
20
|
]
|
|
19
21
|
|
|
20
|
-
try:
|
|
21
|
-
from executorlib.standalone.hdf import get_cache_data
|
|
22
|
-
except ImportError:
|
|
23
|
-
pass
|
|
24
|
-
else:
|
|
25
|
-
__all__ += ["get_cache_data"]
|
|
26
|
-
|
|
27
22
|
__version__ = _get_versions()["version"]
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2025-
|
|
11
|
+
"date": "2025-04-26T14:56:26+0200",
|
|
12
12
|
"dirty": true,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "1.
|
|
14
|
+
"full-revisionid": "093d1fd613b10c43f4ebc4387d2b1c3dbfebf0c0",
|
|
15
|
+
"version": "1.1.0"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import queue
|
|
2
|
+
from concurrent.futures import (
|
|
3
|
+
Executor as FutureExecutor,
|
|
4
|
+
)
|
|
5
|
+
from concurrent.futures import (
|
|
6
|
+
Future,
|
|
7
|
+
)
|
|
8
|
+
from typing import Callable, Optional
|
|
9
|
+
|
|
10
|
+
from executorlib.task_scheduler.base import TaskSchedulerBase
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ExecutorBase(FutureExecutor):
|
|
14
|
+
"""
|
|
15
|
+
Interface class for the executor.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
executor (TaskSchedulerBase): internal executor
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, executor: TaskSchedulerBase):
|
|
22
|
+
self._task_scheduler = executor
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def max_workers(self) -> Optional[int]:
|
|
26
|
+
return self._task_scheduler.max_workers
|
|
27
|
+
|
|
28
|
+
@max_workers.setter
|
|
29
|
+
def max_workers(self, max_workers: int):
|
|
30
|
+
self._task_scheduler.max_workers = max_workers
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def info(self) -> Optional[dict]:
|
|
34
|
+
"""
|
|
35
|
+
Get the information about the executor.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Optional[dict]: Information about the executor.
|
|
39
|
+
"""
|
|
40
|
+
return self._task_scheduler.info
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def future_queue(self) -> Optional[queue.Queue]:
|
|
44
|
+
"""
|
|
45
|
+
Get the future queue.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
queue.Queue: The future queue.
|
|
49
|
+
"""
|
|
50
|
+
return self._task_scheduler.future_queue
|
|
51
|
+
|
|
52
|
+
def submit( # type: ignore
|
|
53
|
+
self,
|
|
54
|
+
fn: Callable,
|
|
55
|
+
/,
|
|
56
|
+
*args,
|
|
57
|
+
resource_dict: Optional[dict] = None,
|
|
58
|
+
**kwargs,
|
|
59
|
+
) -> Future:
|
|
60
|
+
"""
|
|
61
|
+
Submits a callable to be executed with the given arguments.
|
|
62
|
+
|
|
63
|
+
Schedules the callable to be executed as fn(*args, **kwargs) and returns
|
|
64
|
+
a Future instance representing the execution of the callable.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
fn (callable): function to submit for execution
|
|
68
|
+
args: arguments for the submitted function
|
|
69
|
+
kwargs: keyword arguments for the submitted function
|
|
70
|
+
resource_dict (dict): resource dictionary, which defines the resources used for the execution of the
|
|
71
|
+
function. Example resource dictionary: {
|
|
72
|
+
cores: 1,
|
|
73
|
+
threads_per_core: 1,
|
|
74
|
+
gpus_per_worker: 0,
|
|
75
|
+
oversubscribe: False,
|
|
76
|
+
cwd: None,
|
|
77
|
+
executor: None,
|
|
78
|
+
hostname_localhost: False,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
Future: A Future representing the given call.
|
|
83
|
+
"""
|
|
84
|
+
return self._task_scheduler.submit(
|
|
85
|
+
*([fn] + list(args)), resource_dict=resource_dict, **kwargs
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
|
|
89
|
+
"""
|
|
90
|
+
Clean-up the resources associated with the Executor.
|
|
91
|
+
|
|
92
|
+
It is safe to call this method several times. Otherwise, no other
|
|
93
|
+
methods can be called after this one.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
wait (bool): If True then shutdown will not return until all running
|
|
97
|
+
futures have finished executing and the resources used by the
|
|
98
|
+
parallel_executors have been reclaimed.
|
|
99
|
+
cancel_futures (bool): If True then shutdown will cancel all pending
|
|
100
|
+
futures. Futures that are completed or running will not be
|
|
101
|
+
cancelled.
|
|
102
|
+
"""
|
|
103
|
+
self._task_scheduler.shutdown(wait=wait, cancel_futures=cancel_futures)
|
|
104
|
+
|
|
105
|
+
def __len__(self) -> int:
|
|
106
|
+
"""
|
|
107
|
+
Get the length of the executor.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
int: The length of the executor.
|
|
111
|
+
"""
|
|
112
|
+
return len(self._task_scheduler)
|
|
113
|
+
|
|
114
|
+
def __exit__(self, *args, **kwargs) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Exit method called when exiting the context manager.
|
|
117
|
+
"""
|
|
118
|
+
self._task_scheduler.__exit__(*args, **kwargs)
|
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
import contextlib
|
|
2
1
|
from typing import Callable, Optional, Union
|
|
3
2
|
|
|
4
|
-
from executorlib.
|
|
5
|
-
from executorlib.interactive.dependency import DependencyExecutor
|
|
6
|
-
from executorlib.interactive.onetoone import OneTaskPerProcessExecutor
|
|
3
|
+
from executorlib.executor.base import ExecutorBase
|
|
7
4
|
from executorlib.standalone.inputcheck import (
|
|
8
5
|
check_command_line_argument_lst,
|
|
9
6
|
check_init_function,
|
|
@@ -13,15 +10,14 @@ from executorlib.standalone.inputcheck import (
|
|
|
13
10
|
check_refresh_rate,
|
|
14
11
|
validate_number_of_cores,
|
|
15
12
|
)
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
)
|
|
13
|
+
from executorlib.task_scheduler.interactive.blockallocation import (
|
|
14
|
+
BlockAllocationTaskScheduler,
|
|
15
|
+
)
|
|
16
|
+
from executorlib.task_scheduler.interactive.dependency import DependencyTaskScheduler
|
|
17
|
+
from executorlib.task_scheduler.interactive.onetoone import OneProcessTaskScheduler
|
|
22
18
|
|
|
23
19
|
|
|
24
|
-
class FluxJobExecutor:
|
|
20
|
+
class FluxJobExecutor(ExecutorBase):
|
|
25
21
|
"""
|
|
26
22
|
The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
|
|
27
23
|
preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
|
|
@@ -69,7 +65,7 @@ class FluxJobExecutor:
|
|
|
69
65
|
Examples:
|
|
70
66
|
```
|
|
71
67
|
>>> import numpy as np
|
|
72
|
-
>>> from executorlib.
|
|
68
|
+
>>> from executorlib.executor.flux import FluxJobExecutor
|
|
73
69
|
>>>
|
|
74
70
|
>>> def calc(i, j, k):
|
|
75
71
|
>>> from mpi4py import MPI
|
|
@@ -104,27 +100,6 @@ class FluxJobExecutor:
|
|
|
104
100
|
refresh_rate: float = 0.01,
|
|
105
101
|
plot_dependency_graph: bool = False,
|
|
106
102
|
plot_dependency_graph_filename: Optional[str] = None,
|
|
107
|
-
):
|
|
108
|
-
# Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
|
|
109
|
-
pass
|
|
110
|
-
|
|
111
|
-
def __new__(
|
|
112
|
-
cls,
|
|
113
|
-
max_workers: Optional[int] = None,
|
|
114
|
-
cache_directory: Optional[str] = None,
|
|
115
|
-
max_cores: Optional[int] = None,
|
|
116
|
-
resource_dict: Optional[dict] = None,
|
|
117
|
-
flux_executor=None,
|
|
118
|
-
flux_executor_pmi_mode: Optional[str] = None,
|
|
119
|
-
flux_executor_nesting: bool = False,
|
|
120
|
-
flux_log_files: bool = False,
|
|
121
|
-
hostname_localhost: Optional[bool] = None,
|
|
122
|
-
block_allocation: bool = False,
|
|
123
|
-
init_function: Optional[Callable] = None,
|
|
124
|
-
disable_dependencies: bool = False,
|
|
125
|
-
refresh_rate: float = 0.01,
|
|
126
|
-
plot_dependency_graph: bool = False,
|
|
127
|
-
plot_dependency_graph_filename: Optional[str] = None,
|
|
128
103
|
):
|
|
129
104
|
"""
|
|
130
105
|
Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
|
|
@@ -186,7 +161,31 @@ class FluxJobExecutor:
|
|
|
186
161
|
{k: v for k, v in default_resource_dict.items() if k not in resource_dict}
|
|
187
162
|
)
|
|
188
163
|
if not disable_dependencies:
|
|
189
|
-
|
|
164
|
+
super().__init__(
|
|
165
|
+
executor=DependencyTaskScheduler(
|
|
166
|
+
executor=create_flux_executor(
|
|
167
|
+
max_workers=max_workers,
|
|
168
|
+
cache_directory=cache_directory,
|
|
169
|
+
max_cores=max_cores,
|
|
170
|
+
resource_dict=resource_dict,
|
|
171
|
+
flux_executor=flux_executor,
|
|
172
|
+
flux_executor_pmi_mode=flux_executor_pmi_mode,
|
|
173
|
+
flux_executor_nesting=flux_executor_nesting,
|
|
174
|
+
flux_log_files=flux_log_files,
|
|
175
|
+
hostname_localhost=hostname_localhost,
|
|
176
|
+
block_allocation=block_allocation,
|
|
177
|
+
init_function=init_function,
|
|
178
|
+
),
|
|
179
|
+
max_cores=max_cores,
|
|
180
|
+
refresh_rate=refresh_rate,
|
|
181
|
+
plot_dependency_graph=plot_dependency_graph,
|
|
182
|
+
plot_dependency_graph_filename=plot_dependency_graph_filename,
|
|
183
|
+
)
|
|
184
|
+
)
|
|
185
|
+
else:
|
|
186
|
+
check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
|
|
187
|
+
check_refresh_rate(refresh_rate=refresh_rate)
|
|
188
|
+
super().__init__(
|
|
190
189
|
executor=create_flux_executor(
|
|
191
190
|
max_workers=max_workers,
|
|
192
191
|
cache_directory=cache_directory,
|
|
@@ -199,31 +198,11 @@ class FluxJobExecutor:
|
|
|
199
198
|
hostname_localhost=hostname_localhost,
|
|
200
199
|
block_allocation=block_allocation,
|
|
201
200
|
init_function=init_function,
|
|
202
|
-
)
|
|
203
|
-
max_cores=max_cores,
|
|
204
|
-
refresh_rate=refresh_rate,
|
|
205
|
-
plot_dependency_graph=plot_dependency_graph,
|
|
206
|
-
plot_dependency_graph_filename=plot_dependency_graph_filename,
|
|
207
|
-
)
|
|
208
|
-
else:
|
|
209
|
-
check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
|
|
210
|
-
check_refresh_rate(refresh_rate=refresh_rate)
|
|
211
|
-
return create_flux_executor(
|
|
212
|
-
max_workers=max_workers,
|
|
213
|
-
cache_directory=cache_directory,
|
|
214
|
-
max_cores=max_cores,
|
|
215
|
-
resource_dict=resource_dict,
|
|
216
|
-
flux_executor=flux_executor,
|
|
217
|
-
flux_executor_pmi_mode=flux_executor_pmi_mode,
|
|
218
|
-
flux_executor_nesting=flux_executor_nesting,
|
|
219
|
-
flux_log_files=flux_log_files,
|
|
220
|
-
hostname_localhost=hostname_localhost,
|
|
221
|
-
block_allocation=block_allocation,
|
|
222
|
-
init_function=init_function,
|
|
201
|
+
)
|
|
223
202
|
)
|
|
224
203
|
|
|
225
204
|
|
|
226
|
-
class FluxClusterExecutor:
|
|
205
|
+
class FluxClusterExecutor(ExecutorBase):
|
|
227
206
|
"""
|
|
228
207
|
The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
|
|
229
208
|
preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
|
|
@@ -267,7 +246,7 @@ class FluxClusterExecutor:
|
|
|
267
246
|
Examples:
|
|
268
247
|
```
|
|
269
248
|
>>> import numpy as np
|
|
270
|
-
>>> from executorlib.
|
|
249
|
+
>>> from executorlib.executor.flux import FluxClusterExecutor
|
|
271
250
|
>>>
|
|
272
251
|
>>> def calc(i, j, k):
|
|
273
252
|
>>> from mpi4py import MPI
|
|
@@ -299,24 +278,6 @@ class FluxClusterExecutor:
|
|
|
299
278
|
refresh_rate: float = 0.01,
|
|
300
279
|
plot_dependency_graph: bool = False,
|
|
301
280
|
plot_dependency_graph_filename: Optional[str] = None,
|
|
302
|
-
):
|
|
303
|
-
# Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
|
|
304
|
-
pass
|
|
305
|
-
|
|
306
|
-
def __new__(
|
|
307
|
-
cls,
|
|
308
|
-
max_workers: Optional[int] = None,
|
|
309
|
-
cache_directory: Optional[str] = None,
|
|
310
|
-
max_cores: Optional[int] = None,
|
|
311
|
-
resource_dict: Optional[dict] = None,
|
|
312
|
-
pysqa_config_directory: Optional[str] = None,
|
|
313
|
-
hostname_localhost: Optional[bool] = None,
|
|
314
|
-
block_allocation: bool = False,
|
|
315
|
-
init_function: Optional[Callable] = None,
|
|
316
|
-
disable_dependencies: bool = False,
|
|
317
|
-
refresh_rate: float = 0.01,
|
|
318
|
-
plot_dependency_graph: bool = False,
|
|
319
|
-
plot_dependency_graph_filename: Optional[str] = None,
|
|
320
281
|
):
|
|
321
282
|
"""
|
|
322
283
|
Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
|
|
@@ -375,43 +336,49 @@ class FluxClusterExecutor:
|
|
|
375
336
|
{k: v for k, v in default_resource_dict.items() if k not in resource_dict}
|
|
376
337
|
)
|
|
377
338
|
if not plot_dependency_graph:
|
|
378
|
-
from executorlib.
|
|
379
|
-
|
|
380
|
-
return create_file_executor(
|
|
381
|
-
max_workers=max_workers,
|
|
382
|
-
backend="flux_submission",
|
|
383
|
-
max_cores=max_cores,
|
|
384
|
-
cache_directory=cache_directory,
|
|
385
|
-
resource_dict=resource_dict,
|
|
386
|
-
flux_executor=None,
|
|
387
|
-
flux_executor_pmi_mode=None,
|
|
388
|
-
flux_executor_nesting=False,
|
|
389
|
-
flux_log_files=False,
|
|
390
|
-
pysqa_config_directory=pysqa_config_directory,
|
|
391
|
-
hostname_localhost=hostname_localhost,
|
|
392
|
-
block_allocation=block_allocation,
|
|
393
|
-
init_function=init_function,
|
|
394
|
-
disable_dependencies=disable_dependencies,
|
|
339
|
+
from executorlib.task_scheduler.file.task_scheduler import (
|
|
340
|
+
create_file_executor,
|
|
395
341
|
)
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
executor=
|
|
342
|
+
|
|
343
|
+
super().__init__(
|
|
344
|
+
executor=create_file_executor(
|
|
399
345
|
max_workers=max_workers,
|
|
400
|
-
|
|
346
|
+
backend="flux_submission",
|
|
401
347
|
max_cores=max_cores,
|
|
348
|
+
cache_directory=cache_directory,
|
|
402
349
|
resource_dict=resource_dict,
|
|
403
350
|
flux_executor=None,
|
|
404
351
|
flux_executor_pmi_mode=None,
|
|
405
352
|
flux_executor_nesting=False,
|
|
406
353
|
flux_log_files=False,
|
|
354
|
+
pysqa_config_directory=pysqa_config_directory,
|
|
407
355
|
hostname_localhost=hostname_localhost,
|
|
408
356
|
block_allocation=block_allocation,
|
|
409
357
|
init_function=init_function,
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
358
|
+
disable_dependencies=disable_dependencies,
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
else:
|
|
362
|
+
super().__init__(
|
|
363
|
+
executor=DependencyTaskScheduler(
|
|
364
|
+
executor=create_flux_executor(
|
|
365
|
+
max_workers=max_workers,
|
|
366
|
+
cache_directory=cache_directory,
|
|
367
|
+
max_cores=max_cores,
|
|
368
|
+
resource_dict=resource_dict,
|
|
369
|
+
flux_executor=None,
|
|
370
|
+
flux_executor_pmi_mode=None,
|
|
371
|
+
flux_executor_nesting=False,
|
|
372
|
+
flux_log_files=False,
|
|
373
|
+
hostname_localhost=hostname_localhost,
|
|
374
|
+
block_allocation=block_allocation,
|
|
375
|
+
init_function=init_function,
|
|
376
|
+
),
|
|
377
|
+
max_cores=max_cores,
|
|
378
|
+
refresh_rate=refresh_rate,
|
|
379
|
+
plot_dependency_graph=plot_dependency_graph,
|
|
380
|
+
plot_dependency_graph_filename=plot_dependency_graph_filename,
|
|
381
|
+
)
|
|
415
382
|
)
|
|
416
383
|
|
|
417
384
|
|
|
@@ -427,7 +394,7 @@ def create_flux_executor(
|
|
|
427
394
|
hostname_localhost: Optional[bool] = None,
|
|
428
395
|
block_allocation: bool = False,
|
|
429
396
|
init_function: Optional[Callable] = None,
|
|
430
|
-
) -> Union[
|
|
397
|
+
) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
|
|
431
398
|
"""
|
|
432
399
|
Create a flux executor
|
|
433
400
|
|
|
@@ -466,6 +433,11 @@ def create_flux_executor(
|
|
|
466
433
|
Returns:
|
|
467
434
|
InteractiveStepExecutor/ InteractiveExecutor
|
|
468
435
|
"""
|
|
436
|
+
from executorlib.task_scheduler.interactive.fluxspawner import (
|
|
437
|
+
FluxPythonSpawner,
|
|
438
|
+
validate_max_workers,
|
|
439
|
+
)
|
|
440
|
+
|
|
469
441
|
if resource_dict is None:
|
|
470
442
|
resource_dict = {}
|
|
471
443
|
cores_per_worker = resource_dict.get("cores", 1)
|
|
@@ -498,13 +470,13 @@ def create_flux_executor(
|
|
|
498
470
|
cores=cores_per_worker,
|
|
499
471
|
threads_per_core=resource_dict.get("threads_per_core", 1),
|
|
500
472
|
)
|
|
501
|
-
return
|
|
473
|
+
return BlockAllocationTaskScheduler(
|
|
502
474
|
max_workers=max_workers,
|
|
503
475
|
executor_kwargs=resource_dict,
|
|
504
476
|
spawner=FluxPythonSpawner,
|
|
505
477
|
)
|
|
506
478
|
else:
|
|
507
|
-
return
|
|
479
|
+
return OneProcessTaskScheduler(
|
|
508
480
|
max_cores=max_cores,
|
|
509
481
|
max_workers=max_workers,
|
|
510
482
|
executor_kwargs=resource_dict,
|
{executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/single.py
RENAMED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
from typing import Callable, Optional, Union
|
|
2
2
|
|
|
3
|
-
from executorlib.
|
|
4
|
-
from executorlib.interactive.dependency import DependencyExecutor
|
|
5
|
-
from executorlib.interactive.onetoone import OneTaskPerProcessExecutor
|
|
3
|
+
from executorlib.executor.base import ExecutorBase
|
|
6
4
|
from executorlib.standalone.inputcheck import (
|
|
7
5
|
check_command_line_argument_lst,
|
|
8
6
|
check_gpus_per_worker,
|
|
@@ -12,9 +10,14 @@ from executorlib.standalone.inputcheck import (
|
|
|
12
10
|
validate_number_of_cores,
|
|
13
11
|
)
|
|
14
12
|
from executorlib.standalone.interactive.spawner import MpiExecSpawner
|
|
13
|
+
from executorlib.task_scheduler.interactive.blockallocation import (
|
|
14
|
+
BlockAllocationTaskScheduler,
|
|
15
|
+
)
|
|
16
|
+
from executorlib.task_scheduler.interactive.dependency import DependencyTaskScheduler
|
|
17
|
+
from executorlib.task_scheduler.interactive.onetoone import OneProcessTaskScheduler
|
|
15
18
|
|
|
16
19
|
|
|
17
|
-
class SingleNodeExecutor:
|
|
20
|
+
class SingleNodeExecutor(ExecutorBase):
|
|
18
21
|
"""
|
|
19
22
|
The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
|
|
20
23
|
preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
|
|
@@ -57,7 +60,7 @@ class SingleNodeExecutor:
|
|
|
57
60
|
Examples:
|
|
58
61
|
```
|
|
59
62
|
>>> import numpy as np
|
|
60
|
-
>>> from executorlib.
|
|
63
|
+
>>> from executorlib.executor.single import SingleNodeExecutor
|
|
61
64
|
>>>
|
|
62
65
|
>>> def calc(i, j, k):
|
|
63
66
|
>>> from mpi4py import MPI
|
|
@@ -88,23 +91,6 @@ class SingleNodeExecutor:
|
|
|
88
91
|
refresh_rate: float = 0.01,
|
|
89
92
|
plot_dependency_graph: bool = False,
|
|
90
93
|
plot_dependency_graph_filename: Optional[str] = None,
|
|
91
|
-
):
|
|
92
|
-
# Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
|
|
93
|
-
pass
|
|
94
|
-
|
|
95
|
-
def __new__(
|
|
96
|
-
cls,
|
|
97
|
-
max_workers: Optional[int] = None,
|
|
98
|
-
cache_directory: Optional[str] = None,
|
|
99
|
-
max_cores: Optional[int] = None,
|
|
100
|
-
resource_dict: Optional[dict] = None,
|
|
101
|
-
hostname_localhost: Optional[bool] = None,
|
|
102
|
-
block_allocation: bool = False,
|
|
103
|
-
init_function: Optional[Callable] = None,
|
|
104
|
-
disable_dependencies: bool = False,
|
|
105
|
-
refresh_rate: float = 0.01,
|
|
106
|
-
plot_dependency_graph: bool = False,
|
|
107
|
-
plot_dependency_graph_filename: Optional[str] = None,
|
|
108
94
|
):
|
|
109
95
|
"""
|
|
110
96
|
Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
|
|
@@ -162,7 +148,27 @@ class SingleNodeExecutor:
|
|
|
162
148
|
{k: v for k, v in default_resource_dict.items() if k not in resource_dict}
|
|
163
149
|
)
|
|
164
150
|
if not disable_dependencies:
|
|
165
|
-
|
|
151
|
+
super().__init__(
|
|
152
|
+
executor=DependencyTaskScheduler(
|
|
153
|
+
executor=create_single_node_executor(
|
|
154
|
+
max_workers=max_workers,
|
|
155
|
+
cache_directory=cache_directory,
|
|
156
|
+
max_cores=max_cores,
|
|
157
|
+
resource_dict=resource_dict,
|
|
158
|
+
hostname_localhost=hostname_localhost,
|
|
159
|
+
block_allocation=block_allocation,
|
|
160
|
+
init_function=init_function,
|
|
161
|
+
),
|
|
162
|
+
max_cores=max_cores,
|
|
163
|
+
refresh_rate=refresh_rate,
|
|
164
|
+
plot_dependency_graph=plot_dependency_graph,
|
|
165
|
+
plot_dependency_graph_filename=plot_dependency_graph_filename,
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
|
|
170
|
+
check_refresh_rate(refresh_rate=refresh_rate)
|
|
171
|
+
super().__init__(
|
|
166
172
|
executor=create_single_node_executor(
|
|
167
173
|
max_workers=max_workers,
|
|
168
174
|
cache_directory=cache_directory,
|
|
@@ -171,23 +177,7 @@ class SingleNodeExecutor:
|
|
|
171
177
|
hostname_localhost=hostname_localhost,
|
|
172
178
|
block_allocation=block_allocation,
|
|
173
179
|
init_function=init_function,
|
|
174
|
-
)
|
|
175
|
-
max_cores=max_cores,
|
|
176
|
-
refresh_rate=refresh_rate,
|
|
177
|
-
plot_dependency_graph=plot_dependency_graph,
|
|
178
|
-
plot_dependency_graph_filename=plot_dependency_graph_filename,
|
|
179
|
-
)
|
|
180
|
-
else:
|
|
181
|
-
check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
|
|
182
|
-
check_refresh_rate(refresh_rate=refresh_rate)
|
|
183
|
-
return create_single_node_executor(
|
|
184
|
-
max_workers=max_workers,
|
|
185
|
-
cache_directory=cache_directory,
|
|
186
|
-
max_cores=max_cores,
|
|
187
|
-
resource_dict=resource_dict,
|
|
188
|
-
hostname_localhost=hostname_localhost,
|
|
189
|
-
block_allocation=block_allocation,
|
|
190
|
-
init_function=init_function,
|
|
180
|
+
)
|
|
191
181
|
)
|
|
192
182
|
|
|
193
183
|
|
|
@@ -199,7 +189,7 @@ def create_single_node_executor(
|
|
|
199
189
|
hostname_localhost: Optional[bool] = None,
|
|
200
190
|
block_allocation: bool = False,
|
|
201
191
|
init_function: Optional[Callable] = None,
|
|
202
|
-
) -> Union[
|
|
192
|
+
) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
|
|
203
193
|
"""
|
|
204
194
|
Create a single node executor
|
|
205
195
|
|
|
@@ -253,7 +243,7 @@ def create_single_node_executor(
|
|
|
253
243
|
del resource_dict["slurm_cmd_args"]
|
|
254
244
|
if block_allocation:
|
|
255
245
|
resource_dict["init_function"] = init_function
|
|
256
|
-
return
|
|
246
|
+
return BlockAllocationTaskScheduler(
|
|
257
247
|
max_workers=validate_number_of_cores(
|
|
258
248
|
max_cores=max_cores,
|
|
259
249
|
max_workers=max_workers,
|
|
@@ -264,7 +254,7 @@ def create_single_node_executor(
|
|
|
264
254
|
spawner=MpiExecSpawner,
|
|
265
255
|
)
|
|
266
256
|
else:
|
|
267
|
-
return
|
|
257
|
+
return OneProcessTaskScheduler(
|
|
268
258
|
max_cores=max_cores,
|
|
269
259
|
max_workers=max_workers,
|
|
270
260
|
executor_kwargs=resource_dict,
|