executorlib 1.9.2__py3-none-any.whl → 1.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- executorlib/_version.py +8 -18
- executorlib/executor/base.py +12 -0
- executorlib/executor/flux.py +9 -9
- executorlib/executor/single.py +8 -8
- executorlib/executor/slurm.py +8 -8
- executorlib/standalone/batched.py +22 -10
- executorlib/standalone/command.py +5 -5
- executorlib/standalone/hdf.py +11 -5
- executorlib/standalone/inputcheck.py +54 -1
- executorlib/standalone/interactive/arguments.py +15 -4
- executorlib/standalone/interactive/communication.py +6 -0
- executorlib/standalone/select.py +30 -0
- executorlib/standalone/serialize.py +12 -0
- executorlib/standalone/validate.py +54 -1
- executorlib/task_scheduler/base.py +25 -1
- executorlib/task_scheduler/file/backend.py +14 -7
- executorlib/task_scheduler/file/shared.py +97 -3
- executorlib/task_scheduler/file/spawner_pysqa.py +10 -3
- executorlib/task_scheduler/interactive/dependency.py +44 -22
- executorlib/task_scheduler/interactive/spawner_flux.py +5 -5
- executorlib/task_scheduler/interactive/spawner_pysqa.py +4 -4
- executorlib/task_scheduler/interactive/spawner_slurm.py +4 -4
- {executorlib-1.9.2.dist-info → executorlib-1.9.4.dist-info}/METADATA +52 -23
- executorlib-1.9.4.dist-info/RECORD +50 -0
- {executorlib-1.9.2.dist-info → executorlib-1.9.4.dist-info}/WHEEL +1 -1
- executorlib-1.9.2.dist-info/RECORD +0 -50
- {executorlib-1.9.2.dist-info → executorlib-1.9.4.dist-info}/licenses/LICENSE +0 -0
executorlib/_version.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
3
4
|
|
|
4
5
|
__all__ = [
|
|
5
6
|
"__version__",
|
|
@@ -10,25 +11,14 @@ __all__ = [
|
|
|
10
11
|
"commit_id",
|
|
11
12
|
]
|
|
12
13
|
|
|
13
|
-
TYPE_CHECKING = False
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from typing import Tuple
|
|
16
|
-
from typing import Union
|
|
17
|
-
|
|
18
|
-
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
-
COMMIT_ID = Union[str, None]
|
|
20
|
-
else:
|
|
21
|
-
VERSION_TUPLE = object
|
|
22
|
-
COMMIT_ID = object
|
|
23
|
-
|
|
24
14
|
version: str
|
|
25
15
|
__version__: str
|
|
26
|
-
__version_tuple__:
|
|
27
|
-
version_tuple:
|
|
28
|
-
commit_id:
|
|
29
|
-
__commit_id__:
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
30
20
|
|
|
31
|
-
__version__ = version = '1.9.
|
|
32
|
-
__version_tuple__ = version_tuple = (1, 9,
|
|
21
|
+
__version__ = version = '1.9.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 9, 4)
|
|
33
23
|
|
|
34
24
|
__commit_id__ = commit_id = None
|
executorlib/executor/base.py
CHANGED
|
@@ -25,10 +25,22 @@ class BaseExecutor(FutureExecutor, ABC):
|
|
|
25
25
|
|
|
26
26
|
@property
|
|
27
27
|
def max_workers(self) -> Optional[int]:
|
|
28
|
+
"""
|
|
29
|
+
Return the number of parallel workers configured for this executor.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Optional[int]: The maximum number of parallel workers, or None if unconstrained.
|
|
33
|
+
"""
|
|
28
34
|
return self._task_scheduler.max_workers
|
|
29
35
|
|
|
30
36
|
@max_workers.setter
|
|
31
37
|
def max_workers(self, max_workers: int):
|
|
38
|
+
"""
|
|
39
|
+
Set the number of parallel workers on the underlying task scheduler.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
max_workers (int): New maximum number of parallel workers.
|
|
43
|
+
"""
|
|
32
44
|
self._task_scheduler.max_workers = max_workers
|
|
33
45
|
|
|
34
46
|
@property
|
executorlib/executor/flux.py
CHANGED
|
@@ -51,8 +51,8 @@ class FluxJobExecutor(BaseExecutor):
|
|
|
51
51
|
do not allow other tasks to use the same compute node.
|
|
52
52
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
53
53
|
multiple tasks in one file.
|
|
54
|
-
*
|
|
55
|
-
|
|
54
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
55
|
+
allowed to take in seconds.
|
|
56
56
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
57
57
|
influence the scheduling.
|
|
58
58
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -154,7 +154,7 @@ class FluxJobExecutor(BaseExecutor):
|
|
|
154
154
|
- do not allow other tasks to use the same compute node.
|
|
155
155
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
156
156
|
multiple tasks in one file.
|
|
157
|
-
*
|
|
157
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
158
158
|
function is allowed to take in seconds.
|
|
159
159
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
160
160
|
influence the scheduling.
|
|
@@ -283,8 +283,8 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
283
283
|
do not allow other tasks to use the same compute node.
|
|
284
284
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
285
285
|
multiple tasks in one file.
|
|
286
|
-
*
|
|
287
|
-
|
|
286
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
287
|
+
allowed to take in seconds.
|
|
288
288
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
289
289
|
influence the scheduling.
|
|
290
290
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -380,7 +380,7 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
380
380
|
- do not allow other tasks to use the same compute node.
|
|
381
381
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
382
382
|
multiple tasks in one file.
|
|
383
|
-
*
|
|
383
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
384
384
|
function is allowed to take in seconds.
|
|
385
385
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
386
386
|
influence the scheduling.
|
|
@@ -417,7 +417,7 @@ class FluxClusterExecutor(BaseExecutor):
|
|
|
417
417
|
"cwd": None,
|
|
418
418
|
"openmpi_oversubscribe": openmpi_oversubscribe,
|
|
419
419
|
"slurm_cmd_args": [],
|
|
420
|
-
"
|
|
420
|
+
"run_time_max": None,
|
|
421
421
|
}
|
|
422
422
|
if resource_dict is None:
|
|
423
423
|
resource_dict = {}
|
|
@@ -540,8 +540,8 @@ def create_flux_executor(
|
|
|
540
540
|
do not allow other tasks to use the same compute node.
|
|
541
541
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
542
542
|
multiple tasks in one file.
|
|
543
|
-
*
|
|
544
|
-
|
|
543
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
544
|
+
allowed to take in seconds.
|
|
545
545
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
546
546
|
influence the scheduling.
|
|
547
547
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
executorlib/executor/single.py
CHANGED
|
@@ -50,8 +50,8 @@ class SingleNodeExecutor(BaseExecutor):
|
|
|
50
50
|
do not allow other tasks to use the same compute node.
|
|
51
51
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
52
52
|
multiple tasks in one file.
|
|
53
|
-
*
|
|
54
|
-
|
|
53
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
54
|
+
allowed to take in seconds.
|
|
55
55
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
56
56
|
influence the scheduling.
|
|
57
57
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -145,7 +145,7 @@ class SingleNodeExecutor(BaseExecutor):
|
|
|
145
145
|
- do not allow other tasks to use the same compute node.
|
|
146
146
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
147
147
|
multiple tasks in one file.
|
|
148
|
-
*
|
|
148
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
149
149
|
function is allowed to take in seconds.
|
|
150
150
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
151
151
|
influence the scheduling.
|
|
@@ -259,8 +259,8 @@ class TestClusterExecutor(BaseExecutor):
|
|
|
259
259
|
do not allow other tasks to use the same compute node.
|
|
260
260
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
261
261
|
multiple tasks in one file.
|
|
262
|
-
*
|
|
263
|
-
|
|
262
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
263
|
+
allowed to take in seconds.
|
|
264
264
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
265
265
|
influence the scheduling.
|
|
266
266
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -350,7 +350,7 @@ class TestClusterExecutor(BaseExecutor):
|
|
|
350
350
|
- do not allow other tasks to use the same compute node.
|
|
351
351
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
352
352
|
multiple tasks in one file.
|
|
353
|
-
*
|
|
353
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
354
354
|
function is allowed to take in seconds.
|
|
355
355
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
356
356
|
influence the scheduling.
|
|
@@ -479,8 +479,8 @@ def create_single_node_executor(
|
|
|
479
479
|
do not allow other tasks to use the same compute node.
|
|
480
480
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
481
481
|
multiple tasks in one file.
|
|
482
|
-
*
|
|
483
|
-
|
|
482
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
483
|
+
allowed to take in seconds.
|
|
484
484
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
485
485
|
influence the scheduling.
|
|
486
486
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
executorlib/executor/slurm.py
CHANGED
|
@@ -52,8 +52,8 @@ class SlurmClusterExecutor(BaseExecutor):
|
|
|
52
52
|
do not allow other tasks to use the same compute node.
|
|
53
53
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
54
54
|
multiple tasks in one file.
|
|
55
|
-
*
|
|
56
|
-
|
|
55
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
56
|
+
allowed to take in seconds.
|
|
57
57
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
58
58
|
influence the scheduling.
|
|
59
59
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -149,7 +149,7 @@ class SlurmClusterExecutor(BaseExecutor):
|
|
|
149
149
|
- do not allow other tasks to use the same compute node.
|
|
150
150
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
151
151
|
multiple tasks in one file.
|
|
152
|
-
*
|
|
152
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
153
153
|
function is allowed to take in seconds.
|
|
154
154
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
155
155
|
influence the scheduling.
|
|
@@ -293,8 +293,8 @@ class SlurmJobExecutor(BaseExecutor):
|
|
|
293
293
|
do not allow other tasks to use the same compute node.
|
|
294
294
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
295
295
|
multiple tasks in one file.
|
|
296
|
-
*
|
|
297
|
-
|
|
296
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
297
|
+
allowed to take in seconds.
|
|
298
298
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
299
299
|
influence the scheduling.
|
|
300
300
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -390,7 +390,7 @@ class SlurmJobExecutor(BaseExecutor):
|
|
|
390
390
|
- do not allow other tasks to use the same compute node.
|
|
391
391
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
392
392
|
multiple tasks in one file.
|
|
393
|
-
*
|
|
393
|
+
* run_time_max (int): the maximum time the execution of the submitted Python
|
|
394
394
|
function is allowed to take in seconds.
|
|
395
395
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
396
396
|
influence the scheduling.
|
|
@@ -518,8 +518,8 @@ def create_slurm_executor(
|
|
|
518
518
|
do not allow other tasks to use the same compute node.
|
|
519
519
|
* error_log_file (str): path to the error log file, primarily used to merge the log of
|
|
520
520
|
multiple tasks in one file.
|
|
521
|
-
*
|
|
522
|
-
|
|
521
|
+
* run_time_max (int): the maximum time the execution of the submitted Python function is
|
|
522
|
+
allowed to take in seconds.
|
|
523
523
|
* priority (int): the queuing system priority assigned to a given Python function to
|
|
524
524
|
influence the scheduling.
|
|
525
525
|
* slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
|
|
@@ -1,27 +1,39 @@
|
|
|
1
1
|
from concurrent.futures import Future
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
def batched_futures(
|
|
4
|
+
def batched_futures(
|
|
5
|
+
lst: list[Future], nested_skip_lst: list[Future[list]], n: int
|
|
6
|
+
) -> tuple[bool, list[Future]]:
|
|
5
7
|
"""
|
|
6
8
|
Batch n completed future objects. If the number of completed futures is smaller than n and the end of the batch is
|
|
7
|
-
not reached yet, then an empty list is returned. If n future objects are done, which are not included in the
|
|
9
|
+
not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_set
|
|
8
10
|
then they are returned as batch.
|
|
9
11
|
|
|
10
12
|
Args:
|
|
11
13
|
lst (list): list of all future objects
|
|
12
|
-
|
|
14
|
+
nested_skip_lst (list): list of future objects, which contain the list of future objects ids which should be skipped for the batch
|
|
13
15
|
n (int): batch size
|
|
14
16
|
|
|
15
17
|
Returns:
|
|
16
18
|
list: results of the batched futures
|
|
17
19
|
"""
|
|
18
|
-
|
|
20
|
+
skip_set = {fid for f in nested_skip_lst for fid in f.result()}
|
|
19
21
|
|
|
20
22
|
done_lst = []
|
|
21
|
-
|
|
23
|
+
failed_lst = []
|
|
24
|
+
n_expected = min(n, len(lst) - len(skip_set))
|
|
22
25
|
for v in lst:
|
|
23
|
-
if v
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
if id(v) not in skip_set and v.done():
|
|
27
|
+
if v.exception() is not None:
|
|
28
|
+
failed_lst.append(v)
|
|
29
|
+
elif id(v) not in skip_set and v.done():
|
|
30
|
+
done_lst.append(v)
|
|
31
|
+
if len(done_lst) == n_expected:
|
|
32
|
+
return True, done_lst
|
|
33
|
+
if (len(lst) - len(skip_set)) == len(failed_lst):
|
|
34
|
+
return (
|
|
35
|
+
False,
|
|
36
|
+
failed_lst[:n_expected],
|
|
37
|
+
) # raise the exception only after all futures have failed
|
|
38
|
+
else:
|
|
39
|
+
return True, []
|
|
@@ -126,7 +126,7 @@ def generate_slurm_command(
|
|
|
126
126
|
openmpi_oversubscribe: bool = False,
|
|
127
127
|
slurm_cmd_args: Optional[list[str]] = None,
|
|
128
128
|
pmi_mode: Optional[str] = None,
|
|
129
|
-
|
|
129
|
+
run_time_max: Optional[int] = None,
|
|
130
130
|
) -> list[str]:
|
|
131
131
|
"""
|
|
132
132
|
Generate the command list for the SLURM interface.
|
|
@@ -141,7 +141,7 @@ def generate_slurm_command(
|
|
|
141
141
|
openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
|
|
142
142
|
slurm_cmd_args (list[str], optional): Additional command line arguments. Defaults to [].
|
|
143
143
|
pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
|
|
144
|
-
|
|
144
|
+
run_time_max (int): The maximum runtime in seconds for each task. Default: None
|
|
145
145
|
|
|
146
146
|
Returns:
|
|
147
147
|
list[str]: The generated command list.
|
|
@@ -153,7 +153,7 @@ def generate_slurm_command(
|
|
|
153
153
|
command_prepend_lst += ["--mpi=" + pmi_mode]
|
|
154
154
|
if num_nodes is not None:
|
|
155
155
|
command_prepend_lst += ["-N", str(num_nodes)]
|
|
156
|
-
if threads_per_core
|
|
156
|
+
if threads_per_core >= 1:
|
|
157
157
|
command_prepend_lst += ["--cpus-per-task=" + str(threads_per_core)]
|
|
158
158
|
if gpus_per_core > 0:
|
|
159
159
|
command_prepend_lst += ["--gpus-per-task=" + str(gpus_per_core)]
|
|
@@ -161,8 +161,8 @@ def generate_slurm_command(
|
|
|
161
161
|
command_prepend_lst += ["--exact"]
|
|
162
162
|
if openmpi_oversubscribe:
|
|
163
163
|
command_prepend_lst += ["--oversubscribe"]
|
|
164
|
-
if
|
|
165
|
-
command_prepend_lst += ["--time=" + str(
|
|
164
|
+
if run_time_max is not None:
|
|
165
|
+
command_prepend_lst += ["--time=" + str(run_time_max // 60 + 1)]
|
|
166
166
|
if slurm_cmd_args is not None and len(slurm_cmd_args) > 0:
|
|
167
167
|
command_prepend_lst += slurm_cmd_args
|
|
168
168
|
return command_prepend_lst
|
executorlib/standalone/hdf.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import contextlib
|
|
1
2
|
import os
|
|
2
3
|
from concurrent.futures import Future
|
|
3
4
|
from time import sleep
|
|
@@ -34,10 +35,11 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
|
|
|
34
35
|
with h5py.File(file_name_abs, "a") as fname:
|
|
35
36
|
for data_key, data_value in data_dict.items():
|
|
36
37
|
if data_key in group_dict:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
with contextlib.suppress(ValueError):
|
|
39
|
+
fname.create_dataset(
|
|
40
|
+
name="/" + group_dict[data_key],
|
|
41
|
+
data=np.void(cloudpickle.dumps(data_value)),
|
|
42
|
+
)
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
def load(file_name: str) -> dict:
|
|
@@ -171,7 +173,11 @@ def get_cache_files(cache_directory: str) -> list[str]:
|
|
|
171
173
|
file_lst = []
|
|
172
174
|
cache_directory_abs = os.path.abspath(cache_directory)
|
|
173
175
|
for dirpath, _, filenames in os.walk(cache_directory_abs):
|
|
174
|
-
file_lst += [
|
|
176
|
+
file_lst += [
|
|
177
|
+
os.path.join(dirpath, f)
|
|
178
|
+
for f in filenames
|
|
179
|
+
if f.endswith("_o.h5") or f.endswith("_i.h5")
|
|
180
|
+
]
|
|
175
181
|
return file_lst
|
|
176
182
|
|
|
177
183
|
|
|
@@ -141,6 +141,16 @@ def check_init_function(
|
|
|
141
141
|
def check_max_workers_and_cores(
|
|
142
142
|
max_workers: Optional[int], max_cores: Optional[int]
|
|
143
143
|
) -> None:
|
|
144
|
+
"""
|
|
145
|
+
Check that neither max_workers nor max_cores is set when using a pysqa-based backend.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
max_workers (int, optional): Maximum number of workers.
|
|
149
|
+
max_cores (int, optional): Maximum number of cores.
|
|
150
|
+
|
|
151
|
+
Raises:
|
|
152
|
+
ValueError: If max_workers or max_cores is not None.
|
|
153
|
+
"""
|
|
144
154
|
if max_workers is not None:
|
|
145
155
|
raise ValueError(
|
|
146
156
|
"The number of workers cannot be controlled with the pysqa based backend."
|
|
@@ -152,6 +162,15 @@ def check_max_workers_and_cores(
|
|
|
152
162
|
|
|
153
163
|
|
|
154
164
|
def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
|
|
165
|
+
"""
|
|
166
|
+
Check that hostname_localhost is not set when using a pysqa-based backend.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
hostname_localhost (bool, optional): Flag to use localhost for ZMQ connections.
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
ValueError: If hostname_localhost is not None.
|
|
173
|
+
"""
|
|
155
174
|
if hostname_localhost is not None:
|
|
156
175
|
raise ValueError(
|
|
157
176
|
"The option to connect to hosts based on their hostname is not available with the pysqa based backend."
|
|
@@ -159,6 +178,16 @@ def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
|
|
|
159
178
|
|
|
160
179
|
|
|
161
180
|
def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> None:
|
|
181
|
+
"""
|
|
182
|
+
Check that restart_limit is only used together with block_allocation.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
restart_limit (int): Maximum number of times a worker process may be restarted.
|
|
186
|
+
block_allocation (bool): Whether block allocation is enabled. Defaults to True.
|
|
187
|
+
|
|
188
|
+
Raises:
|
|
189
|
+
ValueError: If restart_limit is non-zero and block_allocation is False.
|
|
190
|
+
"""
|
|
162
191
|
if not block_allocation and restart_limit != 0:
|
|
163
192
|
raise ValueError(
|
|
164
193
|
"The option to specify a restart limit for worker processes is only available with block_allocation=True."
|
|
@@ -166,6 +195,15 @@ def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> No
|
|
|
166
195
|
|
|
167
196
|
|
|
168
197
|
def check_pmi_mode(pmi_mode: Optional[str]) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Check that pmi_mode is not set on a local workstation without SLURM or flux.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
pmi_mode (str, optional): PMI interface name (e.g. "pmix", "pmi1", "pmi2").
|
|
203
|
+
|
|
204
|
+
Raises:
|
|
205
|
+
ValueError: If pmi_mode is not None.
|
|
206
|
+
"""
|
|
169
207
|
if pmi_mode is not None:
|
|
170
208
|
raise ValueError(
|
|
171
209
|
"The option to specify the pmi mode is not available on a local workstation, it requires SLURM or flux."
|
|
@@ -199,7 +237,22 @@ def validate_number_of_cores(
|
|
|
199
237
|
set_local_cores: bool = False,
|
|
200
238
|
) -> int:
|
|
201
239
|
"""
|
|
202
|
-
Validate the number of cores and return the
|
|
240
|
+
Validate the number of cores and return the number of workers to use.
|
|
241
|
+
|
|
242
|
+
Precedence: max_cores / cores_per_worker > max_workers > CPU count (with warning).
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
max_cores (int, optional): Total number of cores available.
|
|
246
|
+
max_workers (int, optional): Explicit number of parallel workers.
|
|
247
|
+
cores_per_worker (int, optional): Number of cores allocated to each worker. Defaults to 1.
|
|
248
|
+
set_local_cores (bool): When True, fall back to the local CPU count if neither
|
|
249
|
+
max_cores nor max_workers is given instead of raising an error. Defaults to False.
|
|
250
|
+
|
|
251
|
+
Returns:
|
|
252
|
+
int: Number of parallel workers to use.
|
|
253
|
+
|
|
254
|
+
Raises:
|
|
255
|
+
ValueError: If neither max_cores nor max_workers is set and set_local_cores is False.
|
|
203
256
|
"""
|
|
204
257
|
if max_cores is not None and max_workers is None and cores_per_worker is not None:
|
|
205
258
|
return int(max_cores / cores_per_worker)
|
|
@@ -27,10 +27,21 @@ def get_future_objects_from_input(args: tuple, kwargs: dict):
|
|
|
27
27
|
|
|
28
28
|
find_future_in_list(lst=args)
|
|
29
29
|
find_future_in_list(lst=kwargs.values())
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
|
|
31
|
+
return future_lst
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def check_list_of_futures_is_done(future_lst: list[Future]) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Check if all future objects in the list of future objects are done
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
future_lst (list): list of future objects
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
bool: True if all future objects in the list of future objects are done, False otherwise
|
|
43
|
+
"""
|
|
44
|
+
return len([future for future in future_lst if future.done()]) == len(future_lst)
|
|
34
45
|
|
|
35
46
|
|
|
36
47
|
def get_exception_lst(future_lst: list[Future]) -> list:
|
|
@@ -240,6 +240,9 @@ def interface_connect(host: str, port: str) -> tuple[zmq.Context, zmq.Socket]:
|
|
|
240
240
|
Args:
|
|
241
241
|
host (str): hostname of the host running the SocketInterface instance to connect to.
|
|
242
242
|
port (str): port on the host the SocketInterface instance is running on.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
Tuple[zmq.Context, zmq.Socket]: The ZMQ context and the connected PAIR socket.
|
|
243
246
|
"""
|
|
244
247
|
context = zmq.Context()
|
|
245
248
|
socket = context.socket(zmq.PAIR)
|
|
@@ -265,6 +268,9 @@ def interface_receive(socket: Optional[zmq.Socket]) -> dict:
|
|
|
265
268
|
|
|
266
269
|
Args:
|
|
267
270
|
socket (zmq.Socket): socket for the connection
|
|
271
|
+
|
|
272
|
+
Returns:
|
|
273
|
+
dict: Deserialized dictionary received from the socket, or an empty dict if socket is None.
|
|
268
274
|
"""
|
|
269
275
|
if socket is not None:
|
|
270
276
|
return cloudpickle.loads(socket.recv())
|
executorlib/standalone/select.py
CHANGED
|
@@ -3,21 +3,51 @@ from typing import Any, Optional
|
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class FutureSelector(Future):
|
|
6
|
+
"""
|
|
7
|
+
A Future wrapper that returns a single element from a result that is a tuple, list, or dict.
|
|
8
|
+
|
|
9
|
+
This is used by split_future() and get_item_from_future() to give callers an individual
|
|
10
|
+
Future-like object for each output of a function that returns a collection.
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
future (Future): The underlying future whose result is a collection.
|
|
14
|
+
selector (int | str): Index (for sequences) or key (for mappings) used to extract
|
|
15
|
+
the desired element from the result.
|
|
16
|
+
"""
|
|
17
|
+
|
|
6
18
|
def __init__(self, future: Future, selector: int | str):
|
|
19
|
+
"""
|
|
20
|
+
Args:
|
|
21
|
+
future (Future): The underlying future whose result is a collection.
|
|
22
|
+
selector (int | str): Index or key used to extract an element from the result.
|
|
23
|
+
"""
|
|
7
24
|
self._future = future
|
|
8
25
|
self._selector = selector
|
|
9
26
|
super().__init__()
|
|
10
27
|
|
|
11
28
|
def __getattr__(self, attr: str) -> Any:
|
|
29
|
+
"""Delegate attribute access to the wrapped future."""
|
|
12
30
|
return getattr(self._future, attr)
|
|
13
31
|
|
|
14
32
|
def __setattr__(self, name: str, value: Any):
|
|
33
|
+
"""Set attributes on the wrapped future, except for the two private instance variables."""
|
|
15
34
|
if name in ["_future", "_selector"]:
|
|
16
35
|
super().__setattr__(name, value)
|
|
17
36
|
else:
|
|
18
37
|
setattr(self._future, name, value)
|
|
19
38
|
|
|
20
39
|
def result(self, timeout: Optional[float] = None) -> Any:
|
|
40
|
+
"""
|
|
41
|
+
Return the selected element from the underlying future's result.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
timeout (float, optional): Maximum seconds to wait for the result. Defaults to None
|
|
45
|
+
(wait indefinitely).
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Any: The element at position/key ``selector`` in the underlying result, or None if
|
|
49
|
+
the underlying result is None.
|
|
50
|
+
"""
|
|
21
51
|
result = self._future.result(timeout=timeout)
|
|
22
52
|
if result is not None:
|
|
23
53
|
return result[self._selector]
|
|
@@ -102,6 +102,18 @@ def _get_hash(binary: bytes) -> str:
|
|
|
102
102
|
|
|
103
103
|
|
|
104
104
|
def _get_function_name(fn: Callable) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Return a human-readable name for a callable.
|
|
107
|
+
|
|
108
|
+
For regular functions and methods the ``__name__`` attribute is used. For callable objects
|
|
109
|
+
(classes implementing ``__call__``) the class name is extracted from ``__class__``.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
fn (Callable): The callable whose name should be determined.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
str: A short name string suitable for use as part of a cache key prefix.
|
|
116
|
+
"""
|
|
105
117
|
if hasattr(fn, "__name__"):
|
|
106
118
|
return fn.__name__
|
|
107
119
|
else:
|
|
@@ -14,6 +14,25 @@ except ImportError:
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class ResourceDictValidation(BaseModel):
|
|
17
|
+
"""
|
|
18
|
+
Pydantic (or dataclass fallback) model for validating resource dictionaries passed to task submissions.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
cores (int, optional): Number of MPI cores to be used for each function call.
|
|
22
|
+
threads_per_core (int, optional): Number of OpenMP threads to be used for each function call.
|
|
23
|
+
gpus_per_core (int, optional): Number of GPUs per worker.
|
|
24
|
+
cwd (str, optional): Current working directory where the parallel python task is executed.
|
|
25
|
+
cache_key (str, optional): External cache key to identify tasks on the file system.
|
|
26
|
+
cache_directory (str, optional): The directory to store cache files.
|
|
27
|
+
num_nodes (int, optional): Number of compute nodes used for the evaluation.
|
|
28
|
+
exclusive (bool, optional): Reserve exclusive access to selected compute nodes.
|
|
29
|
+
error_log_file (str, optional): Path to the error log file.
|
|
30
|
+
run_time_max (int, optional): Maximum allowed execution time in seconds.
|
|
31
|
+
priority (int, optional): Queuing system priority for the task.
|
|
32
|
+
slurm_cmd_args (list[str], optional): Additional command line arguments for the srun call.
|
|
33
|
+
submission_template (str, optional): Template for queuing system job submission scripts.
|
|
34
|
+
"""
|
|
35
|
+
|
|
17
36
|
cores: Optional[int] = None
|
|
18
37
|
threads_per_core: Optional[int] = None
|
|
19
38
|
gpus_per_core: Optional[int] = None
|
|
@@ -23,7 +42,7 @@ class ResourceDictValidation(BaseModel):
|
|
|
23
42
|
num_nodes: Optional[int] = None
|
|
24
43
|
exclusive: Optional[bool] = None
|
|
25
44
|
error_log_file: Optional[str] = None
|
|
26
|
-
|
|
45
|
+
run_time_max: Optional[int] = None
|
|
27
46
|
priority: Optional[int] = None
|
|
28
47
|
slurm_cmd_args: Optional[list[str]] = None
|
|
29
48
|
submission_template: Optional[str] = None
|
|
@@ -39,6 +58,18 @@ if not HAS_PYDANTIC:
|
|
|
39
58
|
|
|
40
59
|
|
|
41
60
|
def _get_accepted_keys(class_type) -> list[str]:
|
|
61
|
+
"""
|
|
62
|
+
Return a list of accepted field names from a Pydantic model or dataclass.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
class_type: A Pydantic BaseModel subclass or a dataclass.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
list[str]: Field names declared on the class.
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
TypeError: If the class type is neither a Pydantic model nor a dataclass.
|
|
72
|
+
"""
|
|
42
73
|
if hasattr(class_type, "model_fields"):
|
|
43
74
|
return list(class_type.model_fields.keys())
|
|
44
75
|
elif hasattr(class_type, "__dataclass_fields__"):
|
|
@@ -47,10 +78,32 @@ def _get_accepted_keys(class_type) -> list[str]:
|
|
|
47
78
|
|
|
48
79
|
|
|
49
80
|
def validate_resource_dict(resource_dict: dict) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Validate a resource dictionary against the declared fields of ResourceDictValidation.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
resource_dict (dict): Dictionary of resource requirements to validate. Unknown keys raise
|
|
86
|
+
a validation error when pydantic is available.
|
|
87
|
+
|
|
88
|
+
Raises:
|
|
89
|
+
ValidationError: If any key/value pair violates the ResourceDictValidation schema.
|
|
90
|
+
"""
|
|
50
91
|
_ = ResourceDictValidation(**resource_dict)
|
|
51
92
|
|
|
52
93
|
|
|
53
94
|
def validate_resource_dict_with_optional_keys(resource_dict: dict) -> None:
|
|
95
|
+
"""
|
|
96
|
+
Validate a resource dictionary, allowing unknown keys with a warning instead of an error.
|
|
97
|
+
|
|
98
|
+
Known keys are validated against ResourceDictValidation. Unknown keys are collected and
|
|
99
|
+
emitted as a UserWarning so callers can detect unsupported options without hard failure.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
resource_dict (dict): Dictionary of resource requirements to validate.
|
|
103
|
+
|
|
104
|
+
Raises:
|
|
105
|
+
ValidationError: If any known key/value pair violates the ResourceDictValidation schema.
|
|
106
|
+
"""
|
|
54
107
|
accepted_keys = _get_accepted_keys(class_type=ResourceDictValidation)
|
|
55
108
|
optional_lst = [key for key in resource_dict if key not in accepted_keys]
|
|
56
109
|
validate_dict = {
|