executorlib 1.9.3__py3-none-any.whl → 1.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- executorlib/_version.py +8 -18
- executorlib/executor/base.py +12 -0
- executorlib/standalone/batched.py +22 -10
- executorlib/standalone/command.py +1 -1
- executorlib/standalone/hdf.py +11 -5
- executorlib/standalone/inputcheck.py +54 -1
- executorlib/standalone/interactive/arguments.py +15 -4
- executorlib/standalone/interactive/communication.py +6 -0
- executorlib/standalone/select.py +30 -0
- executorlib/standalone/serialize.py +12 -0
- executorlib/standalone/validate.py +53 -0
- executorlib/task_scheduler/base.py +25 -1
- executorlib/task_scheduler/file/shared.py +97 -3
- executorlib/task_scheduler/interactive/dependency.py +44 -22
- {executorlib-1.9.3.dist-info → executorlib-1.9.4.dist-info}/METADATA +33 -6
- {executorlib-1.9.3.dist-info → executorlib-1.9.4.dist-info}/RECORD +18 -18
- {executorlib-1.9.3.dist-info → executorlib-1.9.4.dist-info}/WHEEL +1 -1
- {executorlib-1.9.3.dist-info → executorlib-1.9.4.dist-info}/licenses/LICENSE +0 -0
executorlib/_version.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
3
4
|
|
|
4
5
|
__all__ = [
|
|
5
6
|
"__version__",
|
|
@@ -10,25 +11,14 @@ __all__ = [
|
|
|
10
11
|
"commit_id",
|
|
11
12
|
]
|
|
12
13
|
|
|
13
|
-
TYPE_CHECKING = False
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from typing import Tuple
|
|
16
|
-
from typing import Union
|
|
17
|
-
|
|
18
|
-
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
-
COMMIT_ID = Union[str, None]
|
|
20
|
-
else:
|
|
21
|
-
VERSION_TUPLE = object
|
|
22
|
-
COMMIT_ID = object
|
|
23
|
-
|
|
24
14
|
version: str
|
|
25
15
|
__version__: str
|
|
26
|
-
__version_tuple__:
|
|
27
|
-
version_tuple:
|
|
28
|
-
commit_id:
|
|
29
|
-
__commit_id__:
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
30
20
|
|
|
31
|
-
__version__ = version = '1.9.
|
|
32
|
-
__version_tuple__ = version_tuple = (1, 9,
|
|
21
|
+
__version__ = version = '1.9.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 9, 4)
|
|
33
23
|
|
|
34
24
|
__commit_id__ = commit_id = None
|
executorlib/executor/base.py
CHANGED
|
@@ -25,10 +25,22 @@ class BaseExecutor(FutureExecutor, ABC):
|
|
|
25
25
|
|
|
26
26
|
@property
|
|
27
27
|
def max_workers(self) -> Optional[int]:
|
|
28
|
+
"""
|
|
29
|
+
Return the number of parallel workers configured for this executor.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Optional[int]: The maximum number of parallel workers, or None if unconstrained.
|
|
33
|
+
"""
|
|
28
34
|
return self._task_scheduler.max_workers
|
|
29
35
|
|
|
30
36
|
@max_workers.setter
|
|
31
37
|
def max_workers(self, max_workers: int):
|
|
38
|
+
"""
|
|
39
|
+
Set the number of parallel workers on the underlying task scheduler.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
max_workers (int): New maximum number of parallel workers.
|
|
43
|
+
"""
|
|
32
44
|
self._task_scheduler.max_workers = max_workers
|
|
33
45
|
|
|
34
46
|
@property
|
|
@@ -1,27 +1,39 @@
|
|
|
1
1
|
from concurrent.futures import Future
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
def batched_futures(
|
|
4
|
+
def batched_futures(
|
|
5
|
+
lst: list[Future], nested_skip_lst: list[Future[list]], n: int
|
|
6
|
+
) -> tuple[bool, list[Future]]:
|
|
5
7
|
"""
|
|
6
8
|
Batch n completed future objects. If the number of completed futures is smaller than n and the end of the batch is
|
|
7
|
-
not reached yet, then an empty list is returned. If n future objects are done, which are not included in the
|
|
9
|
+
not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_set
|
|
8
10
|
then they are returned as batch.
|
|
9
11
|
|
|
10
12
|
Args:
|
|
11
13
|
lst (list): list of all future objects
|
|
12
|
-
|
|
14
|
+
nested_skip_lst (list): list of future objects, which contain the list of future objects ids which should be skipped for the batch
|
|
13
15
|
n (int): batch size
|
|
14
16
|
|
|
15
17
|
Returns:
|
|
16
18
|
list: results of the batched futures
|
|
17
19
|
"""
|
|
18
|
-
|
|
20
|
+
skip_set = {fid for f in nested_skip_lst for fid in f.result()}
|
|
19
21
|
|
|
20
22
|
done_lst = []
|
|
21
|
-
|
|
23
|
+
failed_lst = []
|
|
24
|
+
n_expected = min(n, len(lst) - len(skip_set))
|
|
22
25
|
for v in lst:
|
|
23
|
-
if v
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
if id(v) not in skip_set and v.done():
|
|
27
|
+
if v.exception() is not None:
|
|
28
|
+
failed_lst.append(v)
|
|
29
|
+
elif id(v) not in skip_set and v.done():
|
|
30
|
+
done_lst.append(v)
|
|
31
|
+
if len(done_lst) == n_expected:
|
|
32
|
+
return True, done_lst
|
|
33
|
+
if (len(lst) - len(skip_set)) == len(failed_lst):
|
|
34
|
+
return (
|
|
35
|
+
False,
|
|
36
|
+
failed_lst[:n_expected],
|
|
37
|
+
) # raise the exception only after all futures have failed
|
|
38
|
+
else:
|
|
39
|
+
return True, []
|
|
@@ -153,7 +153,7 @@ def generate_slurm_command(
|
|
|
153
153
|
command_prepend_lst += ["--mpi=" + pmi_mode]
|
|
154
154
|
if num_nodes is not None:
|
|
155
155
|
command_prepend_lst += ["-N", str(num_nodes)]
|
|
156
|
-
if threads_per_core
|
|
156
|
+
if threads_per_core >= 1:
|
|
157
157
|
command_prepend_lst += ["--cpus-per-task=" + str(threads_per_core)]
|
|
158
158
|
if gpus_per_core > 0:
|
|
159
159
|
command_prepend_lst += ["--gpus-per-task=" + str(gpus_per_core)]
|
executorlib/standalone/hdf.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import contextlib
|
|
1
2
|
import os
|
|
2
3
|
from concurrent.futures import Future
|
|
3
4
|
from time import sleep
|
|
@@ -34,10 +35,11 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
|
|
|
34
35
|
with h5py.File(file_name_abs, "a") as fname:
|
|
35
36
|
for data_key, data_value in data_dict.items():
|
|
36
37
|
if data_key in group_dict:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
with contextlib.suppress(ValueError):
|
|
39
|
+
fname.create_dataset(
|
|
40
|
+
name="/" + group_dict[data_key],
|
|
41
|
+
data=np.void(cloudpickle.dumps(data_value)),
|
|
42
|
+
)
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
def load(file_name: str) -> dict:
|
|
@@ -171,7 +173,11 @@ def get_cache_files(cache_directory: str) -> list[str]:
|
|
|
171
173
|
file_lst = []
|
|
172
174
|
cache_directory_abs = os.path.abspath(cache_directory)
|
|
173
175
|
for dirpath, _, filenames in os.walk(cache_directory_abs):
|
|
174
|
-
file_lst += [
|
|
176
|
+
file_lst += [
|
|
177
|
+
os.path.join(dirpath, f)
|
|
178
|
+
for f in filenames
|
|
179
|
+
if f.endswith("_o.h5") or f.endswith("_i.h5")
|
|
180
|
+
]
|
|
175
181
|
return file_lst
|
|
176
182
|
|
|
177
183
|
|
|
@@ -141,6 +141,16 @@ def check_init_function(
|
|
|
141
141
|
def check_max_workers_and_cores(
|
|
142
142
|
max_workers: Optional[int], max_cores: Optional[int]
|
|
143
143
|
) -> None:
|
|
144
|
+
"""
|
|
145
|
+
Check that neither max_workers nor max_cores is set when using a pysqa-based backend.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
max_workers (int, optional): Maximum number of workers.
|
|
149
|
+
max_cores (int, optional): Maximum number of cores.
|
|
150
|
+
|
|
151
|
+
Raises:
|
|
152
|
+
ValueError: If max_workers or max_cores is not None.
|
|
153
|
+
"""
|
|
144
154
|
if max_workers is not None:
|
|
145
155
|
raise ValueError(
|
|
146
156
|
"The number of workers cannot be controlled with the pysqa based backend."
|
|
@@ -152,6 +162,15 @@ def check_max_workers_and_cores(
|
|
|
152
162
|
|
|
153
163
|
|
|
154
164
|
def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
|
|
165
|
+
"""
|
|
166
|
+
Check that hostname_localhost is not set when using a pysqa-based backend.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
hostname_localhost (bool, optional): Flag to use localhost for ZMQ connections.
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
ValueError: If hostname_localhost is not None.
|
|
173
|
+
"""
|
|
155
174
|
if hostname_localhost is not None:
|
|
156
175
|
raise ValueError(
|
|
157
176
|
"The option to connect to hosts based on their hostname is not available with the pysqa based backend."
|
|
@@ -159,6 +178,16 @@ def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
|
|
|
159
178
|
|
|
160
179
|
|
|
161
180
|
def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> None:
|
|
181
|
+
"""
|
|
182
|
+
Check that restart_limit is only used together with block_allocation.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
restart_limit (int): Maximum number of times a worker process may be restarted.
|
|
186
|
+
block_allocation (bool): Whether block allocation is enabled. Defaults to True.
|
|
187
|
+
|
|
188
|
+
Raises:
|
|
189
|
+
ValueError: If restart_limit is non-zero and block_allocation is False.
|
|
190
|
+
"""
|
|
162
191
|
if not block_allocation and restart_limit != 0:
|
|
163
192
|
raise ValueError(
|
|
164
193
|
"The option to specify a restart limit for worker processes is only available with block_allocation=True."
|
|
@@ -166,6 +195,15 @@ def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> No
|
|
|
166
195
|
|
|
167
196
|
|
|
168
197
|
def check_pmi_mode(pmi_mode: Optional[str]) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Check that pmi_mode is not set on a local workstation without SLURM or flux.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
pmi_mode (str, optional): PMI interface name (e.g. "pmix", "pmi1", "pmi2").
|
|
203
|
+
|
|
204
|
+
Raises:
|
|
205
|
+
ValueError: If pmi_mode is not None.
|
|
206
|
+
"""
|
|
169
207
|
if pmi_mode is not None:
|
|
170
208
|
raise ValueError(
|
|
171
209
|
"The option to specify the pmi mode is not available on a local workstation, it requires SLURM or flux."
|
|
@@ -199,7 +237,22 @@ def validate_number_of_cores(
|
|
|
199
237
|
set_local_cores: bool = False,
|
|
200
238
|
) -> int:
|
|
201
239
|
"""
|
|
202
|
-
Validate the number of cores and return the
|
|
240
|
+
Validate the number of cores and return the number of workers to use.
|
|
241
|
+
|
|
242
|
+
Precedence: max_cores / cores_per_worker > max_workers > CPU count (with warning).
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
max_cores (int, optional): Total number of cores available.
|
|
246
|
+
max_workers (int, optional): Explicit number of parallel workers.
|
|
247
|
+
cores_per_worker (int, optional): Number of cores allocated to each worker. Defaults to 1.
|
|
248
|
+
set_local_cores (bool): When True, fall back to the local CPU count if neither
|
|
249
|
+
max_cores nor max_workers is given instead of raising an error. Defaults to False.
|
|
250
|
+
|
|
251
|
+
Returns:
|
|
252
|
+
int: Number of parallel workers to use.
|
|
253
|
+
|
|
254
|
+
Raises:
|
|
255
|
+
ValueError: If neither max_cores nor max_workers is set and set_local_cores is False.
|
|
203
256
|
"""
|
|
204
257
|
if max_cores is not None and max_workers is None and cores_per_worker is not None:
|
|
205
258
|
return int(max_cores / cores_per_worker)
|
|
@@ -27,10 +27,21 @@ def get_future_objects_from_input(args: tuple, kwargs: dict):
|
|
|
27
27
|
|
|
28
28
|
find_future_in_list(lst=args)
|
|
29
29
|
find_future_in_list(lst=kwargs.values())
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
|
|
31
|
+
return future_lst
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def check_list_of_futures_is_done(future_lst: list[Future]) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Check if all future objects in the list of future objects are done
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
future_lst (list): list of future objects
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
bool: True if all future objects in the list of future objects are done, False otherwise
|
|
43
|
+
"""
|
|
44
|
+
return len([future for future in future_lst if future.done()]) == len(future_lst)
|
|
34
45
|
|
|
35
46
|
|
|
36
47
|
def get_exception_lst(future_lst: list[Future]) -> list:
|
|
@@ -240,6 +240,9 @@ def interface_connect(host: str, port: str) -> tuple[zmq.Context, zmq.Socket]:
|
|
|
240
240
|
Args:
|
|
241
241
|
host (str): hostname of the host running the SocketInterface instance to connect to.
|
|
242
242
|
port (str): port on the host the SocketInterface instance is running on.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
Tuple[zmq.Context, zmq.Socket]: The ZMQ context and the connected PAIR socket.
|
|
243
246
|
"""
|
|
244
247
|
context = zmq.Context()
|
|
245
248
|
socket = context.socket(zmq.PAIR)
|
|
@@ -265,6 +268,9 @@ def interface_receive(socket: Optional[zmq.Socket]) -> dict:
|
|
|
265
268
|
|
|
266
269
|
Args:
|
|
267
270
|
socket (zmq.Socket): socket for the connection
|
|
271
|
+
|
|
272
|
+
Returns:
|
|
273
|
+
dict: Deserialized dictionary received from the socket, or an empty dict if socket is None.
|
|
268
274
|
"""
|
|
269
275
|
if socket is not None:
|
|
270
276
|
return cloudpickle.loads(socket.recv())
|
executorlib/standalone/select.py
CHANGED
|
@@ -3,21 +3,51 @@ from typing import Any, Optional
|
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class FutureSelector(Future):
|
|
6
|
+
"""
|
|
7
|
+
A Future wrapper that returns a single element from a result that is a tuple, list, or dict.
|
|
8
|
+
|
|
9
|
+
This is used by split_future() and get_item_from_future() to give callers an individual
|
|
10
|
+
Future-like object for each output of a function that returns a collection.
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
future (Future): The underlying future whose result is a collection.
|
|
14
|
+
selector (int | str): Index (for sequences) or key (for mappings) used to extract
|
|
15
|
+
the desired element from the result.
|
|
16
|
+
"""
|
|
17
|
+
|
|
6
18
|
def __init__(self, future: Future, selector: int | str):
|
|
19
|
+
"""
|
|
20
|
+
Args:
|
|
21
|
+
future (Future): The underlying future whose result is a collection.
|
|
22
|
+
selector (int | str): Index or key used to extract an element from the result.
|
|
23
|
+
"""
|
|
7
24
|
self._future = future
|
|
8
25
|
self._selector = selector
|
|
9
26
|
super().__init__()
|
|
10
27
|
|
|
11
28
|
def __getattr__(self, attr: str) -> Any:
|
|
29
|
+
"""Delegate attribute access to the wrapped future."""
|
|
12
30
|
return getattr(self._future, attr)
|
|
13
31
|
|
|
14
32
|
def __setattr__(self, name: str, value: Any):
|
|
33
|
+
"""Set attributes on the wrapped future, except for the two private instance variables."""
|
|
15
34
|
if name in ["_future", "_selector"]:
|
|
16
35
|
super().__setattr__(name, value)
|
|
17
36
|
else:
|
|
18
37
|
setattr(self._future, name, value)
|
|
19
38
|
|
|
20
39
|
def result(self, timeout: Optional[float] = None) -> Any:
|
|
40
|
+
"""
|
|
41
|
+
Return the selected element from the underlying future's result.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
timeout (float, optional): Maximum seconds to wait for the result. Defaults to None
|
|
45
|
+
(wait indefinitely).
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Any: The element at position/key ``selector`` in the underlying result, or None if
|
|
49
|
+
the underlying result is None.
|
|
50
|
+
"""
|
|
21
51
|
result = self._future.result(timeout=timeout)
|
|
22
52
|
if result is not None:
|
|
23
53
|
return result[self._selector]
|
|
@@ -102,6 +102,18 @@ def _get_hash(binary: bytes) -> str:
|
|
|
102
102
|
|
|
103
103
|
|
|
104
104
|
def _get_function_name(fn: Callable) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Return a human-readable name for a callable.
|
|
107
|
+
|
|
108
|
+
For regular functions and methods the ``__name__`` attribute is used. For callable objects
|
|
109
|
+
(classes implementing ``__call__``) the class name is extracted from ``__class__``.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
fn (Callable): The callable whose name should be determined.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
str: A short name string suitable for use as part of a cache key prefix.
|
|
116
|
+
"""
|
|
105
117
|
if hasattr(fn, "__name__"):
|
|
106
118
|
return fn.__name__
|
|
107
119
|
else:
|
|
@@ -14,6 +14,25 @@ except ImportError:
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class ResourceDictValidation(BaseModel):
|
|
17
|
+
"""
|
|
18
|
+
Pydantic (or dataclass fallback) model for validating resource dictionaries passed to task submissions.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
cores (int, optional): Number of MPI cores to be used for each function call.
|
|
22
|
+
threads_per_core (int, optional): Number of OpenMP threads to be used for each function call.
|
|
23
|
+
gpus_per_core (int, optional): Number of GPUs per worker.
|
|
24
|
+
cwd (str, optional): Current working directory where the parallel python task is executed.
|
|
25
|
+
cache_key (str, optional): External cache key to identify tasks on the file system.
|
|
26
|
+
cache_directory (str, optional): The directory to store cache files.
|
|
27
|
+
num_nodes (int, optional): Number of compute nodes used for the evaluation.
|
|
28
|
+
exclusive (bool, optional): Reserve exclusive access to selected compute nodes.
|
|
29
|
+
error_log_file (str, optional): Path to the error log file.
|
|
30
|
+
run_time_max (int, optional): Maximum allowed execution time in seconds.
|
|
31
|
+
priority (int, optional): Queuing system priority for the task.
|
|
32
|
+
slurm_cmd_args (list[str], optional): Additional command line arguments for the srun call.
|
|
33
|
+
submission_template (str, optional): Template for queuing system job submission scripts.
|
|
34
|
+
"""
|
|
35
|
+
|
|
17
36
|
cores: Optional[int] = None
|
|
18
37
|
threads_per_core: Optional[int] = None
|
|
19
38
|
gpus_per_core: Optional[int] = None
|
|
@@ -39,6 +58,18 @@ if not HAS_PYDANTIC:
|
|
|
39
58
|
|
|
40
59
|
|
|
41
60
|
def _get_accepted_keys(class_type) -> list[str]:
|
|
61
|
+
"""
|
|
62
|
+
Return a list of accepted field names from a Pydantic model or dataclass.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
class_type: A Pydantic BaseModel subclass or a dataclass.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
list[str]: Field names declared on the class.
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
TypeError: If the class type is neither a Pydantic model nor a dataclass.
|
|
72
|
+
"""
|
|
42
73
|
if hasattr(class_type, "model_fields"):
|
|
43
74
|
return list(class_type.model_fields.keys())
|
|
44
75
|
elif hasattr(class_type, "__dataclass_fields__"):
|
|
@@ -47,10 +78,32 @@ def _get_accepted_keys(class_type) -> list[str]:
|
|
|
47
78
|
|
|
48
79
|
|
|
49
80
|
def validate_resource_dict(resource_dict: dict) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Validate a resource dictionary against the declared fields of ResourceDictValidation.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
resource_dict (dict): Dictionary of resource requirements to validate. Unknown keys raise
|
|
86
|
+
a validation error when pydantic is available.
|
|
87
|
+
|
|
88
|
+
Raises:
|
|
89
|
+
ValidationError: If any key/value pair violates the ResourceDictValidation schema.
|
|
90
|
+
"""
|
|
50
91
|
_ = ResourceDictValidation(**resource_dict)
|
|
51
92
|
|
|
52
93
|
|
|
53
94
|
def validate_resource_dict_with_optional_keys(resource_dict: dict) -> None:
|
|
95
|
+
"""
|
|
96
|
+
Validate a resource dictionary, allowing unknown keys with a warning instead of an error.
|
|
97
|
+
|
|
98
|
+
Known keys are validated against ResourceDictValidation. Unknown keys are collected and
|
|
99
|
+
emitted as a UserWarning so callers can detect unsupported options without hard failure.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
resource_dict (dict): Dictionary of resource requirements to validate.
|
|
103
|
+
|
|
104
|
+
Raises:
|
|
105
|
+
ValidationError: If any known key/value pair violates the ResourceDictValidation schema.
|
|
106
|
+
"""
|
|
54
107
|
accepted_keys = _get_accepted_keys(class_type=ResourceDictValidation)
|
|
55
108
|
optional_lst = [key for key in resource_dict if key not in accepted_keys]
|
|
56
109
|
validate_dict = {
|
|
@@ -15,6 +15,12 @@ from executorlib.standalone.serialize import cloudpickle_register
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def validate_resource_dict(resource_dict: dict):
|
|
18
|
+
"""
|
|
19
|
+
No-op resource dict validator used as the default when no validation is required.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
resource_dict (dict): Dictionary of resource requirements (ignored).
|
|
23
|
+
"""
|
|
18
24
|
pass
|
|
19
25
|
|
|
20
26
|
|
|
@@ -32,7 +38,13 @@ class TaskSchedulerBase(FutureExecutor):
|
|
|
32
38
|
validator: Callable = validate_resource_dict,
|
|
33
39
|
):
|
|
34
40
|
"""
|
|
35
|
-
Initialize the
|
|
41
|
+
Initialize the TaskSchedulerBase.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
max_cores (int, optional): Maximum number of cores available to the scheduler.
|
|
45
|
+
Tasks requesting more cores than this will be rejected. Defaults to None (unlimited).
|
|
46
|
+
validator (Callable): Function used to validate per-task resource dicts before
|
|
47
|
+
submission. Defaults to the no-op validate_resource_dict.
|
|
36
48
|
"""
|
|
37
49
|
cloudpickle_register(ind=3)
|
|
38
50
|
self._process_kwargs: dict = {}
|
|
@@ -43,10 +55,22 @@ class TaskSchedulerBase(FutureExecutor):
|
|
|
43
55
|
|
|
44
56
|
@property
|
|
45
57
|
def max_workers(self) -> Optional[int]:
|
|
58
|
+
"""
|
|
59
|
+
Return the configured number of parallel workers, or None if unconstrained.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
Optional[int]: The max_workers value stored in process kwargs, or None.
|
|
63
|
+
"""
|
|
46
64
|
return self._process_kwargs.get("max_workers")
|
|
47
65
|
|
|
48
66
|
@max_workers.setter
|
|
49
67
|
def max_workers(self, max_workers: int):
|
|
68
|
+
"""
|
|
69
|
+
Setting max_workers after construction is not supported by the base scheduler.
|
|
70
|
+
|
|
71
|
+
Raises:
|
|
72
|
+
NotImplementedError: Always.
|
|
73
|
+
"""
|
|
50
74
|
raise NotImplementedError("The max_workers setter is not implemented.")
|
|
51
75
|
|
|
52
76
|
@property
|
|
@@ -91,6 +91,7 @@ def execute_tasks_h5(
|
|
|
91
91
|
process_dict: dict = {}
|
|
92
92
|
cache_dir_dict: dict = {}
|
|
93
93
|
file_name_dict: dict = {}
|
|
94
|
+
duplicate_dict: dict = {}
|
|
94
95
|
while True:
|
|
95
96
|
task_dict = None
|
|
96
97
|
with contextlib.suppress(queue.Empty):
|
|
@@ -101,6 +102,7 @@ def execute_tasks_h5(
|
|
|
101
102
|
cancel_futures=task_dict.get("cancel_futures", False),
|
|
102
103
|
memory_dict=memory_dict,
|
|
103
104
|
process_dict=process_dict,
|
|
105
|
+
duplicate_dict=duplicate_dict,
|
|
104
106
|
cache_dir_dict=cache_dir_dict,
|
|
105
107
|
terminate_function=terminate_function,
|
|
106
108
|
pysqa_config_directory=pysqa_config_directory,
|
|
@@ -175,12 +177,17 @@ def execute_tasks_h5(
|
|
|
175
177
|
process_dict[task_key] = queue_id
|
|
176
178
|
memory_dict[task_key] = task_dict["future"]
|
|
177
179
|
cache_dir_dict[task_key] = cache_directory
|
|
180
|
+
elif memory_dict[task_key] != task_dict["future"]:
|
|
181
|
+
if task_key not in duplicate_dict:
|
|
182
|
+
duplicate_dict[task_key] = []
|
|
183
|
+
duplicate_dict[task_key].append(task_dict["future"])
|
|
178
184
|
future_queue.task_done()
|
|
179
185
|
else:
|
|
180
186
|
memory_dict = _refresh_memory_dict(
|
|
181
187
|
memory_dict=memory_dict,
|
|
182
188
|
cache_dir_dict=cache_dir_dict,
|
|
183
189
|
process_dict=process_dict,
|
|
190
|
+
duplicate_dict=duplicate_dict,
|
|
184
191
|
terminate_function=terminate_function,
|
|
185
192
|
pysqa_config_directory=pysqa_config_directory,
|
|
186
193
|
backend=backend,
|
|
@@ -189,7 +196,10 @@ def execute_tasks_h5(
|
|
|
189
196
|
|
|
190
197
|
|
|
191
198
|
def _check_task_output(
|
|
192
|
-
task_key: str,
|
|
199
|
+
task_key: str,
|
|
200
|
+
future_obj: Future,
|
|
201
|
+
cache_directory: str,
|
|
202
|
+
duplicate_dict: Optional[dict] = None,
|
|
193
203
|
) -> Future:
|
|
194
204
|
"""
|
|
195
205
|
Check the output of a task and set the result of the future object if available.
|
|
@@ -198,7 +208,7 @@ def _check_task_output(
|
|
|
198
208
|
task_key (str): The key of the task.
|
|
199
209
|
future_obj (Future): The future object associated with the task.
|
|
200
210
|
cache_directory (str): The directory where the HDF5 files are stored.
|
|
201
|
-
|
|
211
|
+
duplicate_dict (dict): The dictionary mapping task keys to their associated duplicate future objects.
|
|
202
212
|
Returns:
|
|
203
213
|
Future: The updated future object.
|
|
204
214
|
|
|
@@ -207,11 +217,40 @@ def _check_task_output(
|
|
|
207
217
|
if not os.path.exists(file_name):
|
|
208
218
|
return future_obj
|
|
209
219
|
exec_flag, no_error_flag, result = get_output(file_name=file_name)
|
|
220
|
+
_update_future(
|
|
221
|
+
future_obj=future_obj,
|
|
222
|
+
exec_flag=exec_flag,
|
|
223
|
+
no_error_flag=no_error_flag,
|
|
224
|
+
result=result,
|
|
225
|
+
)
|
|
226
|
+
if duplicate_dict is not None and task_key in duplicate_dict:
|
|
227
|
+
for duplicate_future in duplicate_dict[task_key]:
|
|
228
|
+
_update_future(
|
|
229
|
+
future_obj=duplicate_future,
|
|
230
|
+
exec_flag=exec_flag,
|
|
231
|
+
no_error_flag=no_error_flag,
|
|
232
|
+
result=result,
|
|
233
|
+
)
|
|
234
|
+
del duplicate_dict[task_key]
|
|
235
|
+
return future_obj
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _update_future(
|
|
239
|
+
future_obj: Future, exec_flag: bool, no_error_flag: bool, result: Any
|
|
240
|
+
) -> None:
|
|
241
|
+
"""
|
|
242
|
+
Update the future object with the result of the task execution.
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
future_obj (Future): The future object to be updated.
|
|
246
|
+
exec_flag (bool): Flag indicating whether the task has been executed.
|
|
247
|
+
no_error_flag (bool): Flag indicating whether the task execution resulted in an error.
|
|
248
|
+
result (Any): The result of the task execution.
|
|
249
|
+
"""
|
|
210
250
|
if exec_flag and no_error_flag:
|
|
211
251
|
future_obj.set_result(result)
|
|
212
252
|
elif exec_flag:
|
|
213
253
|
future_obj.set_exception(result)
|
|
214
|
-
return future_obj
|
|
215
254
|
|
|
216
255
|
|
|
217
256
|
def _convert_args_and_kwargs(
|
|
@@ -281,6 +320,7 @@ def _refresh_memory_dict(
|
|
|
281
320
|
memory_dict: dict,
|
|
282
321
|
cache_dir_dict: dict,
|
|
283
322
|
process_dict: dict,
|
|
323
|
+
duplicate_dict: Optional[dict] = None,
|
|
284
324
|
terminate_function: Optional[Callable] = None,
|
|
285
325
|
pysqa_config_directory: Optional[str] = None,
|
|
286
326
|
backend: Optional[str] = None,
|
|
@@ -293,6 +333,7 @@ def _refresh_memory_dict(
|
|
|
293
333
|
memory_dict (dict): dictionary with task keys and future objects
|
|
294
334
|
cache_dir_dict (dict): dictionary with task keys and cache directories
|
|
295
335
|
process_dict (dict): dictionary with task keys and process reference.
|
|
336
|
+
duplicate_dict (dict): dictionary with task keys and duplicate future objects.
|
|
296
337
|
terminate_function (callable): The function to terminate the tasks.
|
|
297
338
|
pysqa_config_directory (str): path to the pysqa config directory (only for pysqa based backend).
|
|
298
339
|
backend (str): name of the backend used to spawn tasks.
|
|
@@ -315,6 +356,7 @@ def _refresh_memory_dict(
|
|
|
315
356
|
task_key=key,
|
|
316
357
|
future_obj=value,
|
|
317
358
|
cache_directory=cache_dir_dict[key],
|
|
359
|
+
duplicate_dict=duplicate_dict,
|
|
318
360
|
)
|
|
319
361
|
for key, value in memory_dict.items()
|
|
320
362
|
if not value.done()
|
|
@@ -354,6 +396,24 @@ def _cancel_processes(
|
|
|
354
396
|
def _get_task_input(
|
|
355
397
|
task_resource_dict: dict, executor_kwargs: dict
|
|
356
398
|
) -> tuple[dict, Optional[str], str, Optional[str]]:
|
|
399
|
+
"""
|
|
400
|
+
Merge per-task resource requirements with executor defaults and extract scheduling metadata.
|
|
401
|
+
|
|
402
|
+
Executor-level kwargs fill in any keys not already present in the per-task resource dict.
|
|
403
|
+
The special keys ``cache_key``, ``cache_directory``, and ``error_log_file`` are popped from
|
|
404
|
+
the merged dict and returned separately so callers do not forward them to the backend.
|
|
405
|
+
|
|
406
|
+
Args:
|
|
407
|
+
task_resource_dict (dict): Per-task resource dict from the submitted future, modified in place.
|
|
408
|
+
executor_kwargs (dict): Executor-level defaults (e.g. cores, cwd, cache_directory).
|
|
409
|
+
|
|
410
|
+
Returns:
|
|
411
|
+
Tuple[dict, Optional[str], str, Optional[str]]:
|
|
412
|
+
- merged resource dict (without scheduling-only keys)
|
|
413
|
+
- cache_key (str or None)
|
|
414
|
+
- cache_directory (str, absolute path)
|
|
415
|
+
- error_log_file (str or None)
|
|
416
|
+
"""
|
|
357
417
|
task_resource_dict.update(
|
|
358
418
|
{k: v for k, v in executor_kwargs.items() if k not in task_resource_dict}
|
|
359
419
|
)
|
|
@@ -364,6 +424,13 @@ def _get_task_input(
|
|
|
364
424
|
|
|
365
425
|
|
|
366
426
|
def _cancel_futures(future_dict: dict):
|
|
427
|
+
"""
|
|
428
|
+
Cancel all pending futures in the dictionary.
|
|
429
|
+
|
|
430
|
+
Args:
|
|
431
|
+
future_dict (dict): Mapping of task keys to Future objects. Already-done futures are
|
|
432
|
+
skipped; pending ones are cancelled.
|
|
433
|
+
"""
|
|
367
434
|
for value in future_dict.values():
|
|
368
435
|
if not value.done():
|
|
369
436
|
value.cancel()
|
|
@@ -375,17 +442,42 @@ def _shutdown_executor(
|
|
|
375
442
|
memory_dict: dict,
|
|
376
443
|
process_dict: dict,
|
|
377
444
|
cache_dir_dict: dict,
|
|
445
|
+
duplicate_dict: Optional[dict] = None,
|
|
378
446
|
terminate_function: Optional[Callable] = None,
|
|
379
447
|
pysqa_config_directory: Optional[str] = None,
|
|
380
448
|
backend: Optional[str] = None,
|
|
381
449
|
refresh_rate: float = 0.01,
|
|
382
450
|
):
|
|
451
|
+
"""
|
|
452
|
+
Shut down the file-based executor, optionally waiting for or cancelling pending tasks.
|
|
453
|
+
|
|
454
|
+
The four combinations of ``wait`` / ``cancel_futures`` mirror the semantics of
|
|
455
|
+
concurrent.futures.Executor.shutdown():
|
|
456
|
+
|
|
457
|
+
* wait=True, cancel_futures=False – poll until all tasks finish.
|
|
458
|
+
* wait=True, cancel_futures=True – cancel pending futures then wait for running ones.
|
|
459
|
+
* wait=False, cancel_futures=True – cancel everything immediately without blocking.
|
|
460
|
+
* wait=False, cancel_futures=False – detach tasks and mark futures cancelled.
|
|
461
|
+
|
|
462
|
+
Args:
|
|
463
|
+
wait (bool): Whether to block until all outstanding tasks have resolved.
|
|
464
|
+
cancel_futures (bool): Whether to cancel futures that have not yet started.
|
|
465
|
+
memory_dict (dict): Mapping of task keys to their Future objects.
|
|
466
|
+
process_dict (dict): Mapping of task keys to process handles or queue IDs.
|
|
467
|
+
duplicate_dict (dict): Mapping of task keys to lists of duplicate Future objects.
|
|
468
|
+
cache_dir_dict (dict): Mapping of task keys to the cache directory for each task.
|
|
469
|
+
terminate_function (Callable, optional): Function used to terminate running processes.
|
|
470
|
+
pysqa_config_directory (str, optional): Path to the pysqa config directory.
|
|
471
|
+
backend (str, optional): Name of the backend ("slurm", "flux", or None for subprocess).
|
|
472
|
+
refresh_rate (float): Polling interval in seconds when waiting for tasks. Defaults to 0.01.
|
|
473
|
+
"""
|
|
383
474
|
if wait and not cancel_futures:
|
|
384
475
|
while len(memory_dict) > 0:
|
|
385
476
|
memory_dict = _refresh_memory_dict(
|
|
386
477
|
memory_dict=memory_dict,
|
|
387
478
|
cache_dir_dict=cache_dir_dict,
|
|
388
479
|
process_dict=process_dict,
|
|
480
|
+
duplicate_dict=duplicate_dict,
|
|
389
481
|
terminate_function=terminate_function,
|
|
390
482
|
pysqa_config_directory=pysqa_config_directory,
|
|
391
483
|
backend=backend,
|
|
@@ -400,6 +492,7 @@ def _shutdown_executor(
|
|
|
400
492
|
memory_dict=memory_dict,
|
|
401
493
|
cache_dir_dict=cache_dir_dict,
|
|
402
494
|
process_dict=process_dict,
|
|
495
|
+
duplicate_dict=duplicate_dict,
|
|
403
496
|
terminate_function=terminate_function,
|
|
404
497
|
pysqa_config_directory=pysqa_config_directory,
|
|
405
498
|
backend=backend,
|
|
@@ -418,6 +511,7 @@ def _shutdown_executor(
|
|
|
418
511
|
memory_dict=memory_dict,
|
|
419
512
|
cache_dir_dict=cache_dir_dict,
|
|
420
513
|
process_dict=process_dict,
|
|
514
|
+
duplicate_dict=duplicate_dict,
|
|
421
515
|
terminate_function=terminate_function,
|
|
422
516
|
pysqa_config_directory=pysqa_config_directory,
|
|
423
517
|
backend=backend,
|
|
@@ -7,6 +7,7 @@ from typing import Any, Callable, Optional
|
|
|
7
7
|
from executorlib.standalone.batched import batched_futures
|
|
8
8
|
from executorlib.standalone.interactive.arguments import (
|
|
9
9
|
check_exception_was_raised,
|
|
10
|
+
check_list_of_futures_is_done,
|
|
10
11
|
get_exception_lst,
|
|
11
12
|
get_future_objects_from_input,
|
|
12
13
|
update_futures_in_input,
|
|
@@ -177,6 +178,7 @@ class DependencyTaskScheduler(TaskSchedulerBase):
|
|
|
177
178
|
future_lst: list[Future] = []
|
|
178
179
|
for _ in range(len(iterable) // n + (1 if len(iterable) % n > 0 else 0)):
|
|
179
180
|
f: Future = Future()
|
|
181
|
+
f_skip: Future = Future()
|
|
180
182
|
if self._future_queue is not None:
|
|
181
183
|
self._future_queue.put(
|
|
182
184
|
{
|
|
@@ -184,10 +186,12 @@ class DependencyTaskScheduler(TaskSchedulerBase):
|
|
|
184
186
|
"args": (),
|
|
185
187
|
"kwargs": {"lst": iterable, "n": n, "skip_lst": skip_lst},
|
|
186
188
|
"future": f,
|
|
189
|
+
"future_lst": iterable,
|
|
190
|
+
"future_skip": f_skip,
|
|
187
191
|
"resource_dict": {},
|
|
188
192
|
}
|
|
189
193
|
)
|
|
190
|
-
skip_lst = skip_lst.copy() + [
|
|
194
|
+
skip_lst = skip_lst.copy() + [f_skip] # be careful
|
|
191
195
|
future_lst.append(f)
|
|
192
196
|
|
|
193
197
|
return future_lst
|
|
@@ -247,7 +251,7 @@ def _execute_tasks_with_dependencies(
|
|
|
247
251
|
executor (TaskSchedulerBase): Executor to execute the tasks with after the dependencies are resolved.
|
|
248
252
|
refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
|
|
249
253
|
"""
|
|
250
|
-
|
|
254
|
+
future_dependency_lst: list = []
|
|
251
255
|
while True:
|
|
252
256
|
try:
|
|
253
257
|
task_dict = future_queue.get_nowait()
|
|
@@ -256,10 +260,10 @@ def _execute_tasks_with_dependencies(
|
|
|
256
260
|
if ( # shutdown the executor
|
|
257
261
|
task_dict is not None and "shutdown" in task_dict and task_dict["shutdown"]
|
|
258
262
|
):
|
|
259
|
-
while len(
|
|
263
|
+
while len(future_dependency_lst) > 0:
|
|
260
264
|
# Check functions in the wait list and execute them if all future objects are now ready
|
|
261
|
-
|
|
262
|
-
|
|
265
|
+
future_dependency_lst = _handle_future_dependencies(
|
|
266
|
+
future_dependency_lst=future_dependency_lst,
|
|
263
267
|
executor_queue=executor_queue,
|
|
264
268
|
refresh_rate=refresh_rate,
|
|
265
269
|
)
|
|
@@ -267,7 +271,7 @@ def _execute_tasks_with_dependencies(
|
|
|
267
271
|
future_queue.task_done()
|
|
268
272
|
future_queue.join()
|
|
269
273
|
break
|
|
270
|
-
|
|
274
|
+
elif ( # handle internal tasks for getting and setting information about the executor
|
|
271
275
|
task_dict is not None and "internal" in task_dict and task_dict["internal"]
|
|
272
276
|
):
|
|
273
277
|
if task_dict["task"] == "get_info":
|
|
@@ -281,12 +285,24 @@ def _execute_tasks_with_dependencies(
|
|
|
281
285
|
task_dict["future"].set_result(False)
|
|
282
286
|
else:
|
|
283
287
|
task_dict["future"].set_result(True)
|
|
288
|
+
elif ( # handle batched function submitted to the executor
|
|
289
|
+
task_dict is not None
|
|
290
|
+
and "fn" in task_dict
|
|
291
|
+
and task_dict["fn"] == "batched"
|
|
292
|
+
and "future" in task_dict
|
|
293
|
+
):
|
|
294
|
+
future_dependency_lst.append(task_dict)
|
|
295
|
+
future_queue.task_done()
|
|
284
296
|
elif ( # handle function submitted to the executor
|
|
285
|
-
task_dict is not None
|
|
297
|
+
task_dict is not None
|
|
298
|
+
and "fn" in task_dict
|
|
299
|
+
and task_dict["fn"] != "batched"
|
|
300
|
+
and "future" in task_dict
|
|
286
301
|
):
|
|
287
|
-
future_lst
|
|
302
|
+
future_lst = get_future_objects_from_input(
|
|
288
303
|
args=task_dict["args"], kwargs=task_dict["kwargs"]
|
|
289
304
|
)
|
|
305
|
+
ready_flag = check_list_of_futures_is_done(future_lst=future_lst)
|
|
290
306
|
exception_lst = get_exception_lst(future_lst=future_lst)
|
|
291
307
|
if not check_exception_was_raised(future_obj=task_dict["future"]):
|
|
292
308
|
if len(exception_lst) > 0:
|
|
@@ -299,12 +315,12 @@ def _execute_tasks_with_dependencies(
|
|
|
299
315
|
executor_queue.put(task_dict)
|
|
300
316
|
else: # Otherwise add the function to the wait list
|
|
301
317
|
task_dict["future_lst"] = future_lst
|
|
302
|
-
|
|
318
|
+
future_dependency_lst.append(task_dict)
|
|
303
319
|
future_queue.task_done()
|
|
304
|
-
elif len(
|
|
320
|
+
elif len(future_dependency_lst) > 0:
|
|
305
321
|
# Check functions in the wait list and execute them if all future objects are now ready
|
|
306
|
-
|
|
307
|
-
|
|
322
|
+
future_dependency_lst = _handle_future_dependencies(
|
|
323
|
+
future_dependency_lst=future_dependency_lst,
|
|
308
324
|
executor_queue=executor_queue,
|
|
309
325
|
refresh_rate=refresh_rate,
|
|
310
326
|
)
|
|
@@ -313,14 +329,16 @@ def _execute_tasks_with_dependencies(
|
|
|
313
329
|
sleep(refresh_rate)
|
|
314
330
|
|
|
315
331
|
|
|
316
|
-
def
|
|
317
|
-
|
|
332
|
+
def _handle_future_dependencies(
|
|
333
|
+
future_dependency_lst: list[dict],
|
|
334
|
+
executor_queue: queue.Queue,
|
|
335
|
+
refresh_rate: float = 0.01,
|
|
318
336
|
) -> list:
|
|
319
337
|
"""
|
|
320
338
|
Submit the waiting tasks, which future inputs have been completed, to the executor
|
|
321
339
|
|
|
322
340
|
Args:
|
|
323
|
-
|
|
341
|
+
future_dependency_lst (list): List of waiting tasks
|
|
324
342
|
executor_queue (Queue): Queue of the internal executor
|
|
325
343
|
refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
|
|
326
344
|
|
|
@@ -328,9 +346,9 @@ def _update_waiting_task(
|
|
|
328
346
|
list: list tasks which future inputs have not been completed
|
|
329
347
|
"""
|
|
330
348
|
wait_tmp_lst = []
|
|
331
|
-
for task_wait_dict in
|
|
349
|
+
for task_wait_dict in future_dependency_lst:
|
|
332
350
|
exception_lst = get_exception_lst(future_lst=task_wait_dict["future_lst"])
|
|
333
|
-
if len(exception_lst) > 0:
|
|
351
|
+
if len(exception_lst) > 0 and task_wait_dict["fn"] != "batched":
|
|
334
352
|
task_wait_dict["future"].set_exception(exception_lst[0])
|
|
335
353
|
elif task_wait_dict["fn"] != "batched" and all(
|
|
336
354
|
future.done() for future in task_wait_dict["future_lst"]
|
|
@@ -343,17 +361,21 @@ def _update_waiting_task(
|
|
|
343
361
|
elif task_wait_dict["fn"] == "batched" and all(
|
|
344
362
|
future.done() for future in task_wait_dict["kwargs"]["skip_lst"]
|
|
345
363
|
):
|
|
346
|
-
done_lst = batched_futures(
|
|
364
|
+
success, done_lst = batched_futures(
|
|
347
365
|
lst=task_wait_dict["kwargs"]["lst"],
|
|
348
366
|
n=task_wait_dict["kwargs"]["n"],
|
|
349
|
-
|
|
367
|
+
nested_skip_lst=task_wait_dict["kwargs"]["skip_lst"],
|
|
350
368
|
)
|
|
351
|
-
if len(done_lst) == 0:
|
|
369
|
+
if success and len(done_lst) == 0:
|
|
352
370
|
wait_tmp_lst.append(task_wait_dict)
|
|
371
|
+
elif success and len(done_lst) > 0:
|
|
372
|
+
task_wait_dict["future"].set_result([f.result() for f in done_lst])
|
|
373
|
+
task_wait_dict["future_skip"].set_result([id(f) for f in done_lst])
|
|
353
374
|
else:
|
|
354
|
-
task_wait_dict["future"].
|
|
375
|
+
task_wait_dict["future"].set_exception(done_lst[0].exception())
|
|
376
|
+
task_wait_dict["future_skip"].set_result([id(f) for f in done_lst])
|
|
355
377
|
else:
|
|
356
378
|
wait_tmp_lst.append(task_wait_dict)
|
|
357
|
-
if len(
|
|
379
|
+
if len(future_dependency_lst) == len(wait_tmp_lst):
|
|
358
380
|
sleep(refresh_rate)
|
|
359
381
|
return wait_tmp_lst
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: executorlib
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.4
|
|
4
4
|
Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
5
5
|
Project-URL: Homepage, https://github.com/pyiron/executorlib
|
|
6
6
|
Project-URL: Documentation, https://executorlib.readthedocs.io
|
|
@@ -36,7 +36,7 @@ License: BSD 3-Clause License
|
|
|
36
36
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
37
37
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
38
38
|
License-File: LICENSE
|
|
39
|
-
Keywords: executor,flux-framework,high performance computing,hpc,slurm,task scheduler
|
|
39
|
+
Keywords: concurrent.futures,distributed computing,executor,flux-framework,gpu,high performance computing,hpc,mpi4py,parallel computing,python,scientific computing,slurm,task scheduler
|
|
40
40
|
Classifier: Development Status :: 5 - Production/Stable
|
|
41
41
|
Classifier: Intended Audience :: Science/Research
|
|
42
42
|
Classifier: License :: OSI Approved :: BSD License
|
|
@@ -52,16 +52,16 @@ Requires-Dist: pyzmq<=27.1.0,>=25.0.0
|
|
|
52
52
|
Provides-Extra: all
|
|
53
53
|
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'all'
|
|
54
54
|
Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'all'
|
|
55
|
-
Requires-Dist: mpi4py<=4.1.
|
|
55
|
+
Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
|
|
56
56
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
|
|
57
57
|
Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'all'
|
|
58
58
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
|
|
59
|
-
Requires-Dist: pysqa==0.4.
|
|
59
|
+
Requires-Dist: pysqa==0.4.2; extra == 'all'
|
|
60
60
|
Provides-Extra: cache
|
|
61
61
|
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
|
|
62
62
|
Provides-Extra: cluster
|
|
63
63
|
Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
|
|
64
|
-
Requires-Dist: pysqa==0.4.
|
|
64
|
+
Requires-Dist: pysqa==0.4.2; extra == 'cluster'
|
|
65
65
|
Provides-Extra: graph
|
|
66
66
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
|
|
67
67
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
|
|
@@ -70,7 +70,7 @@ Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'graphnotebook'
|
|
|
70
70
|
Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graphnotebook'
|
|
71
71
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graphnotebook'
|
|
72
72
|
Provides-Extra: mpi
|
|
73
|
-
Requires-Dist: mpi4py<=4.1.
|
|
73
|
+
Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'mpi'
|
|
74
74
|
Provides-Extra: validation
|
|
75
75
|
Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'validation'
|
|
76
76
|
Description-Content-Type: text/markdown
|
|
@@ -96,6 +96,26 @@ Up-scale python functions for high performance computing (HPC) with executorlib.
|
|
|
96
96
|
machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for
|
|
97
97
|
iterative development in interactive environments like jupyter notebooks.
|
|
98
98
|
|
|
99
|
+
## Why executorlib?
|
|
100
|
+
executorlib is the lightest path to take *existing* Python functions and scale them across high performance computing
|
|
101
|
+
(HPC) nodes — with per-function-call resource control and native [SLURM](https://slurm.schedmd.com) and
|
|
102
|
+
[flux](http://flux-framework.org) integration — without rewriting your code into a new paradigm. It extends the standard
|
|
103
|
+
library [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects) you already
|
|
104
|
+
know, rather than asking you to adopt a new data, actor, or workflow model.
|
|
105
|
+
|
|
106
|
+
| | executorlib | [Futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
|
|
107
|
+
|---|---|---|---|---|---|
|
|
108
|
+
| Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️ | ❌ |
|
|
109
|
+
| Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |
|
|
110
|
+
| Native HPC scheduler (SLURM/flux) | ✅ | ❌ | ⚠️ | ✅ | ⚠️ |
|
|
111
|
+
| MPI-parallel functions | ✅ | ❌ | ⚠️ | ⚠️ | ⚠️ |
|
|
112
|
+
| Caching of results | ✅ | ❌ | ⚠️ | ✅ | ❌ |
|
|
113
|
+
| Setup / learning overhead | Low | Very low | Medium | Medium | Medium |
|
|
114
|
+
|
|
115
|
+
✅ first-class · ⚠️ possible via add-on/config · ❌ not supported. See the full
|
|
116
|
+
[comparison: when to use which](https://executorlib.readthedocs.io/en/latest/comparison.html) for honest guidance on
|
|
117
|
+
when another tool is the better fit.
|
|
118
|
+
|
|
99
119
|
## Examples
|
|
100
120
|
The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
|
|
101
121
|
with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the
|
|
@@ -207,6 +227,13 @@ as hierarchical job scheduler within the allocations.
|
|
|
207
227
|
* [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-job-executor)
|
|
208
228
|
* [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
|
|
209
229
|
* [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
|
|
230
|
+
* [Comparison](https://executorlib.readthedocs.io/en/latest/comparison.html#comparison)
|
|
231
|
+
* [At a glace](https://executorlib.readthedocs.io/en/latest/comparison.html#at-a-glance)
|
|
232
|
+
* [Concurrent futures](https://executorlib.readthedocs.io/en/latest/comparison.html#concurrent-futures-the-python-standard-library)
|
|
233
|
+
* [Dask](https://executorlib.readthedocs.io/en/latest/comparison.html#dask)
|
|
234
|
+
* [Parsl](https://executorlib.readthedocs.io/en/latest/comparison.html#parsl)
|
|
235
|
+
* [Ray](https://executorlib.readthedocs.io/en/latest/comparison.html#ray)
|
|
236
|
+
* [Choose executorlib when](https://executorlib.readthedocs.io/en/latest/comparison.html#choose-executorlib-when)
|
|
210
237
|
* [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html)
|
|
211
238
|
* [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
|
|
212
239
|
* [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
executorlib/__init__.py,sha256=IY_DkT706LPIv2_iIe3whrQNGoDzaDFYBvXyUepoZpw,4780
|
|
2
|
-
executorlib/_version.py,sha256=
|
|
2
|
+
executorlib/_version.py,sha256=VdLE9NNjOWcm4_Kh_W9w_lFN3bh0oesrwyTL3nQnUKg,520
|
|
3
3
|
executorlib/api.py,sha256=F6ibDJnVcPGxxPe-UScFJHprWHiVsETGhgAvS1T-9ZI,1365
|
|
4
4
|
executorlib/backend/__init__.py,sha256=2b_iwds4VUrJe92nKtQXkoGxkJiXeEDSlbvCRN_-de8,456
|
|
5
5
|
executorlib/backend/cache_parallel.py,sha256=ldb2iYzrhcXJZ0LcH75FpTLO_6F51mleU9JGAilF00k,2020
|
|
@@ -7,44 +7,44 @@ executorlib/backend/cache_serial.py,sha256=9T3bAxVivGxvzHnXmvysyEPP4sw--VCwXCn2M
|
|
|
7
7
|
executorlib/backend/interactive_parallel.py,sha256=HimGnWOdTXInC3o7W4wMUSTTlj6MsMPLJLkcNTaK_SQ,3869
|
|
8
8
|
executorlib/backend/interactive_serial.py,sha256=NxlSX566rQM-pHiy2rQja2rr3lFwdnv0EvIyXk8H7co,2982
|
|
9
9
|
executorlib/executor/__init__.py,sha256=oR2ABLAMYV_TNs1eG-3-MAKHFS7OKB-1KOzCqKNbtWA,310
|
|
10
|
-
executorlib/executor/base.py,sha256=
|
|
10
|
+
executorlib/executor/base.py,sha256=UKJViNoshthAgF2I4FoK0nZT0FL7WgkrzWFYhRiU1H4,6957
|
|
11
11
|
executorlib/executor/flux.py,sha256=g8K1GMfn9KMM4udj3LdmDMahavPBBCOusYVG36W1hAY,40779
|
|
12
12
|
executorlib/executor/single.py,sha256=U-mflJIQSBMS7atK7RQktx87LGkJhLSBbh2Uz9kOLlY,36021
|
|
13
13
|
executorlib/executor/slurm.py,sha256=Prvts9IgDHgCCMYoWpNdeoKMZ4h1ICQ05fkomh25Yl4,38337
|
|
14
14
|
executorlib/standalone/__init__.py,sha256=n_Q4NUFu35Fn0qlWIhmoC_FyUkmFGlWn1cNc_7EscF0,290
|
|
15
|
-
executorlib/standalone/batched.py,sha256=
|
|
16
|
-
executorlib/standalone/command.py,sha256=
|
|
15
|
+
executorlib/standalone/batched.py,sha256=o3TtsFJir8nP1O_NZFfW0Q78CoZAFgHMyroWlg520nE,1446
|
|
16
|
+
executorlib/standalone/command.py,sha256=4Xndog84zJdku1ke1DwdUamkWvzRrMz-VDCi5cRXi0M,6548
|
|
17
17
|
executorlib/standalone/error.py,sha256=aaMQoRsoLzLCbYT-E4LZxGM9pm_AhjMesRjVt0d_LDk,714
|
|
18
|
-
executorlib/standalone/hdf.py,sha256=
|
|
19
|
-
executorlib/standalone/inputcheck.py,sha256=
|
|
18
|
+
executorlib/standalone/hdf.py,sha256=nE2mTH1U8O7sm2TPJkfRU0uEgWJOhLnYxI6yZord8Eo,7623
|
|
19
|
+
executorlib/standalone/inputcheck.py,sha256=kSWUzqdMOhILYXW4Isumn1EGH3SegxPUMWRkUWfEleU,10041
|
|
20
20
|
executorlib/standalone/queue.py,sha256=KFEBFQXGvcaf5qcuCjT5kaZg7ucWOTGOtrD96tIBlq0,589
|
|
21
21
|
executorlib/standalone/scheduler.py,sha256=LhCuIdaw9BjrP3GerbHUIqwsvxERdtH5JQoeyAqASIU,2325
|
|
22
|
-
executorlib/standalone/select.py,sha256=
|
|
23
|
-
executorlib/standalone/serialize.py,sha256=
|
|
24
|
-
executorlib/standalone/validate.py,sha256=
|
|
22
|
+
executorlib/standalone/select.py,sha256=FUSXYbhoc_uP_vPji8L4PcK4b2Gwcai7rXZ5dKRVjZM,3061
|
|
23
|
+
executorlib/standalone/serialize.py,sha256=7DxcuCYZVqLf8wa7EXsN-_8fSpUJqyQ-5G6NT863SgM,4716
|
|
24
|
+
executorlib/standalone/validate.py,sha256=hHG32xomLhbqnqd6Vn4AO05bQ7B3WZ1rngTkMsODwwk,4430
|
|
25
25
|
executorlib/standalone/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
-
executorlib/standalone/interactive/arguments.py,sha256=
|
|
26
|
+
executorlib/standalone/interactive/arguments.py,sha256=iaVXf7Wq9MUOu3WeYSi2t-f8Si_pyAPyykcnq8bf8cY,3239
|
|
27
27
|
executorlib/standalone/interactive/backend.py,sha256=gGgcvER8w75bM_p1uplSsr6yCtBBlOS_6wR1iQNsntc,3006
|
|
28
|
-
executorlib/standalone/interactive/communication.py,sha256=
|
|
28
|
+
executorlib/standalone/interactive/communication.py,sha256=ZYngnW-myohI5jjF1RDHfMEX8dVDN2Ow6kBjMVkAfxw,10448
|
|
29
29
|
executorlib/standalone/interactive/spawner.py,sha256=4TEPL4iI_rF25kOaW-eqY74jK93LFZ05cYOHUjIuo4M,6384
|
|
30
30
|
executorlib/task_scheduler/__init__.py,sha256=-sw60rfnEvwFiOAONFHpE3izPA9CPtp5cQ7vRetw2Mw,249
|
|
31
|
-
executorlib/task_scheduler/base.py,sha256=
|
|
31
|
+
executorlib/task_scheduler/base.py,sha256=A9nQb8FYKbZGjQP1G5ccT75ODbEqqiM0OVybHAxjqzE,9718
|
|
32
32
|
executorlib/task_scheduler/file/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
executorlib/task_scheduler/file/backend.py,sha256=tojXcYytAsfABugsOMfkx9oAh01do_DcSneVldMfeK0,2876
|
|
34
|
-
executorlib/task_scheduler/file/shared.py,sha256=
|
|
34
|
+
executorlib/task_scheduler/file/shared.py,sha256=zRODWHzqZBzrEjmJenBNBwwM2mqMcnNIXpOfTGsyYBo,21128
|
|
35
35
|
executorlib/task_scheduler/file/spawner_pysqa.py,sha256=j7GdUQgo6vZKqbXtdXL3doaCK8InrDdPJOn0kWGY7-4,6032
|
|
36
36
|
executorlib/task_scheduler/file/spawner_subprocess.py,sha256=BejOMi5zLkIVUUywv5hl0YdnXkGujGo4nXMpmmuHq2k,3089
|
|
37
37
|
executorlib/task_scheduler/file/task_scheduler.py,sha256=c57K8d3Sfh4p--xvAy5LsCtTq9GOdymaUeN0h4POyS8,6019
|
|
38
38
|
executorlib/task_scheduler/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
39
|
executorlib/task_scheduler/interactive/blockallocation.py,sha256=HcfrYEuopD7wLUoxuoawRRNdaH4dTsPnFTb7luR7G5Y,17738
|
|
40
|
-
executorlib/task_scheduler/interactive/dependency.py,sha256=
|
|
40
|
+
executorlib/task_scheduler/interactive/dependency.py,sha256=zQMfI6AjeUv3A1qyPzFah2BKYGWgGLRyEIdZ0XTjq1M,15222
|
|
41
41
|
executorlib/task_scheduler/interactive/dependency_plot.py,sha256=iZBZ9UMDeYOMFjD2-kYOxOdKeodj7TuyshI4HbH4XCU,11788
|
|
42
42
|
executorlib/task_scheduler/interactive/onetoone.py,sha256=cTXgTXR9kKzzlx6EfGumMsEm7DUa7BD9JSeBpdPiupI,13312
|
|
43
43
|
executorlib/task_scheduler/interactive/shared.py,sha256=-krntmZFOOE5U0HdFwuU8JZPqZQmPW4-cePDWuvkTgM,6305
|
|
44
44
|
executorlib/task_scheduler/interactive/spawner_flux.py,sha256=M7CPWvzY7MDkplJ4awymh7WbMYwKsDPTKxA2Op9wfNo,7075
|
|
45
45
|
executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=96NqUfoIBUFd7KjY02dpIL9ZoS_u5beRuNw_1FUKH6Q,10393
|
|
46
46
|
executorlib/task_scheduler/interactive/spawner_slurm.py,sha256=K5mj63PE8FYbkw9WWm4uEjoA5q0HM42z1d2wjAmO5YA,3713
|
|
47
|
-
executorlib-1.9.
|
|
48
|
-
executorlib-1.9.
|
|
49
|
-
executorlib-1.9.
|
|
50
|
-
executorlib-1.9.
|
|
47
|
+
executorlib-1.9.4.dist-info/METADATA,sha256=4augHmAbGQ-7H0r4Zh6vaFvTh-yTwZMSyuLVYsp7-Qo,18249
|
|
48
|
+
executorlib-1.9.4.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
49
|
+
executorlib-1.9.4.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
|
|
50
|
+
executorlib-1.9.4.dist-info/RECORD,,
|
|
File without changes
|