executorlib 1.9.2__py3-none-any.whl → 1.9.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
executorlib/_version.py CHANGED
@@ -1,5 +1,6 @@
1
- # file generated by setuptools-scm
1
+ # file generated by vcs-versioning
2
2
  # don't change, don't track in version control
3
+ from __future__ import annotations
3
4
 
4
5
  __all__ = [
5
6
  "__version__",
@@ -10,25 +11,14 @@ __all__ = [
10
11
  "commit_id",
11
12
  ]
12
13
 
13
- TYPE_CHECKING = False
14
- if TYPE_CHECKING:
15
- from typing import Tuple
16
- from typing import Union
17
-
18
- VERSION_TUPLE = Tuple[Union[int, str], ...]
19
- COMMIT_ID = Union[str, None]
20
- else:
21
- VERSION_TUPLE = object
22
- COMMIT_ID = object
23
-
24
14
  version: str
25
15
  __version__: str
26
- __version_tuple__: VERSION_TUPLE
27
- version_tuple: VERSION_TUPLE
28
- commit_id: COMMIT_ID
29
- __commit_id__: COMMIT_ID
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
30
20
 
31
- __version__ = version = '1.9.2'
32
- __version_tuple__ = version_tuple = (1, 9, 2)
21
+ __version__ = version = '1.9.4'
22
+ __version_tuple__ = version_tuple = (1, 9, 4)
33
23
 
34
24
  __commit_id__ = commit_id = None
@@ -25,10 +25,22 @@ class BaseExecutor(FutureExecutor, ABC):
25
25
 
26
26
  @property
27
27
  def max_workers(self) -> Optional[int]:
28
+ """
29
+ Return the number of parallel workers configured for this executor.
30
+
31
+ Returns:
32
+ Optional[int]: The maximum number of parallel workers, or None if unconstrained.
33
+ """
28
34
  return self._task_scheduler.max_workers
29
35
 
30
36
  @max_workers.setter
31
37
  def max_workers(self, max_workers: int):
38
+ """
39
+ Set the number of parallel workers on the underlying task scheduler.
40
+
41
+ Args:
42
+ max_workers (int): New maximum number of parallel workers.
43
+ """
32
44
  self._task_scheduler.max_workers = max_workers
33
45
 
34
46
  @property
@@ -51,8 +51,8 @@ class FluxJobExecutor(BaseExecutor):
51
51
  do not allow other tasks to use the same compute node.
52
52
  * error_log_file (str): path to the error log file, primarily used to merge the log of
53
53
  multiple tasks in one file.
54
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
55
- allowed to take in seconds.
54
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
55
+ allowed to take in seconds.
56
56
  * priority (int): the queuing system priority assigned to a given Python function to
57
57
  influence the scheduling.
58
58
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -154,7 +154,7 @@ class FluxJobExecutor(BaseExecutor):
154
154
  - do not allow other tasks to use the same compute node.
155
155
  * error_log_file (str): path to the error log file, primarily used to merge the log of
156
156
  multiple tasks in one file.
157
- * run_time_limit (int): the maximum time the execution of the submitted Python
157
+ * run_time_max (int): the maximum time the execution of the submitted Python
158
158
  function is allowed to take in seconds.
159
159
  * priority (int): the queuing system priority assigned to a given Python function to
160
160
  influence the scheduling.
@@ -283,8 +283,8 @@ class FluxClusterExecutor(BaseExecutor):
283
283
  do not allow other tasks to use the same compute node.
284
284
  * error_log_file (str): path to the error log file, primarily used to merge the log of
285
285
  multiple tasks in one file.
286
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
287
- allowed to take in seconds.
286
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
287
+ allowed to take in seconds.
288
288
  * priority (int): the queuing system priority assigned to a given Python function to
289
289
  influence the scheduling.
290
290
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -380,7 +380,7 @@ class FluxClusterExecutor(BaseExecutor):
380
380
  - do not allow other tasks to use the same compute node.
381
381
  * error_log_file (str): path to the error log file, primarily used to merge the log of
382
382
  multiple tasks in one file.
383
- * run_time_limit (int): the maximum time the execution of the submitted Python
383
+ * run_time_max (int): the maximum time the execution of the submitted Python
384
384
  function is allowed to take in seconds.
385
385
  * priority (int): the queuing system priority assigned to a given Python function to
386
386
  influence the scheduling.
@@ -417,7 +417,7 @@ class FluxClusterExecutor(BaseExecutor):
417
417
  "cwd": None,
418
418
  "openmpi_oversubscribe": openmpi_oversubscribe,
419
419
  "slurm_cmd_args": [],
420
- "run_time_limit": None,
420
+ "run_time_max": None,
421
421
  }
422
422
  if resource_dict is None:
423
423
  resource_dict = {}
@@ -540,8 +540,8 @@ def create_flux_executor(
540
540
  do not allow other tasks to use the same compute node.
541
541
  * error_log_file (str): path to the error log file, primarily used to merge the log of
542
542
  multiple tasks in one file.
543
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
544
- allowed to take in seconds.
543
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
544
+ allowed to take in seconds.
545
545
  * priority (int): the queuing system priority assigned to a given Python function to
546
546
  influence the scheduling.
547
547
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -50,8 +50,8 @@ class SingleNodeExecutor(BaseExecutor):
50
50
  do not allow other tasks to use the same compute node.
51
51
  * error_log_file (str): path to the error log file, primarily used to merge the log of
52
52
  multiple tasks in one file.
53
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
54
- allowed to take in seconds.
53
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
54
+ allowed to take in seconds.
55
55
  * priority (int): the queuing system priority assigned to a given Python function to
56
56
  influence the scheduling.
57
57
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -145,7 +145,7 @@ class SingleNodeExecutor(BaseExecutor):
145
145
  - do not allow other tasks to use the same compute node.
146
146
  * error_log_file (str): path to the error log file, primarily used to merge the log of
147
147
  multiple tasks in one file.
148
- * run_time_limit (int): the maximum time the execution of the submitted Python
148
+ * run_time_max (int): the maximum time the execution of the submitted Python
149
149
  function is allowed to take in seconds.
150
150
  * priority (int): the queuing system priority assigned to a given Python function to
151
151
  influence the scheduling.
@@ -259,8 +259,8 @@ class TestClusterExecutor(BaseExecutor):
259
259
  do not allow other tasks to use the same compute node.
260
260
  * error_log_file (str): path to the error log file, primarily used to merge the log of
261
261
  multiple tasks in one file.
262
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
263
- allowed to take in seconds.
262
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
263
+ allowed to take in seconds.
264
264
  * priority (int): the queuing system priority assigned to a given Python function to
265
265
  influence the scheduling.
266
266
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -350,7 +350,7 @@ class TestClusterExecutor(BaseExecutor):
350
350
  - do not allow other tasks to use the same compute node.
351
351
  * error_log_file (str): path to the error log file, primarily used to merge the log of
352
352
  multiple tasks in one file.
353
- * run_time_limit (int): the maximum time the execution of the submitted Python
353
+ * run_time_max (int): the maximum time the execution of the submitted Python
354
354
  function is allowed to take in seconds.
355
355
  * priority (int): the queuing system priority assigned to a given Python function to
356
356
  influence the scheduling.
@@ -479,8 +479,8 @@ def create_single_node_executor(
479
479
  do not allow other tasks to use the same compute node.
480
480
  * error_log_file (str): path to the error log file, primarily used to merge the log of
481
481
  multiple tasks in one file.
482
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
483
- allowed to take in seconds.
482
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
483
+ allowed to take in seconds.
484
484
  * priority (int): the queuing system priority assigned to a given Python function to
485
485
  influence the scheduling.
486
486
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -52,8 +52,8 @@ class SlurmClusterExecutor(BaseExecutor):
52
52
  do not allow other tasks to use the same compute node.
53
53
  * error_log_file (str): path to the error log file, primarily used to merge the log of
54
54
  multiple tasks in one file.
55
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
56
- allowed to take in seconds.
55
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
56
+ allowed to take in seconds.
57
57
  * priority (int): the queuing system priority assigned to a given Python function to
58
58
  influence the scheduling.
59
59
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -149,7 +149,7 @@ class SlurmClusterExecutor(BaseExecutor):
149
149
  - do not allow other tasks to use the same compute node.
150
150
  * error_log_file (str): path to the error log file, primarily used to merge the log of
151
151
  multiple tasks in one file.
152
- * run_time_limit (int): the maximum time the execution of the submitted Python
152
+ * run_time_max (int): the maximum time the execution of the submitted Python
153
153
  function is allowed to take in seconds.
154
154
  * priority (int): the queuing system priority assigned to a given Python function to
155
155
  influence the scheduling.
@@ -293,8 +293,8 @@ class SlurmJobExecutor(BaseExecutor):
293
293
  do not allow other tasks to use the same compute node.
294
294
  * error_log_file (str): path to the error log file, primarily used to merge the log of
295
295
  multiple tasks in one file.
296
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
297
- allowed to take in seconds.
296
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
297
+ allowed to take in seconds.
298
298
  * priority (int): the queuing system priority assigned to a given Python function to
299
299
  influence the scheduling.
300
300
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -390,7 +390,7 @@ class SlurmJobExecutor(BaseExecutor):
390
390
  - do not allow other tasks to use the same compute node.
391
391
  * error_log_file (str): path to the error log file, primarily used to merge the log of
392
392
  multiple tasks in one file.
393
- * run_time_limit (int): the maximum time the execution of the submitted Python
393
+ * run_time_max (int): the maximum time the execution of the submitted Python
394
394
  function is allowed to take in seconds.
395
395
  * priority (int): the queuing system priority assigned to a given Python function to
396
396
  influence the scheduling.
@@ -518,8 +518,8 @@ def create_slurm_executor(
518
518
  do not allow other tasks to use the same compute node.
519
519
  * error_log_file (str): path to the error log file, primarily used to merge the log of
520
520
  multiple tasks in one file.
521
- * run_time_limit (int): the maximum time the execution of the submitted Python function is
522
- allowed to take in seconds.
521
+ * run_time_max (int): the maximum time the execution of the submitted Python function is
522
+ allowed to take in seconds.
523
523
  * priority (int): the queuing system priority assigned to a given Python function to
524
524
  influence the scheduling.
525
525
  * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
@@ -1,27 +1,39 @@
1
1
  from concurrent.futures import Future
2
2
 
3
3
 
4
- def batched_futures(lst: list[Future], skip_lst: list[list], n: int) -> list[list]:
4
+ def batched_futures(
5
+ lst: list[Future], nested_skip_lst: list[Future[list]], n: int
6
+ ) -> tuple[bool, list[Future]]:
5
7
  """
6
8
  Batch n completed future objects. If the number of completed futures is smaller than n and the end of the batch is
7
- not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_lst
9
+ not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_set
8
10
  then they are returned as batch.
9
11
 
10
12
  Args:
11
13
  lst (list): list of all future objects
12
- skip_lst (list): list of previous batches of future objects
14
+ nested_skip_lst (list): list of future objects, which contain the list of future objects ids which should be skipped for the batch
13
15
  n (int): batch size
14
16
 
15
17
  Returns:
16
18
  list: results of the batched futures
17
19
  """
18
- skipped_elements_lst = [item for items in skip_lst for item in items]
20
+ skip_set = {fid for f in nested_skip_lst for fid in f.result()}
19
21
 
20
22
  done_lst = []
21
- n_expected = min(n, len(lst) - len(skipped_elements_lst))
23
+ failed_lst = []
24
+ n_expected = min(n, len(lst) - len(skip_set))
22
25
  for v in lst:
23
- if v.done() and v.result() not in skipped_elements_lst:
24
- done_lst.append(v.result())
25
- if len(done_lst) == n_expected:
26
- return done_lst
27
- return []
26
+ if id(v) not in skip_set and v.done():
27
+ if v.exception() is not None:
28
+ failed_lst.append(v)
29
+ elif id(v) not in skip_set and v.done():
30
+ done_lst.append(v)
31
+ if len(done_lst) == n_expected:
32
+ return True, done_lst
33
+ if (len(lst) - len(skip_set)) == len(failed_lst):
34
+ return (
35
+ False,
36
+ failed_lst[:n_expected],
37
+ ) # raise the exception only after all futures have failed
38
+ else:
39
+ return True, []
@@ -126,7 +126,7 @@ def generate_slurm_command(
126
126
  openmpi_oversubscribe: bool = False,
127
127
  slurm_cmd_args: Optional[list[str]] = None,
128
128
  pmi_mode: Optional[str] = None,
129
- run_time_limit: Optional[int] = None,
129
+ run_time_max: Optional[int] = None,
130
130
  ) -> list[str]:
131
131
  """
132
132
  Generate the command list for the SLURM interface.
@@ -141,7 +141,7 @@ def generate_slurm_command(
141
141
  openmpi_oversubscribe (bool, optional): Whether to oversubscribe the cores. Defaults to False.
142
142
  slurm_cmd_args (list[str], optional): Additional command line arguments. Defaults to [].
143
143
  pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
144
- run_time_limit (int): The maximum runtime in seconds for each task. Default: None
144
+ run_time_max (int): The maximum runtime in seconds for each task. Default: None
145
145
 
146
146
  Returns:
147
147
  list[str]: The generated command list.
@@ -153,7 +153,7 @@ def generate_slurm_command(
153
153
  command_prepend_lst += ["--mpi=" + pmi_mode]
154
154
  if num_nodes is not None:
155
155
  command_prepend_lst += ["-N", str(num_nodes)]
156
- if threads_per_core > 1:
156
+ if threads_per_core >= 1:
157
157
  command_prepend_lst += ["--cpus-per-task=" + str(threads_per_core)]
158
158
  if gpus_per_core > 0:
159
159
  command_prepend_lst += ["--gpus-per-task=" + str(gpus_per_core)]
@@ -161,8 +161,8 @@ def generate_slurm_command(
161
161
  command_prepend_lst += ["--exact"]
162
162
  if openmpi_oversubscribe:
163
163
  command_prepend_lst += ["--oversubscribe"]
164
- if run_time_limit is not None:
165
- command_prepend_lst += ["--time=" + str(run_time_limit // 60 + 1)]
164
+ if run_time_max is not None:
165
+ command_prepend_lst += ["--time=" + str(run_time_max // 60 + 1)]
166
166
  if slurm_cmd_args is not None and len(slurm_cmd_args) > 0:
167
167
  command_prepend_lst += slurm_cmd_args
168
168
  return command_prepend_lst
@@ -1,3 +1,4 @@
1
+ import contextlib
1
2
  import os
2
3
  from concurrent.futures import Future
3
4
  from time import sleep
@@ -34,10 +35,11 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
34
35
  with h5py.File(file_name_abs, "a") as fname:
35
36
  for data_key, data_value in data_dict.items():
36
37
  if data_key in group_dict:
37
- fname.create_dataset(
38
- name="/" + group_dict[data_key],
39
- data=np.void(cloudpickle.dumps(data_value)),
40
- )
38
+ with contextlib.suppress(ValueError):
39
+ fname.create_dataset(
40
+ name="/" + group_dict[data_key],
41
+ data=np.void(cloudpickle.dumps(data_value)),
42
+ )
41
43
 
42
44
 
43
45
  def load(file_name: str) -> dict:
@@ -171,7 +173,11 @@ def get_cache_files(cache_directory: str) -> list[str]:
171
173
  file_lst = []
172
174
  cache_directory_abs = os.path.abspath(cache_directory)
173
175
  for dirpath, _, filenames in os.walk(cache_directory_abs):
174
- file_lst += [os.path.join(dirpath, f) for f in filenames if f.endswith("_o.h5")]
176
+ file_lst += [
177
+ os.path.join(dirpath, f)
178
+ for f in filenames
179
+ if f.endswith("_o.h5") or f.endswith("_i.h5")
180
+ ]
175
181
  return file_lst
176
182
 
177
183
 
@@ -141,6 +141,16 @@ def check_init_function(
141
141
  def check_max_workers_and_cores(
142
142
  max_workers: Optional[int], max_cores: Optional[int]
143
143
  ) -> None:
144
+ """
145
+ Check that neither max_workers nor max_cores is set when using a pysqa-based backend.
146
+
147
+ Args:
148
+ max_workers (int, optional): Maximum number of workers.
149
+ max_cores (int, optional): Maximum number of cores.
150
+
151
+ Raises:
152
+ ValueError: If max_workers or max_cores is not None.
153
+ """
144
154
  if max_workers is not None:
145
155
  raise ValueError(
146
156
  "The number of workers cannot be controlled with the pysqa based backend."
@@ -152,6 +162,15 @@ def check_max_workers_and_cores(
152
162
 
153
163
 
154
164
  def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
165
+ """
166
+ Check that hostname_localhost is not set when using a pysqa-based backend.
167
+
168
+ Args:
169
+ hostname_localhost (bool, optional): Flag to use localhost for ZMQ connections.
170
+
171
+ Raises:
172
+ ValueError: If hostname_localhost is not None.
173
+ """
155
174
  if hostname_localhost is not None:
156
175
  raise ValueError(
157
176
  "The option to connect to hosts based on their hostname is not available with the pysqa based backend."
@@ -159,6 +178,16 @@ def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None:
159
178
 
160
179
 
161
180
  def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> None:
181
+ """
182
+ Check that restart_limit is only used together with block_allocation.
183
+
184
+ Args:
185
+ restart_limit (int): Maximum number of times a worker process may be restarted.
186
+ block_allocation (bool): Whether block allocation is enabled. Defaults to True.
187
+
188
+ Raises:
189
+ ValueError: If restart_limit is non-zero and block_allocation is False.
190
+ """
162
191
  if not block_allocation and restart_limit != 0:
163
192
  raise ValueError(
164
193
  "The option to specify a restart limit for worker processes is only available with block_allocation=True."
@@ -166,6 +195,15 @@ def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> No
166
195
 
167
196
 
168
197
  def check_pmi_mode(pmi_mode: Optional[str]) -> None:
198
+ """
199
+ Check that pmi_mode is not set on a local workstation without SLURM or flux.
200
+
201
+ Args:
202
+ pmi_mode (str, optional): PMI interface name (e.g. "pmix", "pmi1", "pmi2").
203
+
204
+ Raises:
205
+ ValueError: If pmi_mode is not None.
206
+ """
169
207
  if pmi_mode is not None:
170
208
  raise ValueError(
171
209
  "The option to specify the pmi mode is not available on a local workstation, it requires SLURM or flux."
@@ -199,7 +237,22 @@ def validate_number_of_cores(
199
237
  set_local_cores: bool = False,
200
238
  ) -> int:
201
239
  """
202
- Validate the number of cores and return the appropriate value.
240
+ Validate the number of cores and return the number of workers to use.
241
+
242
+ Precedence: max_cores / cores_per_worker > max_workers > CPU count (with warning).
243
+
244
+ Args:
245
+ max_cores (int, optional): Total number of cores available.
246
+ max_workers (int, optional): Explicit number of parallel workers.
247
+ cores_per_worker (int, optional): Number of cores allocated to each worker. Defaults to 1.
248
+ set_local_cores (bool): When True, fall back to the local CPU count if neither
249
+ max_cores nor max_workers is given instead of raising an error. Defaults to False.
250
+
251
+ Returns:
252
+ int: Number of parallel workers to use.
253
+
254
+ Raises:
255
+ ValueError: If neither max_cores nor max_workers is set and set_local_cores is False.
203
256
  """
204
257
  if max_cores is not None and max_workers is None and cores_per_worker is not None:
205
258
  return int(max_cores / cores_per_worker)
@@ -27,10 +27,21 @@ def get_future_objects_from_input(args: tuple, kwargs: dict):
27
27
 
28
28
  find_future_in_list(lst=args)
29
29
  find_future_in_list(lst=kwargs.values())
30
- boolean_flag = len([future for future in future_lst if future.done()]) == len(
31
- future_lst
32
- )
33
- return future_lst, boolean_flag
30
+
31
+ return future_lst
32
+
33
+
34
+ def check_list_of_futures_is_done(future_lst: list[Future]) -> bool:
35
+ """
36
+ Check if all future objects in the list of future objects are done
37
+
38
+ Args:
39
+ future_lst (list): list of future objects
40
+
41
+ Returns:
42
+ bool: True if all future objects in the list of future objects are done, False otherwise
43
+ """
44
+ return len([future for future in future_lst if future.done()]) == len(future_lst)
34
45
 
35
46
 
36
47
  def get_exception_lst(future_lst: list[Future]) -> list:
@@ -240,6 +240,9 @@ def interface_connect(host: str, port: str) -> tuple[zmq.Context, zmq.Socket]:
240
240
  Args:
241
241
  host (str): hostname of the host running the SocketInterface instance to connect to.
242
242
  port (str): port on the host the SocketInterface instance is running on.
243
+
244
+ Returns:
245
+ Tuple[zmq.Context, zmq.Socket]: The ZMQ context and the connected PAIR socket.
243
246
  """
244
247
  context = zmq.Context()
245
248
  socket = context.socket(zmq.PAIR)
@@ -265,6 +268,9 @@ def interface_receive(socket: Optional[zmq.Socket]) -> dict:
265
268
 
266
269
  Args:
267
270
  socket (zmq.Socket): socket for the connection
271
+
272
+ Returns:
273
+ dict: Deserialized dictionary received from the socket, or an empty dict if socket is None.
268
274
  """
269
275
  if socket is not None:
270
276
  return cloudpickle.loads(socket.recv())
@@ -3,21 +3,51 @@ from typing import Any, Optional
3
3
 
4
4
 
5
5
  class FutureSelector(Future):
6
+ """
7
+ A Future wrapper that returns a single element from a result that is a tuple, list, or dict.
8
+
9
+ This is used by split_future() and get_item_from_future() to give callers an individual
10
+ Future-like object for each output of a function that returns a collection.
11
+
12
+ Args:
13
+ future (Future): The underlying future whose result is a collection.
14
+ selector (int | str): Index (for sequences) or key (for mappings) used to extract
15
+ the desired element from the result.
16
+ """
17
+
6
18
  def __init__(self, future: Future, selector: int | str):
19
+ """
20
+ Args:
21
+ future (Future): The underlying future whose result is a collection.
22
+ selector (int | str): Index or key used to extract an element from the result.
23
+ """
7
24
  self._future = future
8
25
  self._selector = selector
9
26
  super().__init__()
10
27
 
11
28
  def __getattr__(self, attr: str) -> Any:
29
+ """Delegate attribute access to the wrapped future."""
12
30
  return getattr(self._future, attr)
13
31
 
14
32
  def __setattr__(self, name: str, value: Any):
33
+ """Set attributes on the wrapped future, except for the two private instance variables."""
15
34
  if name in ["_future", "_selector"]:
16
35
  super().__setattr__(name, value)
17
36
  else:
18
37
  setattr(self._future, name, value)
19
38
 
20
39
  def result(self, timeout: Optional[float] = None) -> Any:
40
+ """
41
+ Return the selected element from the underlying future's result.
42
+
43
+ Args:
44
+ timeout (float, optional): Maximum seconds to wait for the result. Defaults to None
45
+ (wait indefinitely).
46
+
47
+ Returns:
48
+ Any: The element at position/key ``selector`` in the underlying result, or None if
49
+ the underlying result is None.
50
+ """
21
51
  result = self._future.result(timeout=timeout)
22
52
  if result is not None:
23
53
  return result[self._selector]
@@ -102,6 +102,18 @@ def _get_hash(binary: bytes) -> str:
102
102
 
103
103
 
104
104
  def _get_function_name(fn: Callable) -> str:
105
+ """
106
+ Return a human-readable name for a callable.
107
+
108
+ For regular functions and methods the ``__name__`` attribute is used. For callable objects
109
+ (classes implementing ``__call__``) the class name is extracted from ``__class__``.
110
+
111
+ Args:
112
+ fn (Callable): The callable whose name should be determined.
113
+
114
+ Returns:
115
+ str: A short name string suitable for use as part of a cache key prefix.
116
+ """
105
117
  if hasattr(fn, "__name__"):
106
118
  return fn.__name__
107
119
  else:
@@ -14,6 +14,25 @@ except ImportError:
14
14
 
15
15
 
16
16
  class ResourceDictValidation(BaseModel):
17
+ """
18
+ Pydantic (or dataclass fallback) model for validating resource dictionaries passed to task submissions.
19
+
20
+ Attributes:
21
+ cores (int, optional): Number of MPI cores to be used for each function call.
22
+ threads_per_core (int, optional): Number of OpenMP threads to be used for each function call.
23
+ gpus_per_core (int, optional): Number of GPUs per worker.
24
+ cwd (str, optional): Current working directory where the parallel python task is executed.
25
+ cache_key (str, optional): External cache key to identify tasks on the file system.
26
+ cache_directory (str, optional): The directory to store cache files.
27
+ num_nodes (int, optional): Number of compute nodes used for the evaluation.
28
+ exclusive (bool, optional): Reserve exclusive access to selected compute nodes.
29
+ error_log_file (str, optional): Path to the error log file.
30
+ run_time_max (int, optional): Maximum allowed execution time in seconds.
31
+ priority (int, optional): Queuing system priority for the task.
32
+ slurm_cmd_args (list[str], optional): Additional command line arguments for the srun call.
33
+ submission_template (str, optional): Template for queuing system job submission scripts.
34
+ """
35
+
17
36
  cores: Optional[int] = None
18
37
  threads_per_core: Optional[int] = None
19
38
  gpus_per_core: Optional[int] = None
@@ -23,7 +42,7 @@ class ResourceDictValidation(BaseModel):
23
42
  num_nodes: Optional[int] = None
24
43
  exclusive: Optional[bool] = None
25
44
  error_log_file: Optional[str] = None
26
- run_time_limit: Optional[int] = None
45
+ run_time_max: Optional[int] = None
27
46
  priority: Optional[int] = None
28
47
  slurm_cmd_args: Optional[list[str]] = None
29
48
  submission_template: Optional[str] = None
@@ -39,6 +58,18 @@ if not HAS_PYDANTIC:
39
58
 
40
59
 
41
60
  def _get_accepted_keys(class_type) -> list[str]:
61
+ """
62
+ Return a list of accepted field names from a Pydantic model or dataclass.
63
+
64
+ Args:
65
+ class_type: A Pydantic BaseModel subclass or a dataclass.
66
+
67
+ Returns:
68
+ list[str]: Field names declared on the class.
69
+
70
+ Raises:
71
+ TypeError: If the class type is neither a Pydantic model nor a dataclass.
72
+ """
42
73
  if hasattr(class_type, "model_fields"):
43
74
  return list(class_type.model_fields.keys())
44
75
  elif hasattr(class_type, "__dataclass_fields__"):
@@ -47,10 +78,32 @@ def _get_accepted_keys(class_type) -> list[str]:
47
78
 
48
79
 
49
80
  def validate_resource_dict(resource_dict: dict) -> None:
81
+ """
82
+ Validate a resource dictionary against the declared fields of ResourceDictValidation.
83
+
84
+ Args:
85
+ resource_dict (dict): Dictionary of resource requirements to validate. Unknown keys raise
86
+ a validation error when pydantic is available.
87
+
88
+ Raises:
89
+ ValidationError: If any key/value pair violates the ResourceDictValidation schema.
90
+ """
50
91
  _ = ResourceDictValidation(**resource_dict)
51
92
 
52
93
 
53
94
  def validate_resource_dict_with_optional_keys(resource_dict: dict) -> None:
95
+ """
96
+ Validate a resource dictionary, allowing unknown keys with a warning instead of an error.
97
+
98
+ Known keys are validated against ResourceDictValidation. Unknown keys are collected and
99
+ emitted as a UserWarning so callers can detect unsupported options without hard failure.
100
+
101
+ Args:
102
+ resource_dict (dict): Dictionary of resource requirements to validate.
103
+
104
+ Raises:
105
+ ValidationError: If any known key/value pair violates the ResourceDictValidation schema.
106
+ """
54
107
  accepted_keys = _get_accepted_keys(class_type=ResourceDictValidation)
55
108
  optional_lst = [key for key in resource_dict if key not in accepted_keys]
56
109
  validate_dict = {