executorlib 1.0.0__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {executorlib-1.0.0/executorlib.egg-info → executorlib-1.1.0}/PKG-INFO +4 -2
  2. {executorlib-1.0.0 → executorlib-1.1.0}/README.md +2 -0
  3. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/__init__.py +5 -10
  4. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/_version.py +3 -3
  5. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/cache_parallel.py +4 -1
  6. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/cache_serial.py +1 -1
  7. executorlib-1.1.0/executorlib/executor/base.py +118 -0
  8. {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/flux.py +76 -104
  9. {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/single.py +33 -43
  10. {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/executor}/slurm.py +71 -90
  11. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/__init__.py +13 -5
  12. executorlib-1.1.0/executorlib/standalone/cache.py +42 -0
  13. executorlib-1.0.0/executorlib/base/executor.py → executorlib-1.1.0/executorlib/task_scheduler/base.py +2 -1
  14. {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/backend.py +2 -2
  15. {executorlib-1.0.0/executorlib/standalone → executorlib-1.1.0/executorlib/task_scheduler/file}/hdf.py +10 -27
  16. {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/queue_spawner.py +1 -1
  17. {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/shared.py +1 -1
  18. executorlib-1.0.0/executorlib/cache/executor.py → executorlib-1.1.0/executorlib/task_scheduler/file/task_scheduler.py +9 -9
  19. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/blockallocation.py +7 -6
  20. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/dependency.py +7 -5
  21. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/onetoone.py +6 -6
  22. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/shared.py +1 -1
  23. {executorlib-1.0.0 → executorlib-1.1.0/executorlib.egg-info}/PKG-INFO +4 -2
  24. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/SOURCES.txt +22 -20
  25. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/requires.txt +1 -1
  26. {executorlib-1.0.0 → executorlib-1.1.0}/pyproject.toml +1 -1
  27. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_base_executor_queue.py +1 -1
  28. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_backend_execute.py +3 -3
  29. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_fileexecutor_mpi.py +3 -3
  30. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_cache_fileexecutor_serial.py +8 -8
  31. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxjobexecutor.py +1 -1
  32. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxjobexecutor_plot.py +17 -17
  33. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxpythonspawner.py +8 -8
  34. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_interactive_dependencies.py +5 -5
  35. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_interactive_slurmspawner.py +2 -2
  36. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_mpiexecspawner.py +28 -28
  37. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_cache.py +2 -2
  38. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_dependencies.py +2 -2
  39. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_plot_dependency.py +28 -28
  40. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_resize.py +2 -2
  41. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_shell_executor.py +1 -1
  42. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_shell_interactive.py +1 -1
  43. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_hdf.py +1 -1
  44. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_backend.py +1 -1
  45. {executorlib-1.0.0 → executorlib-1.1.0}/LICENSE +0 -0
  46. {executorlib-1.0.0 → executorlib-1.1.0}/MANIFEST.in +0 -0
  47. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/__init__.py +0 -0
  48. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/interactive_parallel.py +0 -0
  49. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/backend/interactive_serial.py +0 -0
  50. {executorlib-1.0.0/executorlib/base → executorlib-1.1.0/executorlib/executor}/__init__.py +0 -0
  51. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/command.py +0 -0
  52. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/inputcheck.py +0 -0
  53. {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/standalone/interactive}/__init__.py +0 -0
  54. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/arguments.py +0 -0
  55. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/backend.py +0 -0
  56. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/communication.py +0 -0
  57. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/interactive/spawner.py +0 -0
  58. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/plot.py +0 -0
  59. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/queue.py +0 -0
  60. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib/standalone/serialize.py +0 -0
  61. {executorlib-1.0.0/executorlib/interactive → executorlib-1.1.0/executorlib/task_scheduler}/__init__.py +0 -0
  62. {executorlib-1.0.0/executorlib/interfaces → executorlib-1.1.0/executorlib/task_scheduler/file}/__init__.py +0 -0
  63. {executorlib-1.0.0/executorlib/cache → executorlib-1.1.0/executorlib/task_scheduler/file}/subprocess_spawner.py +0 -0
  64. {executorlib-1.0.0/executorlib/standalone → executorlib-1.1.0/executorlib/task_scheduler}/interactive/__init__.py +0 -0
  65. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/fluxspawner.py +0 -0
  66. {executorlib-1.0.0/executorlib → executorlib-1.1.0/executorlib/task_scheduler}/interactive/slurmspawner.py +0 -0
  67. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/dependency_links.txt +0 -0
  68. {executorlib-1.0.0 → executorlib-1.1.0}/executorlib.egg-info/top_level.txt +0 -0
  69. {executorlib-1.0.0 → executorlib-1.1.0}/setup.cfg +0 -0
  70. {executorlib-1.0.0 → executorlib-1.1.0}/setup.py +0 -0
  71. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_backend_interactive_serial.py +0 -0
  72. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_fluxclusterexecutor.py +0 -0
  73. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_integration_pyiron_workflow.py +0 -0
  74. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_mpi.py +0 -0
  75. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_singlenodeexecutor_noblock.py +0 -0
  76. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_inputcheck.py +0 -0
  77. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_arguments.py +0 -0
  78. {executorlib-1.0.0 → executorlib-1.1.0}/tests/test_standalone_interactive_communication.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: executorlib
3
- Version: 1.0.0
3
+ Version: 1.1.0
4
4
  Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
5
5
  Author-email: Jan Janssen <janssen@lanl.gov>
6
6
  License: BSD 3-Clause License
@@ -51,7 +51,7 @@ Requires-Python: <3.14,>=3.9
51
51
  Description-Content-Type: text/markdown
52
52
  License-File: LICENSE
53
53
  Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
54
- Requires-Dist: pyzmq<=26.3.0,>=25.0.0
54
+ Requires-Dist: pyzmq<=26.4.0,>=25.0.0
55
55
  Provides-Extra: cache
56
56
  Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cache"
57
57
  Provides-Extra: graph
@@ -232,6 +232,8 @@ as hierarchical job scheduler within the allocations.
232
232
  * [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
233
233
  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
234
234
  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
235
+ * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
235
236
  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
237
+ * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
236
238
  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
237
239
  * [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
@@ -156,6 +156,8 @@ as hierarchical job scheduler within the allocations.
156
156
  * [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
157
157
  * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
158
158
  * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
159
+ * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy)
159
160
  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
161
+ * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries)
160
162
  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
161
163
  * [Interface](https://executorlib.readthedocs.io/en/latest/api.html)
@@ -1,15 +1,17 @@
1
1
  from executorlib._version import get_versions as _get_versions
2
- from executorlib.interfaces.flux import (
2
+ from executorlib.executor.flux import (
3
3
  FluxClusterExecutor,
4
4
  FluxJobExecutor,
5
5
  )
6
- from executorlib.interfaces.single import SingleNodeExecutor
7
- from executorlib.interfaces.slurm import (
6
+ from executorlib.executor.single import SingleNodeExecutor
7
+ from executorlib.executor.slurm import (
8
8
  SlurmClusterExecutor,
9
9
  SlurmJobExecutor,
10
10
  )
11
+ from executorlib.standalone.cache import get_cache_data
11
12
 
12
13
  __all__: list[str] = [
14
+ "get_cache_data",
13
15
  "FluxJobExecutor",
14
16
  "FluxClusterExecutor",
15
17
  "SingleNodeExecutor",
@@ -17,11 +19,4 @@ __all__: list[str] = [
17
19
  "SlurmClusterExecutor",
18
20
  ]
19
21
 
20
- try:
21
- from executorlib.standalone.hdf import get_cache_data
22
- except ImportError:
23
- pass
24
- else:
25
- __all__ += ["get_cache_data"]
26
-
27
22
  __version__ = _get_versions()["version"]
@@ -8,11 +8,11 @@ import json
8
8
 
9
9
  version_json = '''
10
10
  {
11
- "date": "2025-03-29T11:38:43+0100",
11
+ "date": "2025-04-26T14:56:26+0200",
12
12
  "dirty": true,
13
13
  "error": null,
14
- "full-revisionid": "5de07acc29643a232dbc50303435a399df53cf3b",
15
- "version": "1.0.0"
14
+ "full-revisionid": "093d1fd613b10c43f4ebc4387d2b1c3dbfebf0c0",
15
+ "version": "1.1.0"
16
16
  }
17
17
  ''' # END VERSION_JSON
18
18
 
@@ -4,7 +4,10 @@ import time
4
4
 
5
5
  import cloudpickle
6
6
 
7
- from executorlib.cache.backend import backend_load_file, backend_write_file
7
+ from executorlib.task_scheduler.file.backend import (
8
+ backend_load_file,
9
+ backend_write_file,
10
+ )
8
11
 
9
12
 
10
13
  def main() -> None:
@@ -1,6 +1,6 @@
1
1
  import sys
2
2
 
3
- from executorlib.cache.backend import backend_execute_task_in_file
3
+ from executorlib.task_scheduler.file.backend import backend_execute_task_in_file
4
4
 
5
5
  if __name__ == "__main__":
6
6
  backend_execute_task_in_file(file_name=sys.argv[1])
@@ -0,0 +1,118 @@
1
+ import queue
2
+ from concurrent.futures import (
3
+ Executor as FutureExecutor,
4
+ )
5
+ from concurrent.futures import (
6
+ Future,
7
+ )
8
+ from typing import Callable, Optional
9
+
10
+ from executorlib.task_scheduler.base import TaskSchedulerBase
11
+
12
+
13
+ class ExecutorBase(FutureExecutor):
14
+ """
15
+ Interface class for the executor.
16
+
17
+ Args:
18
+ executor (TaskSchedulerBase): internal executor
19
+ """
20
+
21
+ def __init__(self, executor: TaskSchedulerBase):
22
+ self._task_scheduler = executor
23
+
24
+ @property
25
+ def max_workers(self) -> Optional[int]:
26
+ return self._task_scheduler.max_workers
27
+
28
+ @max_workers.setter
29
+ def max_workers(self, max_workers: int):
30
+ self._task_scheduler.max_workers = max_workers
31
+
32
+ @property
33
+ def info(self) -> Optional[dict]:
34
+ """
35
+ Get the information about the executor.
36
+
37
+ Returns:
38
+ Optional[dict]: Information about the executor.
39
+ """
40
+ return self._task_scheduler.info
41
+
42
+ @property
43
+ def future_queue(self) -> Optional[queue.Queue]:
44
+ """
45
+ Get the future queue.
46
+
47
+ Returns:
48
+ queue.Queue: The future queue.
49
+ """
50
+ return self._task_scheduler.future_queue
51
+
52
+ def submit( # type: ignore
53
+ self,
54
+ fn: Callable,
55
+ /,
56
+ *args,
57
+ resource_dict: Optional[dict] = None,
58
+ **kwargs,
59
+ ) -> Future:
60
+ """
61
+ Submits a callable to be executed with the given arguments.
62
+
63
+ Schedules the callable to be executed as fn(*args, **kwargs) and returns
64
+ a Future instance representing the execution of the callable.
65
+
66
+ Args:
67
+ fn (callable): function to submit for execution
68
+ args: arguments for the submitted function
69
+ kwargs: keyword arguments for the submitted function
70
+ resource_dict (dict): resource dictionary, which defines the resources used for the execution of the
71
+ function. Example resource dictionary: {
72
+ cores: 1,
73
+ threads_per_core: 1,
74
+ gpus_per_worker: 0,
75
+ oversubscribe: False,
76
+ cwd: None,
77
+ executor: None,
78
+ hostname_localhost: False,
79
+ }
80
+
81
+ Returns:
82
+ Future: A Future representing the given call.
83
+ """
84
+ return self._task_scheduler.submit(
85
+ *([fn] + list(args)), resource_dict=resource_dict, **kwargs
86
+ )
87
+
88
+ def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
89
+ """
90
+ Clean-up the resources associated with the Executor.
91
+
92
+ It is safe to call this method several times. Otherwise, no other
93
+ methods can be called after this one.
94
+
95
+ Args:
96
+ wait (bool): If True then shutdown will not return until all running
97
+ futures have finished executing and the resources used by the
98
+ parallel_executors have been reclaimed.
99
+ cancel_futures (bool): If True then shutdown will cancel all pending
100
+ futures. Futures that are completed or running will not be
101
+ cancelled.
102
+ """
103
+ self._task_scheduler.shutdown(wait=wait, cancel_futures=cancel_futures)
104
+
105
+ def __len__(self) -> int:
106
+ """
107
+ Get the length of the executor.
108
+
109
+ Returns:
110
+ int: The length of the executor.
111
+ """
112
+ return len(self._task_scheduler)
113
+
114
+ def __exit__(self, *args, **kwargs) -> None:
115
+ """
116
+ Exit method called when exiting the context manager.
117
+ """
118
+ self._task_scheduler.__exit__(*args, **kwargs)
@@ -1,9 +1,6 @@
1
- import contextlib
2
1
  from typing import Callable, Optional, Union
3
2
 
4
- from executorlib.interactive.blockallocation import BlockAllocationExecutor
5
- from executorlib.interactive.dependency import DependencyExecutor
6
- from executorlib.interactive.onetoone import OneTaskPerProcessExecutor
3
+ from executorlib.executor.base import ExecutorBase
7
4
  from executorlib.standalone.inputcheck import (
8
5
  check_command_line_argument_lst,
9
6
  check_init_function,
@@ -13,15 +10,14 @@ from executorlib.standalone.inputcheck import (
13
10
  check_refresh_rate,
14
11
  validate_number_of_cores,
15
12
  )
16
-
17
- with contextlib.suppress(ImportError):
18
- from executorlib.interactive.fluxspawner import (
19
- FluxPythonSpawner,
20
- validate_max_workers,
21
- )
13
+ from executorlib.task_scheduler.interactive.blockallocation import (
14
+ BlockAllocationTaskScheduler,
15
+ )
16
+ from executorlib.task_scheduler.interactive.dependency import DependencyTaskScheduler
17
+ from executorlib.task_scheduler.interactive.onetoone import OneProcessTaskScheduler
22
18
 
23
19
 
24
- class FluxJobExecutor:
20
+ class FluxJobExecutor(ExecutorBase):
25
21
  """
26
22
  The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
27
23
  preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
@@ -69,7 +65,7 @@ class FluxJobExecutor:
69
65
  Examples:
70
66
  ```
71
67
  >>> import numpy as np
72
- >>> from executorlib.interfaces.flux import FluxJobExecutor
68
+ >>> from executorlib.executor.flux import FluxJobExecutor
73
69
  >>>
74
70
  >>> def calc(i, j, k):
75
71
  >>> from mpi4py import MPI
@@ -104,27 +100,6 @@ class FluxJobExecutor:
104
100
  refresh_rate: float = 0.01,
105
101
  plot_dependency_graph: bool = False,
106
102
  plot_dependency_graph_filename: Optional[str] = None,
107
- ):
108
- # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
109
- pass
110
-
111
- def __new__(
112
- cls,
113
- max_workers: Optional[int] = None,
114
- cache_directory: Optional[str] = None,
115
- max_cores: Optional[int] = None,
116
- resource_dict: Optional[dict] = None,
117
- flux_executor=None,
118
- flux_executor_pmi_mode: Optional[str] = None,
119
- flux_executor_nesting: bool = False,
120
- flux_log_files: bool = False,
121
- hostname_localhost: Optional[bool] = None,
122
- block_allocation: bool = False,
123
- init_function: Optional[Callable] = None,
124
- disable_dependencies: bool = False,
125
- refresh_rate: float = 0.01,
126
- plot_dependency_graph: bool = False,
127
- plot_dependency_graph_filename: Optional[str] = None,
128
103
  ):
129
104
  """
130
105
  Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
@@ -186,7 +161,31 @@ class FluxJobExecutor:
186
161
  {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
187
162
  )
188
163
  if not disable_dependencies:
189
- return DependencyExecutor(
164
+ super().__init__(
165
+ executor=DependencyTaskScheduler(
166
+ executor=create_flux_executor(
167
+ max_workers=max_workers,
168
+ cache_directory=cache_directory,
169
+ max_cores=max_cores,
170
+ resource_dict=resource_dict,
171
+ flux_executor=flux_executor,
172
+ flux_executor_pmi_mode=flux_executor_pmi_mode,
173
+ flux_executor_nesting=flux_executor_nesting,
174
+ flux_log_files=flux_log_files,
175
+ hostname_localhost=hostname_localhost,
176
+ block_allocation=block_allocation,
177
+ init_function=init_function,
178
+ ),
179
+ max_cores=max_cores,
180
+ refresh_rate=refresh_rate,
181
+ plot_dependency_graph=plot_dependency_graph,
182
+ plot_dependency_graph_filename=plot_dependency_graph_filename,
183
+ )
184
+ )
185
+ else:
186
+ check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
187
+ check_refresh_rate(refresh_rate=refresh_rate)
188
+ super().__init__(
190
189
  executor=create_flux_executor(
191
190
  max_workers=max_workers,
192
191
  cache_directory=cache_directory,
@@ -199,31 +198,11 @@ class FluxJobExecutor:
199
198
  hostname_localhost=hostname_localhost,
200
199
  block_allocation=block_allocation,
201
200
  init_function=init_function,
202
- ),
203
- max_cores=max_cores,
204
- refresh_rate=refresh_rate,
205
- plot_dependency_graph=plot_dependency_graph,
206
- plot_dependency_graph_filename=plot_dependency_graph_filename,
207
- )
208
- else:
209
- check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
210
- check_refresh_rate(refresh_rate=refresh_rate)
211
- return create_flux_executor(
212
- max_workers=max_workers,
213
- cache_directory=cache_directory,
214
- max_cores=max_cores,
215
- resource_dict=resource_dict,
216
- flux_executor=flux_executor,
217
- flux_executor_pmi_mode=flux_executor_pmi_mode,
218
- flux_executor_nesting=flux_executor_nesting,
219
- flux_log_files=flux_log_files,
220
- hostname_localhost=hostname_localhost,
221
- block_allocation=block_allocation,
222
- init_function=init_function,
201
+ )
223
202
  )
224
203
 
225
204
 
226
- class FluxClusterExecutor:
205
+ class FluxClusterExecutor(ExecutorBase):
227
206
  """
228
207
  The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
229
208
  preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
@@ -267,7 +246,7 @@ class FluxClusterExecutor:
267
246
  Examples:
268
247
  ```
269
248
  >>> import numpy as np
270
- >>> from executorlib.interfaces.flux import FluxClusterExecutor
249
+ >>> from executorlib.executor.flux import FluxClusterExecutor
271
250
  >>>
272
251
  >>> def calc(i, j, k):
273
252
  >>> from mpi4py import MPI
@@ -299,24 +278,6 @@ class FluxClusterExecutor:
299
278
  refresh_rate: float = 0.01,
300
279
  plot_dependency_graph: bool = False,
301
280
  plot_dependency_graph_filename: Optional[str] = None,
302
- ):
303
- # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
304
- pass
305
-
306
- def __new__(
307
- cls,
308
- max_workers: Optional[int] = None,
309
- cache_directory: Optional[str] = None,
310
- max_cores: Optional[int] = None,
311
- resource_dict: Optional[dict] = None,
312
- pysqa_config_directory: Optional[str] = None,
313
- hostname_localhost: Optional[bool] = None,
314
- block_allocation: bool = False,
315
- init_function: Optional[Callable] = None,
316
- disable_dependencies: bool = False,
317
- refresh_rate: float = 0.01,
318
- plot_dependency_graph: bool = False,
319
- plot_dependency_graph_filename: Optional[str] = None,
320
281
  ):
321
282
  """
322
283
  Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
@@ -375,43 +336,49 @@ class FluxClusterExecutor:
375
336
  {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
376
337
  )
377
338
  if not plot_dependency_graph:
378
- from executorlib.cache.executor import create_file_executor
379
-
380
- return create_file_executor(
381
- max_workers=max_workers,
382
- backend="flux_submission",
383
- max_cores=max_cores,
384
- cache_directory=cache_directory,
385
- resource_dict=resource_dict,
386
- flux_executor=None,
387
- flux_executor_pmi_mode=None,
388
- flux_executor_nesting=False,
389
- flux_log_files=False,
390
- pysqa_config_directory=pysqa_config_directory,
391
- hostname_localhost=hostname_localhost,
392
- block_allocation=block_allocation,
393
- init_function=init_function,
394
- disable_dependencies=disable_dependencies,
339
+ from executorlib.task_scheduler.file.task_scheduler import (
340
+ create_file_executor,
395
341
  )
396
- else:
397
- return DependencyExecutor(
398
- executor=create_flux_executor(
342
+
343
+ super().__init__(
344
+ executor=create_file_executor(
399
345
  max_workers=max_workers,
400
- cache_directory=cache_directory,
346
+ backend="flux_submission",
401
347
  max_cores=max_cores,
348
+ cache_directory=cache_directory,
402
349
  resource_dict=resource_dict,
403
350
  flux_executor=None,
404
351
  flux_executor_pmi_mode=None,
405
352
  flux_executor_nesting=False,
406
353
  flux_log_files=False,
354
+ pysqa_config_directory=pysqa_config_directory,
407
355
  hostname_localhost=hostname_localhost,
408
356
  block_allocation=block_allocation,
409
357
  init_function=init_function,
410
- ),
411
- max_cores=max_cores,
412
- refresh_rate=refresh_rate,
413
- plot_dependency_graph=plot_dependency_graph,
414
- plot_dependency_graph_filename=plot_dependency_graph_filename,
358
+ disable_dependencies=disable_dependencies,
359
+ )
360
+ )
361
+ else:
362
+ super().__init__(
363
+ executor=DependencyTaskScheduler(
364
+ executor=create_flux_executor(
365
+ max_workers=max_workers,
366
+ cache_directory=cache_directory,
367
+ max_cores=max_cores,
368
+ resource_dict=resource_dict,
369
+ flux_executor=None,
370
+ flux_executor_pmi_mode=None,
371
+ flux_executor_nesting=False,
372
+ flux_log_files=False,
373
+ hostname_localhost=hostname_localhost,
374
+ block_allocation=block_allocation,
375
+ init_function=init_function,
376
+ ),
377
+ max_cores=max_cores,
378
+ refresh_rate=refresh_rate,
379
+ plot_dependency_graph=plot_dependency_graph,
380
+ plot_dependency_graph_filename=plot_dependency_graph_filename,
381
+ )
415
382
  )
416
383
 
417
384
 
@@ -427,7 +394,7 @@ def create_flux_executor(
427
394
  hostname_localhost: Optional[bool] = None,
428
395
  block_allocation: bool = False,
429
396
  init_function: Optional[Callable] = None,
430
- ) -> Union[OneTaskPerProcessExecutor, BlockAllocationExecutor]:
397
+ ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
431
398
  """
432
399
  Create a flux executor
433
400
 
@@ -466,6 +433,11 @@ def create_flux_executor(
466
433
  Returns:
467
434
  InteractiveStepExecutor/ InteractiveExecutor
468
435
  """
436
+ from executorlib.task_scheduler.interactive.fluxspawner import (
437
+ FluxPythonSpawner,
438
+ validate_max_workers,
439
+ )
440
+
469
441
  if resource_dict is None:
470
442
  resource_dict = {}
471
443
  cores_per_worker = resource_dict.get("cores", 1)
@@ -498,13 +470,13 @@ def create_flux_executor(
498
470
  cores=cores_per_worker,
499
471
  threads_per_core=resource_dict.get("threads_per_core", 1),
500
472
  )
501
- return BlockAllocationExecutor(
473
+ return BlockAllocationTaskScheduler(
502
474
  max_workers=max_workers,
503
475
  executor_kwargs=resource_dict,
504
476
  spawner=FluxPythonSpawner,
505
477
  )
506
478
  else:
507
- return OneTaskPerProcessExecutor(
479
+ return OneProcessTaskScheduler(
508
480
  max_cores=max_cores,
509
481
  max_workers=max_workers,
510
482
  executor_kwargs=resource_dict,
@@ -1,8 +1,6 @@
1
1
  from typing import Callable, Optional, Union
2
2
 
3
- from executorlib.interactive.blockallocation import BlockAllocationExecutor
4
- from executorlib.interactive.dependency import DependencyExecutor
5
- from executorlib.interactive.onetoone import OneTaskPerProcessExecutor
3
+ from executorlib.executor.base import ExecutorBase
6
4
  from executorlib.standalone.inputcheck import (
7
5
  check_command_line_argument_lst,
8
6
  check_gpus_per_worker,
@@ -12,9 +10,14 @@ from executorlib.standalone.inputcheck import (
12
10
  validate_number_of_cores,
13
11
  )
14
12
  from executorlib.standalone.interactive.spawner import MpiExecSpawner
13
+ from executorlib.task_scheduler.interactive.blockallocation import (
14
+ BlockAllocationTaskScheduler,
15
+ )
16
+ from executorlib.task_scheduler.interactive.dependency import DependencyTaskScheduler
17
+ from executorlib.task_scheduler.interactive.onetoone import OneProcessTaskScheduler
15
18
 
16
19
 
17
- class SingleNodeExecutor:
20
+ class SingleNodeExecutor(ExecutorBase):
18
21
  """
19
22
  The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or
20
23
  preferable the flux framework for distributing python functions within a given resource allocation. In contrast to
@@ -57,7 +60,7 @@ class SingleNodeExecutor:
57
60
  Examples:
58
61
  ```
59
62
  >>> import numpy as np
60
- >>> from executorlib.interfaces.single import SingleNodeExecutor
63
+ >>> from executorlib.executor.single import SingleNodeExecutor
61
64
  >>>
62
65
  >>> def calc(i, j, k):
63
66
  >>> from mpi4py import MPI
@@ -88,23 +91,6 @@ class SingleNodeExecutor:
88
91
  refresh_rate: float = 0.01,
89
92
  plot_dependency_graph: bool = False,
90
93
  plot_dependency_graph_filename: Optional[str] = None,
91
- ):
92
- # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion.
93
- pass
94
-
95
- def __new__(
96
- cls,
97
- max_workers: Optional[int] = None,
98
- cache_directory: Optional[str] = None,
99
- max_cores: Optional[int] = None,
100
- resource_dict: Optional[dict] = None,
101
- hostname_localhost: Optional[bool] = None,
102
- block_allocation: bool = False,
103
- init_function: Optional[Callable] = None,
104
- disable_dependencies: bool = False,
105
- refresh_rate: float = 0.01,
106
- plot_dependency_graph: bool = False,
107
- plot_dependency_graph_filename: Optional[str] = None,
108
94
  ):
109
95
  """
110
96
  Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor,
@@ -162,7 +148,27 @@ class SingleNodeExecutor:
162
148
  {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
163
149
  )
164
150
  if not disable_dependencies:
165
- return DependencyExecutor(
151
+ super().__init__(
152
+ executor=DependencyTaskScheduler(
153
+ executor=create_single_node_executor(
154
+ max_workers=max_workers,
155
+ cache_directory=cache_directory,
156
+ max_cores=max_cores,
157
+ resource_dict=resource_dict,
158
+ hostname_localhost=hostname_localhost,
159
+ block_allocation=block_allocation,
160
+ init_function=init_function,
161
+ ),
162
+ max_cores=max_cores,
163
+ refresh_rate=refresh_rate,
164
+ plot_dependency_graph=plot_dependency_graph,
165
+ plot_dependency_graph_filename=plot_dependency_graph_filename,
166
+ )
167
+ )
168
+ else:
169
+ check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
170
+ check_refresh_rate(refresh_rate=refresh_rate)
171
+ super().__init__(
166
172
  executor=create_single_node_executor(
167
173
  max_workers=max_workers,
168
174
  cache_directory=cache_directory,
@@ -171,23 +177,7 @@ class SingleNodeExecutor:
171
177
  hostname_localhost=hostname_localhost,
172
178
  block_allocation=block_allocation,
173
179
  init_function=init_function,
174
- ),
175
- max_cores=max_cores,
176
- refresh_rate=refresh_rate,
177
- plot_dependency_graph=plot_dependency_graph,
178
- plot_dependency_graph_filename=plot_dependency_graph_filename,
179
- )
180
- else:
181
- check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph)
182
- check_refresh_rate(refresh_rate=refresh_rate)
183
- return create_single_node_executor(
184
- max_workers=max_workers,
185
- cache_directory=cache_directory,
186
- max_cores=max_cores,
187
- resource_dict=resource_dict,
188
- hostname_localhost=hostname_localhost,
189
- block_allocation=block_allocation,
190
- init_function=init_function,
180
+ )
191
181
  )
192
182
 
193
183
 
@@ -199,7 +189,7 @@ def create_single_node_executor(
199
189
  hostname_localhost: Optional[bool] = None,
200
190
  block_allocation: bool = False,
201
191
  init_function: Optional[Callable] = None,
202
- ) -> Union[OneTaskPerProcessExecutor, BlockAllocationExecutor]:
192
+ ) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]:
203
193
  """
204
194
  Create a single node executor
205
195
 
@@ -253,7 +243,7 @@ def create_single_node_executor(
253
243
  del resource_dict["slurm_cmd_args"]
254
244
  if block_allocation:
255
245
  resource_dict["init_function"] = init_function
256
- return BlockAllocationExecutor(
246
+ return BlockAllocationTaskScheduler(
257
247
  max_workers=validate_number_of_cores(
258
248
  max_cores=max_cores,
259
249
  max_workers=max_workers,
@@ -264,7 +254,7 @@ def create_single_node_executor(
264
254
  spawner=MpiExecSpawner,
265
255
  )
266
256
  else:
267
- return OneTaskPerProcessExecutor(
257
+ return OneProcessTaskScheduler(
268
258
  max_cores=max_cores,
269
259
  max_workers=max_workers,
270
260
  executor_kwargs=resource_dict,