cluster-tools 2.0.7__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/PKG-INFO +2 -2
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/__init__.py +1 -1
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/call.py +3 -8
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/file_wait_thread.py +6 -11
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/multiprocessing_logging_handler.py +3 -2
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/pickling.py +7 -7
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/reflection.py +1 -1
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/tailf.py +5 -4
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/warning.py +5 -10
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executor_protocol.py +8 -12
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/dask.py +17 -22
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/multiprocessing_.py +16 -21
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/multiprocessing_pickle.py +2 -1
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/sequential.py +5 -5
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/executors/sequential_pickle.py +2 -1
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/remote.py +11 -17
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/cluster_executor.py +36 -46
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/kube.py +15 -17
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/pbs.py +22 -28
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/slurm.py +39 -53
- cluster_tools-2.2.0/cluster_tools/version.py +1 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/PKG-INFO +2 -2
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/pyproject.toml +9 -8
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_all.py +20 -32
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_deref_main.py +1 -3
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_kubernetes.py +1 -2
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_multiprocessing.py +12 -12
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_slurm.py +25 -25
- cluster_tools-2.0.7/cluster_tools/version.py +0 -1
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/README.md +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/string_.py +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/schedulers/__init__.py +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/SOURCES.txt +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/dependency_links.txt +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/requires.txt +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools.egg-info/top_level.txt +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/setup.cfg +0 -0
- {cluster_tools-2.0.7 → cluster_tools-2.2.0}/tests/test_dask.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
Project-URL: Repository, https://github.com/scalableminds/webknossos-libs
|
|
8
8
|
Project-URL: Changelog, https://github.com/scalableminds/webknossos-libs/blob/master/cluster_tools/Changelog.md
|
|
9
|
-
Requires-Python: >=3.
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
Requires-Dist: typing-extensions~=4.12.0
|
|
12
12
|
Provides-Extra: kubernetes
|
|
@@ -117,4 +117,4 @@ def get_executor(environment: str, **kwargs: Any) -> "Executor":
|
|
|
117
117
|
return SequentialPickleExecutor(**kwargs)
|
|
118
118
|
elif environment == "multiprocessing_with_pickling":
|
|
119
119
|
return MultiprocessingPickleExecutor(**kwargs)
|
|
120
|
-
raise Exception("Unknown executor: {}"
|
|
120
|
+
raise Exception(f"Unknown executor: {environment}")
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import subprocess
|
|
2
|
-
from typing import Optional, Tuple
|
|
3
2
|
|
|
4
3
|
|
|
5
|
-
def call(command: str, stdin:
|
|
4
|
+
def call(command: str, stdin: str | None = None) -> tuple[str, str, int]:
|
|
6
5
|
"""Invokes a shell command as a subprocess, optionally with some
|
|
7
6
|
data sent to the standard input. Returns the standard output data,
|
|
8
7
|
the standard error, and the return code.
|
|
@@ -31,14 +30,10 @@ class CommandError(Exception):
|
|
|
31
30
|
self.stderr = stderr
|
|
32
31
|
|
|
33
32
|
def __str__(self) -> str:
|
|
34
|
-
return "
|
|
35
|
-
repr(self.command),
|
|
36
|
-
self.code,
|
|
37
|
-
repr(self.stderr),
|
|
38
|
-
)
|
|
33
|
+
return f"{self.command!r} exited with status {self.code}: {self.stderr!r}"
|
|
39
34
|
|
|
40
35
|
|
|
41
|
-
def chcall(command: str, stdin:
|
|
36
|
+
def chcall(command: str, stdin: str | None = None) -> tuple[str, str]:
|
|
42
37
|
"""Like ``call`` but raises an exception when the return code is
|
|
43
38
|
nonzero. Only returns the stdout and stderr data.
|
|
44
39
|
"""
|
|
@@ -2,7 +2,8 @@ import logging
|
|
|
2
2
|
import os
|
|
3
3
|
import threading
|
|
4
4
|
import time
|
|
5
|
-
from
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
6
7
|
|
|
7
8
|
if TYPE_CHECKING:
|
|
8
9
|
from cluster_tools.schedulers.cluster_executor import ClusterExecutor
|
|
@@ -28,8 +29,8 @@ class FileWaitThread(threading.Thread):
|
|
|
28
29
|
threading.Thread.__init__(self)
|
|
29
30
|
self.callback = callback
|
|
30
31
|
self.interval = interval
|
|
31
|
-
self.waiting:
|
|
32
|
-
self.retryMap:
|
|
32
|
+
self.waiting: dict[str, str] = {}
|
|
33
|
+
self.retryMap: dict[str, int] = {}
|
|
33
34
|
self.lock = threading.Lock()
|
|
34
35
|
self.shutdown = False
|
|
35
36
|
self.executor = executor
|
|
@@ -85,17 +86,11 @@ class FileWaitThread(threading.Thread):
|
|
|
85
86
|
if self.retryMap[filename] <= FileWaitThread.MAX_RETRY:
|
|
86
87
|
# Retry by looping again
|
|
87
88
|
logging.warning(
|
|
88
|
-
"Job state is completed, but {} couldn't be found. Retrying {}/{}"
|
|
89
|
-
filename,
|
|
90
|
-
self.retryMap[filename],
|
|
91
|
-
FileWaitThread.MAX_RETRY,
|
|
92
|
-
)
|
|
89
|
+
f"Job state is completed, but {filename} couldn't be found. Retrying {self.retryMap[filename]}/{FileWaitThread.MAX_RETRY}"
|
|
93
90
|
)
|
|
94
91
|
else:
|
|
95
92
|
logging.error(
|
|
96
|
-
"Job state is completed, but {} couldn't be found."
|
|
97
|
-
filename
|
|
98
|
-
)
|
|
93
|
+
f"Job state is completed, but {filename} couldn't be found."
|
|
99
94
|
)
|
|
100
95
|
handle_completed_job(job_id, filename, True)
|
|
101
96
|
|
{cluster_tools-2.0.7 → cluster_tools-2.2.0}/cluster_tools/_utils/multiprocessing_logging_handler.py
RENAMED
|
@@ -6,11 +6,12 @@ import sys
|
|
|
6
6
|
import threading
|
|
7
7
|
import traceback
|
|
8
8
|
import warnings
|
|
9
|
+
from collections.abc import Callable, Sequence
|
|
9
10
|
from logging import getLogger
|
|
10
11
|
from logging.handlers import QueueHandler
|
|
11
12
|
from queue import Empty as QueueEmpty
|
|
12
13
|
from queue import Queue
|
|
13
|
-
from typing import Any
|
|
14
|
+
from typing import Any
|
|
14
15
|
|
|
15
16
|
# Inspired by https://stackoverflow.com/a/894284
|
|
16
17
|
|
|
@@ -106,7 +107,7 @@ class _MultiprocessingLoggingHandler(logging.Handler):
|
|
|
106
107
|
|
|
107
108
|
|
|
108
109
|
def _setup_logging_multiprocessing(
|
|
109
|
-
queues:
|
|
110
|
+
queues: list[Queue], levels: list[int], filters: Sequence[Any]
|
|
110
111
|
) -> None:
|
|
111
112
|
"""Re-setup logging in a multiprocessing context (only needed if a start_method other than
|
|
112
113
|
fork is used) by setting up QueueHandler loggers for each queue and level
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import pickle
|
|
2
2
|
import sys
|
|
3
|
-
from typing import Any, BinaryIO
|
|
3
|
+
from typing import Any, BinaryIO
|
|
4
4
|
|
|
5
5
|
from cluster_tools._utils.warning import warn_after
|
|
6
6
|
|
|
@@ -26,25 +26,25 @@ def dump(*args: Any, **kwargs: Any) -> None:
|
|
|
26
26
|
|
|
27
27
|
@warn_after("pickle.loads", WARNING_TIMEOUT)
|
|
28
28
|
def loads(*args: Any, **kwargs: Any) -> Any:
|
|
29
|
-
assert (
|
|
30
|
-
"
|
|
31
|
-
)
|
|
29
|
+
assert "custom_main_path" not in kwargs, (
|
|
30
|
+
"loads does not implement support for the argument custom_main_path"
|
|
31
|
+
)
|
|
32
32
|
return pickle.loads(*args, **kwargs)
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class _RenameUnpickler(pickle.Unpickler):
|
|
36
|
-
custom_main_path:
|
|
36
|
+
custom_main_path: str | None
|
|
37
37
|
|
|
38
38
|
def find_class(self, module: str, name: str) -> Any:
|
|
39
39
|
renamed_module = module
|
|
40
40
|
if module == "__main__" and self.custom_main_path is not None:
|
|
41
41
|
renamed_module = self.custom_main_path
|
|
42
42
|
|
|
43
|
-
return super(
|
|
43
|
+
return super().find_class(renamed_module, name)
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
@warn_after("pickle.load", WARNING_TIMEOUT)
|
|
47
|
-
def load(f: BinaryIO, custom_main_path:
|
|
47
|
+
def load(f: BinaryIO, custom_main_path: str | None = None) -> Any:
|
|
48
48
|
unpickler = _RenameUnpickler(f)
|
|
49
49
|
unpickler.custom_main_path = custom_main_path
|
|
50
50
|
return unpickler.load()
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
import os
|
|
6
6
|
import sys
|
|
7
7
|
import time
|
|
8
|
-
from
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from typing import Any
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
class Tail:
|
|
@@ -58,11 +59,11 @@ class Tail:
|
|
|
58
59
|
def check_file_validity(self, file_: str) -> None:
|
|
59
60
|
"""Check whether the a given file exists, readable and is a file"""
|
|
60
61
|
if not os.access(file_, os.F_OK):
|
|
61
|
-
raise TailError("File '
|
|
62
|
+
raise TailError(f"File '{file_}' does not exist")
|
|
62
63
|
if not os.access(file_, os.R_OK):
|
|
63
|
-
raise TailError("File '
|
|
64
|
+
raise TailError(f"File '{file_}' not readable")
|
|
64
65
|
if os.path.isdir(file_):
|
|
65
|
-
raise TailError("File '
|
|
66
|
+
raise TailError(f"File '{file_}' is a directory")
|
|
66
67
|
|
|
67
68
|
|
|
68
69
|
class TailError(Exception):
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import threading
|
|
3
3
|
import time
|
|
4
|
+
from collections.abc import Callable
|
|
4
5
|
from concurrent.futures import Future
|
|
5
|
-
from typing import
|
|
6
|
+
from typing import TypeVar
|
|
6
7
|
|
|
7
8
|
from typing_extensions import ParamSpec
|
|
8
9
|
|
|
@@ -24,9 +25,7 @@ def warn_after(
|
|
|
24
25
|
|
|
25
26
|
def warn_function() -> None:
|
|
26
27
|
logging.warning(
|
|
27
|
-
"Function {} is taking suspiciously long (longer than {} seconds)"
|
|
28
|
-
job, seconds
|
|
29
|
-
)
|
|
28
|
+
f"Function {job} is taking suspiciously long (longer than {seconds} seconds)"
|
|
30
29
|
)
|
|
31
30
|
exceeded_timeout[0] = True
|
|
32
31
|
|
|
@@ -38,9 +37,7 @@ def warn_after(
|
|
|
38
37
|
if exceeded_timeout[0]:
|
|
39
38
|
end_time = time.time()
|
|
40
39
|
logging.warning(
|
|
41
|
-
"Function {} succeeded after all (took {} seconds)"
|
|
42
|
-
job, int(end_time - start_time)
|
|
43
|
-
)
|
|
40
|
+
f"Function {job} succeeded after all (took {int(end_time - start_time)} seconds)"
|
|
44
41
|
)
|
|
45
42
|
finally:
|
|
46
43
|
timer.cancel()
|
|
@@ -56,9 +53,7 @@ def enrich_future_with_uncaught_warning(f: Future) -> None:
|
|
|
56
53
|
maybe_exception = future.exception()
|
|
57
54
|
if maybe_exception is not None:
|
|
58
55
|
logging.error(
|
|
59
|
-
"A future crashed with an exception: {}. Future: {}"
|
|
60
|
-
maybe_exception, future
|
|
61
|
-
)
|
|
56
|
+
f"A future crashed with an exception: {maybe_exception}. Future: {future}"
|
|
62
57
|
)
|
|
63
58
|
|
|
64
59
|
if not hasattr(f, "is_wrapped_by_cluster_tools"):
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
1
2
|
from concurrent.futures import Future
|
|
3
|
+
from contextlib import AbstractContextManager
|
|
2
4
|
from os import PathLike
|
|
3
5
|
from typing import (
|
|
4
|
-
Callable,
|
|
5
|
-
ContextManager,
|
|
6
|
-
Iterable,
|
|
7
|
-
Iterator,
|
|
8
|
-
List,
|
|
9
|
-
Optional,
|
|
10
6
|
Protocol,
|
|
11
7
|
TypeVar,
|
|
12
8
|
)
|
|
@@ -18,9 +14,9 @@ _P = ParamSpec("_P")
|
|
|
18
14
|
_S = TypeVar("_S")
|
|
19
15
|
|
|
20
16
|
|
|
21
|
-
class Executor(Protocol,
|
|
17
|
+
class Executor(Protocol, AbstractContextManager["Executor"]):
|
|
22
18
|
@classmethod
|
|
23
|
-
def as_completed(cls, futures:
|
|
19
|
+
def as_completed(cls, futures: list[Future[_T]]) -> Iterator[Future[_T]]: ...
|
|
24
20
|
|
|
25
21
|
def submit(
|
|
26
22
|
self,
|
|
@@ -34,15 +30,15 @@ class Executor(Protocol, ContextManager["Executor"]):
|
|
|
34
30
|
self,
|
|
35
31
|
fn: Callable[[_S], _T],
|
|
36
32
|
args: Iterable[_S],
|
|
37
|
-
output_pickle_path_getter:
|
|
38
|
-
) ->
|
|
33
|
+
output_pickle_path_getter: Callable[[_S], PathLike] | None = None,
|
|
34
|
+
) -> list[Future[_T]]: ...
|
|
39
35
|
|
|
40
36
|
def map(
|
|
41
37
|
self,
|
|
42
38
|
fn: Callable[[_S], _T],
|
|
43
39
|
iterables: Iterable[_S],
|
|
44
|
-
timeout:
|
|
45
|
-
chunksize:
|
|
40
|
+
timeout: float | None = None,
|
|
41
|
+
chunksize: int | None = None,
|
|
46
42
|
) -> Iterator[_T]: ...
|
|
47
43
|
|
|
48
44
|
def forward_log(self, fut: Future[_T]) -> _T: ...
|
|
@@ -2,20 +2,15 @@ import logging
|
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
4
|
import signal
|
|
5
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
5
6
|
from concurrent import futures
|
|
6
7
|
from concurrent.futures import Future
|
|
7
8
|
from functools import partial
|
|
8
9
|
from multiprocessing import Queue, get_context
|
|
10
|
+
from pathlib import Path
|
|
9
11
|
from typing import (
|
|
10
12
|
TYPE_CHECKING,
|
|
11
13
|
Any,
|
|
12
|
-
Callable,
|
|
13
|
-
Dict,
|
|
14
|
-
Iterable,
|
|
15
|
-
Iterator,
|
|
16
|
-
List,
|
|
17
|
-
Optional,
|
|
18
|
-
Set,
|
|
19
14
|
TypeVar,
|
|
20
15
|
cast,
|
|
21
16
|
)
|
|
@@ -41,7 +36,7 @@ def _run_in_nanny(
|
|
|
41
36
|
queue: Queue, __fn: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
|
|
42
37
|
) -> None:
|
|
43
38
|
try:
|
|
44
|
-
__env = cast(
|
|
39
|
+
__env = cast(dict[str, str], kwargs.pop("__env"))
|
|
45
40
|
for key, value in __env.items():
|
|
46
41
|
os.environ[key] = value
|
|
47
42
|
|
|
@@ -82,7 +77,7 @@ def _parse_mem(size: str) -> int:
|
|
|
82
77
|
def _handle_kill_through_weakref(
|
|
83
78
|
executor_ref: "ReferenceType[DaskExecutor]",
|
|
84
79
|
existing_sigint_handler: Any,
|
|
85
|
-
signum:
|
|
80
|
+
signum: int | None,
|
|
86
81
|
frame: Any,
|
|
87
82
|
) -> None:
|
|
88
83
|
executor = executor_ref()
|
|
@@ -107,12 +102,12 @@ class DaskExecutor(futures.Executor):
|
|
|
107
102
|
"""
|
|
108
103
|
|
|
109
104
|
client: "Client"
|
|
110
|
-
pending_futures:
|
|
111
|
-
job_resources:
|
|
105
|
+
pending_futures: set[Future]
|
|
106
|
+
job_resources: dict[str, Any] | None
|
|
112
107
|
is_shutting_down = False
|
|
113
108
|
|
|
114
109
|
def __init__(
|
|
115
|
-
self, client: "Client", job_resources:
|
|
110
|
+
self, client: "Client", job_resources: dict[str, Any] | None = None
|
|
116
111
|
) -> None:
|
|
117
112
|
try:
|
|
118
113
|
import distributed # noqa: F401 unused import
|
|
@@ -148,7 +143,7 @@ class DaskExecutor(futures.Executor):
|
|
|
148
143
|
@classmethod
|
|
149
144
|
def from_config(
|
|
150
145
|
cls,
|
|
151
|
-
job_resources:
|
|
146
|
+
job_resources: dict[str, str],
|
|
152
147
|
**_kwargs: Any,
|
|
153
148
|
) -> "DaskExecutor":
|
|
154
149
|
from distributed import Client
|
|
@@ -162,7 +157,7 @@ class DaskExecutor(futures.Executor):
|
|
|
162
157
|
return cls(client, job_resources=job_resources)
|
|
163
158
|
|
|
164
159
|
@classmethod
|
|
165
|
-
def as_completed(cls, futures:
|
|
160
|
+
def as_completed(cls, futures: list[Future[_T]]) -> Iterator[Future[_T]]:
|
|
166
161
|
from distributed import as_completed
|
|
167
162
|
|
|
168
163
|
return as_completed(futures)
|
|
@@ -183,7 +178,7 @@ class DaskExecutor(futures.Executor):
|
|
|
183
178
|
Callable[_P, _T],
|
|
184
179
|
partial(
|
|
185
180
|
MultiprocessingExecutor._execute_and_persist_function,
|
|
186
|
-
output_pickle_path,
|
|
181
|
+
Path(output_pickle_path),
|
|
187
182
|
__fn,
|
|
188
183
|
),
|
|
189
184
|
)
|
|
@@ -200,8 +195,8 @@ class DaskExecutor(futures.Executor):
|
|
|
200
195
|
currently_available_workers = self.client.scheduler_info()["workers"]
|
|
201
196
|
|
|
202
197
|
def check_resources(
|
|
203
|
-
job_resources:
|
|
204
|
-
worker_resources:
|
|
198
|
+
job_resources: dict[str, float] | None,
|
|
199
|
+
worker_resources: dict[str, float] | None,
|
|
205
200
|
) -> bool:
|
|
206
201
|
if job_resources is None or len(job_resources) == 0:
|
|
207
202
|
return True
|
|
@@ -242,8 +237,8 @@ class DaskExecutor(futures.Executor):
|
|
|
242
237
|
args: Iterable[
|
|
243
238
|
_S
|
|
244
239
|
], # TODO change: allow more than one arg per call # noqa FIX002 Line contains TODO
|
|
245
|
-
output_pickle_path_getter:
|
|
246
|
-
) ->
|
|
240
|
+
output_pickle_path_getter: Callable[[_S], os.PathLike] | None = None,
|
|
241
|
+
) -> list[Future[_T]]:
|
|
247
242
|
if output_pickle_path_getter is not None:
|
|
248
243
|
futs = [
|
|
249
244
|
self.submit( # type: ignore[call-arg]
|
|
@@ -264,8 +259,8 @@ class DaskExecutor(futures.Executor):
|
|
|
264
259
|
self,
|
|
265
260
|
fn: Callable[[_S], _T],
|
|
266
261
|
iterables: Iterable[_S],
|
|
267
|
-
timeout:
|
|
268
|
-
chunksize:
|
|
262
|
+
timeout: float | None = None,
|
|
263
|
+
chunksize: int | None = None,
|
|
269
264
|
) -> Iterator[_T]:
|
|
270
265
|
if chunksize is None:
|
|
271
266
|
chunksize = 1
|
|
@@ -277,7 +272,7 @@ class DaskExecutor(futures.Executor):
|
|
|
277
272
|
def handle_kill(
|
|
278
273
|
self,
|
|
279
274
|
existing_sigint_handler: Any,
|
|
280
|
-
signum:
|
|
275
|
+
signum: int | None,
|
|
281
276
|
frame: Any,
|
|
282
277
|
) -> None:
|
|
283
278
|
if self.is_shutting_down:
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import multiprocessing
|
|
3
3
|
import os
|
|
4
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
4
5
|
from concurrent import futures
|
|
5
6
|
from concurrent.futures import Future, ProcessPoolExecutor
|
|
6
7
|
from functools import partial
|
|
@@ -8,12 +9,6 @@ from multiprocessing.context import BaseContext
|
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import (
|
|
10
11
|
Any,
|
|
11
|
-
Callable,
|
|
12
|
-
Iterable,
|
|
13
|
-
Iterator,
|
|
14
|
-
List,
|
|
15
|
-
Optional,
|
|
16
|
-
Tuple,
|
|
17
12
|
TypedDict,
|
|
18
13
|
TypeVar,
|
|
19
14
|
cast,
|
|
@@ -51,11 +46,11 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
51
46
|
def __init__(
|
|
52
47
|
self,
|
|
53
48
|
*,
|
|
54
|
-
max_workers:
|
|
55
|
-
start_method:
|
|
56
|
-
mp_context:
|
|
57
|
-
initializer:
|
|
58
|
-
initargs:
|
|
49
|
+
max_workers: int | None = None,
|
|
50
|
+
start_method: str | None = None,
|
|
51
|
+
mp_context: BaseContext | None = None,
|
|
52
|
+
initializer: Callable | None = None,
|
|
53
|
+
initargs: tuple = (),
|
|
59
54
|
**__kwargs: Any,
|
|
60
55
|
) -> None:
|
|
61
56
|
if mp_context is None:
|
|
@@ -68,9 +63,9 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
68
63
|
else:
|
|
69
64
|
mp_context = multiprocessing.get_context("spawn")
|
|
70
65
|
else:
|
|
71
|
-
assert (
|
|
72
|
-
start_method
|
|
73
|
-
)
|
|
66
|
+
assert start_method is None, (
|
|
67
|
+
"Cannot use both `start_method` and `mp_context` kwargs."
|
|
68
|
+
)
|
|
74
69
|
|
|
75
70
|
super().__init__(
|
|
76
71
|
mp_context=mp_context,
|
|
@@ -84,7 +79,7 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
84
79
|
self._mp_logging_handler_pool = _MultiprocessingLoggingHandlerPool()
|
|
85
80
|
|
|
86
81
|
@classmethod
|
|
87
|
-
def as_completed(cls, futs:
|
|
82
|
+
def as_completed(cls, futs: list[Future[_T]]) -> Iterator[Future[_T]]:
|
|
88
83
|
return futures.as_completed(futs)
|
|
89
84
|
|
|
90
85
|
def submit( # type: ignore[override]
|
|
@@ -141,8 +136,8 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
141
136
|
self,
|
|
142
137
|
fn: Callable[[_S], _T],
|
|
143
138
|
iterables: Iterable[_S],
|
|
144
|
-
timeout:
|
|
145
|
-
chunksize:
|
|
139
|
+
timeout: float | None = None,
|
|
140
|
+
chunksize: int | None = None,
|
|
146
141
|
) -> Iterator[_T]:
|
|
147
142
|
if chunksize is None:
|
|
148
143
|
chunksize = 1
|
|
@@ -151,10 +146,10 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
151
146
|
@staticmethod
|
|
152
147
|
def _setup_logging_and_execute(
|
|
153
148
|
multiprocessing_logging_setup_fn: Callable[[], None],
|
|
154
|
-
fn: Callable[_P,
|
|
149
|
+
fn: Callable[_P, _T],
|
|
155
150
|
*args: Any,
|
|
156
151
|
**kwargs: Any,
|
|
157
|
-
) ->
|
|
152
|
+
) -> _T:
|
|
158
153
|
multiprocessing_logging_setup_fn()
|
|
159
154
|
return fn(*args, **kwargs)
|
|
160
155
|
|
|
@@ -186,8 +181,8 @@ class MultiprocessingExecutor(ProcessPoolExecutor):
|
|
|
186
181
|
self,
|
|
187
182
|
fn: Callable[[_S], _T],
|
|
188
183
|
args: Iterable[_S],
|
|
189
|
-
output_pickle_path_getter:
|
|
190
|
-
) ->
|
|
184
|
+
output_pickle_path_getter: Callable[[_S], os.PathLike] | None = None,
|
|
185
|
+
) -> list[Future[_T]]:
|
|
191
186
|
if output_pickle_path_getter is not None:
|
|
192
187
|
futs = [
|
|
193
188
|
self.submit( # type: ignore[call-arg]
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import warnings
|
|
2
|
-
from collections.abc import Iterable, Iterator
|
|
2
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
3
3
|
from concurrent.futures import Executor, Future, as_completed
|
|
4
4
|
from os import PathLike
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Any,
|
|
6
|
+
from typing import Any, TypeVar, cast
|
|
7
7
|
|
|
8
8
|
from typing_extensions import ParamSpec
|
|
9
9
|
|
|
@@ -59,7 +59,7 @@ class SequentialExecutor(Executor):
|
|
|
59
59
|
self,
|
|
60
60
|
fn: Callable[[_S], _T],
|
|
61
61
|
args: Iterable[_S],
|
|
62
|
-
output_pickle_path_getter:
|
|
62
|
+
output_pickle_path_getter: Callable[[_S], PathLike] | None = None,
|
|
63
63
|
) -> list[Future[_T]]:
|
|
64
64
|
if output_pickle_path_getter is not None:
|
|
65
65
|
futs = [
|
|
@@ -81,8 +81,8 @@ class SequentialExecutor(Executor):
|
|
|
81
81
|
self,
|
|
82
82
|
fn: Callable[[_S], _T],
|
|
83
83
|
iterables: Iterable[_S],
|
|
84
|
-
timeout:
|
|
85
|
-
chunksize:
|
|
84
|
+
timeout: float | None = None,
|
|
85
|
+
chunksize: int | None = None,
|
|
86
86
|
) -> Iterator[_T]:
|
|
87
87
|
if timeout is not None:
|
|
88
88
|
warnings.warn(
|
|
@@ -4,7 +4,7 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
6
6
|
import traceback
|
|
7
|
-
from typing import Any
|
|
7
|
+
from typing import Any
|
|
8
8
|
|
|
9
9
|
from cluster_tools._utils import pickling
|
|
10
10
|
from cluster_tools._utils.string_ import with_preliminary_postfix
|
|
@@ -14,7 +14,7 @@ from cluster_tools.schedulers.pbs import PBSExecutor
|
|
|
14
14
|
from cluster_tools.schedulers.slurm import SlurmExecutor
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def get_executor_class(executor_key: str) ->
|
|
17
|
+
def get_executor_class(executor_key: str) -> type[ClusterExecutor]:
|
|
18
18
|
return {
|
|
19
19
|
"slurm": SlurmExecutor,
|
|
20
20
|
"pbs": PBSExecutor,
|
|
@@ -29,21 +29,19 @@ def format_remote_exc() -> str:
|
|
|
29
29
|
return "".join(traceback.format_exception(typ, value, tb))
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def get_custom_main_path(
|
|
33
|
-
workerid: str, executor: Type[ClusterExecutor]
|
|
34
|
-
) -> Optional[str]:
|
|
32
|
+
def get_custom_main_path(workerid: str, executor: type[ClusterExecutor]) -> str | None:
|
|
35
33
|
custom_main_path = None
|
|
36
34
|
main_meta_path = executor.get_main_meta_path(cfut_dir, workerid)
|
|
37
35
|
if os.path.exists(main_meta_path):
|
|
38
|
-
with open(main_meta_path,
|
|
36
|
+
with open(main_meta_path, encoding="utf-8") as file:
|
|
39
37
|
custom_main_path = file.read()
|
|
40
38
|
return custom_main_path
|
|
41
39
|
|
|
42
40
|
|
|
43
41
|
def worker(
|
|
44
|
-
executor:
|
|
42
|
+
executor: type[ClusterExecutor],
|
|
45
43
|
workerid: str,
|
|
46
|
-
job_array_index:
|
|
44
|
+
job_array_index: int | None,
|
|
47
45
|
job_array_index_offset: str,
|
|
48
46
|
cfut_dir: str,
|
|
49
47
|
) -> None:
|
|
@@ -75,9 +73,7 @@ def worker(
|
|
|
75
73
|
setup_logging(meta_data, executor, cfut_dir)
|
|
76
74
|
|
|
77
75
|
logging.info(
|
|
78
|
-
"Job computation started (jobid={}, workerid_with_idx={})."
|
|
79
|
-
executor.get_current_job_id(), workerid_with_idx
|
|
80
|
-
)
|
|
76
|
+
f"Job computation started (jobid={executor.get_current_job_id()}, workerid_with_idx={workerid_with_idx})."
|
|
81
77
|
)
|
|
82
78
|
result = True, fun(*args, **kwargs)
|
|
83
79
|
logging.info("Job computation completed.")
|
|
@@ -98,13 +94,13 @@ def worker(
|
|
|
98
94
|
tempfile = str(destfile) + ".tmp"
|
|
99
95
|
with open(tempfile, "wb") as f:
|
|
100
96
|
f.write(out)
|
|
101
|
-
logging.debug("Pickle file written to {}."
|
|
97
|
+
logging.debug(f"Pickle file written to {tempfile}.")
|
|
102
98
|
os.rename(tempfile, destfile)
|
|
103
|
-
logging.debug("Pickle file renamed to {}."
|
|
99
|
+
logging.debug(f"Pickle file renamed to {destfile}.")
|
|
104
100
|
|
|
105
101
|
|
|
106
102
|
def setup_logging(
|
|
107
|
-
meta_data:
|
|
103
|
+
meta_data: dict[str, Any], executor: type[ClusterExecutor], cfut_dir: str
|
|
108
104
|
) -> None:
|
|
109
105
|
if "logging_setup_fn" in meta_data:
|
|
110
106
|
logging.debug("Using supplied logging_setup_fn to setup logging.")
|
|
@@ -118,9 +114,7 @@ def setup_logging(
|
|
|
118
114
|
logging_config = meta_data.get("logging_config", dict())
|
|
119
115
|
|
|
120
116
|
logging.debug(
|
|
121
|
-
"Setting up logging.basicConfig (potentially overwriting logging configuration of the main script). Config: {}"
|
|
122
|
-
logging_config
|
|
123
|
-
)
|
|
117
|
+
f"Setting up logging.basicConfig (potentially overwriting logging configuration of the main script). Config: {logging_config}"
|
|
124
118
|
)
|
|
125
119
|
|
|
126
120
|
# Call basicConfig which is necessary for the logging to work.
|