cluster-tools 2.4.4__tar.gz → 2.4.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/PKG-INFO +4 -4
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/cluster_executor.py +92 -36
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/kube.py +1 -5
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/pbs.py +2 -2
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/slurm.py +7 -2
- cluster_tools-2.4.6/cluster_tools/version.py +1 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/PKG-INFO +4 -4
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/pyproject.toml +4 -3
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_slurm.py +99 -34
- cluster_tools-2.4.4/cluster_tools/version.py +0 -1
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/LICENSE +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/README.md +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/__init__.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/call.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/file_wait_thread.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/pickling.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/reflection.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/string_.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/tailf.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/warning.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executor_protocol.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/dask.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/multiprocessing_.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/sequential.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/sequential_pickle.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/remote.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/__init__.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/SOURCES.txt +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/dependency_links.txt +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/requires.txt +2 -2
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/top_level.txt +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/setup.cfg +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_all.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_dask.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_deref_main.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_kubernetes.py +0 -0
- {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_multiprocessing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.6
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,10 +10,10 @@ Requires-Python: >=3.10
|
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: typing-extensions~=4.12.0
|
|
13
|
-
Provides-Extra: kubernetes
|
|
14
|
-
Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
|
|
15
13
|
Provides-Extra: dask
|
|
16
|
-
Requires-Dist:
|
|
14
|
+
Requires-Dist: distributed~=2023.9.1; extra == "dask"
|
|
15
|
+
Provides-Extra: kubernetes
|
|
16
|
+
Requires-Dist: kubernetes~=27.2.0; extra == "kubernetes"
|
|
17
17
|
Provides-Extra: all
|
|
18
18
|
Requires-Dist: cluster_tools[kubernetes]; extra == "all"
|
|
19
19
|
Requires-Dist: cluster_tools[dask]; extra == "all"
|
|
@@ -9,13 +9,13 @@ from collections.abc import Callable, Iterable, Iterator
|
|
|
9
9
|
from concurrent import futures
|
|
10
10
|
from concurrent.futures import Future
|
|
11
11
|
from functools import partial
|
|
12
|
+
from types import FrameType, TracebackType
|
|
12
13
|
from typing import (
|
|
13
14
|
Any,
|
|
14
15
|
Literal,
|
|
15
16
|
TypeVar,
|
|
16
17
|
cast,
|
|
17
18
|
)
|
|
18
|
-
from weakref import ReferenceType, ref
|
|
19
19
|
|
|
20
20
|
from typing_extensions import ParamSpec
|
|
21
21
|
|
|
@@ -38,18 +38,6 @@ _P = ParamSpec("_P")
|
|
|
38
38
|
_S = TypeVar("_S")
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def _handle_kill_through_weakref(
|
|
42
|
-
executor_ref: "ReferenceType[ClusterExecutor]",
|
|
43
|
-
existing_sigint_handler: Any,
|
|
44
|
-
signum: int | None,
|
|
45
|
-
frame: Any,
|
|
46
|
-
) -> None:
|
|
47
|
-
executor = executor_ref()
|
|
48
|
-
if executor is None:
|
|
49
|
-
return
|
|
50
|
-
executor.handle_kill(existing_sigint_handler, signum, frame)
|
|
51
|
-
|
|
52
|
-
|
|
53
41
|
def join_messages(strings: list[str]) -> str:
|
|
54
42
|
return " ".join(x.strip() for x in strings if x.strip())
|
|
55
43
|
|
|
@@ -79,6 +67,9 @@ class RemoteTimeLimitException(RemoteResourceLimitException):
|
|
|
79
67
|
class ClusterExecutor(futures.Executor):
|
|
80
68
|
"""Futures executor for executing jobs on a cluster."""
|
|
81
69
|
|
|
70
|
+
_shutdown_hooks: list[Callable[[], None]] = []
|
|
71
|
+
_installed_signal_handler: bool = False
|
|
72
|
+
|
|
82
73
|
def __init__(
|
|
83
74
|
self,
|
|
84
75
|
debug: bool = False,
|
|
@@ -103,7 +94,6 @@ class ClusterExecutor(futures.Executor):
|
|
|
103
94
|
self.job_resources = job_resources
|
|
104
95
|
self.additional_setup_lines = additional_setup_lines or []
|
|
105
96
|
self.job_name = job_name
|
|
106
|
-
self.was_requested_to_shutdown = False
|
|
107
97
|
self.cfut_dir = (
|
|
108
98
|
cfut_dir if cfut_dir is not None else os.getenv("CFUT_DIR", ".cfut")
|
|
109
99
|
)
|
|
@@ -130,15 +120,7 @@ class ClusterExecutor(futures.Executor):
|
|
|
130
120
|
|
|
131
121
|
os.makedirs(self.cfut_dir, exist_ok=True)
|
|
132
122
|
|
|
133
|
-
|
|
134
|
-
# existing signal handler of the process or the
|
|
135
|
-
# shutdown of the main process which sends SIGTERM signals to terminate all
|
|
136
|
-
# child processes.
|
|
137
|
-
existing_sigint_handler = signal.getsignal(signal.SIGINT)
|
|
138
|
-
signal.signal(
|
|
139
|
-
signal.SIGINT,
|
|
140
|
-
partial(_handle_kill_through_weakref, ref(self), existing_sigint_handler),
|
|
141
|
-
)
|
|
123
|
+
self._register_shutdown_hook(self.handle_kill)
|
|
142
124
|
|
|
143
125
|
self.metadata = {}
|
|
144
126
|
assert not ("logging_config" in kwargs and "logging_setup_fn" in kwargs), (
|
|
@@ -158,26 +140,81 @@ class ClusterExecutor(futures.Executor):
|
|
|
158
140
|
def executor_key(cls) -> str:
|
|
159
141
|
pass
|
|
160
142
|
|
|
161
|
-
|
|
162
|
-
|
|
143
|
+
@classmethod
|
|
144
|
+
def _ensure_signal_handlers_are_installed(cls) -> None:
|
|
145
|
+
# Only overwrite the signal handler once
|
|
146
|
+
if cls._installed_signal_handler:
|
|
147
|
+
return
|
|
148
|
+
|
|
149
|
+
# Clean up if a SIGINT or SIGTERM signal is received. However, do not
|
|
150
|
+
# interfere with the existing signal handler of the process and execute
|
|
151
|
+
# it afterwards.
|
|
152
|
+
existing_sigint_handler = signal.getsignal(signal.SIGINT)
|
|
153
|
+
signal.signal(
|
|
154
|
+
signal.SIGINT,
|
|
155
|
+
partial(cls._handle_shutdown, existing_sigint_handler),
|
|
156
|
+
)
|
|
157
|
+
existing_sigterm_handler = signal.getsignal(signal.SIGTERM)
|
|
158
|
+
signal.signal(
|
|
159
|
+
signal.SIGTERM,
|
|
160
|
+
partial(cls._handle_shutdown, existing_sigterm_handler),
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
cls._installed_signal_handler = True
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def _register_shutdown_hook(cls, hook: Callable[[], None]) -> None:
|
|
167
|
+
cls._shutdown_hooks.append(hook)
|
|
168
|
+
cls._ensure_signal_handlers_are_installed()
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def _deregister_shutdown_hook(cls, hook: Callable[[], None]) -> None:
|
|
172
|
+
if hook in cls._shutdown_hooks:
|
|
173
|
+
cls._shutdown_hooks.remove(hook)
|
|
174
|
+
else:
|
|
175
|
+
logging.warning(
|
|
176
|
+
"Cannot deregister executors shutdown hook since it's not registered."
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
@classmethod
|
|
180
|
+
def _handle_shutdown(
|
|
181
|
+
cls,
|
|
182
|
+
existing_signal_handler: Callable[[int, FrameType | None], None] | int | None,
|
|
183
|
+
signum: int,
|
|
184
|
+
frame: Any,
|
|
163
185
|
) -> None:
|
|
186
|
+
logging.critical(
|
|
187
|
+
f"[{cls.__name__}] Caught signal {signal.Signals(signum).name}, running shutdown hooks"
|
|
188
|
+
)
|
|
189
|
+
try:
|
|
190
|
+
for hook in cls._shutdown_hooks:
|
|
191
|
+
hook()
|
|
192
|
+
except Exception as e:
|
|
193
|
+
print(f"Error during shutdown: {e}")
|
|
194
|
+
|
|
195
|
+
if (
|
|
196
|
+
callable(existing_signal_handler)
|
|
197
|
+
and existing_signal_handler
|
|
198
|
+
not in (
|
|
199
|
+
signal.SIG_DFL, # For completeness sake (since it's not callable anyways). The system's default signal handler
|
|
200
|
+
signal.SIG_IGN, # For completeness sake (since it's not callable anyways). The instruction to ignore a signal
|
|
201
|
+
signal.default_int_handler, # Python's default SIGINT handler
|
|
202
|
+
)
|
|
203
|
+
):
|
|
204
|
+
existing_signal_handler(signum, frame)
|
|
205
|
+
|
|
206
|
+
def handle_kill(self) -> None:
|
|
164
207
|
if self.is_shutting_down:
|
|
165
208
|
return
|
|
166
209
|
|
|
167
210
|
self.is_shutting_down = True
|
|
168
211
|
|
|
169
|
-
self.inner_handle_kill(signum, frame)
|
|
170
212
|
self.wait_thread.stop()
|
|
213
|
+
self.inner_handle_kill()
|
|
171
214
|
self.clean_up()
|
|
172
215
|
|
|
173
|
-
if (
|
|
174
|
-
existing_sigint_handler != signal.default_int_handler
|
|
175
|
-
and callable(existing_sigint_handler) # Could also be signal.SIG_IGN
|
|
176
|
-
):
|
|
177
|
-
existing_sigint_handler(signum, frame)
|
|
178
|
-
|
|
179
216
|
@abstractmethod
|
|
180
|
-
def inner_handle_kill(self
|
|
217
|
+
def inner_handle_kill(self) -> None:
|
|
181
218
|
pass
|
|
182
219
|
|
|
183
220
|
@abstractmethod
|
|
@@ -363,9 +400,9 @@ class ClusterExecutor(futures.Executor):
|
|
|
363
400
|
self._maybe_mark_logs_for_cleanup(jobid)
|
|
364
401
|
|
|
365
402
|
def ensure_not_shutdown(self) -> None:
|
|
366
|
-
if self.
|
|
403
|
+
if self.is_shutting_down:
|
|
367
404
|
raise RuntimeError(
|
|
368
|
-
"submit() was invoked on a ClusterExecutor instance even though shutdown() was executed for that instance."
|
|
405
|
+
"submit() was invoked on a ClusterExecutor instance even though shutdown() or handle_kill() was executed for that instance."
|
|
369
406
|
)
|
|
370
407
|
|
|
371
408
|
def create_enriched_future(self) -> Future:
|
|
@@ -591,17 +628,35 @@ class ClusterExecutor(futures.Executor):
|
|
|
591
628
|
should_keep_output,
|
|
592
629
|
)
|
|
593
630
|
|
|
631
|
+
# Overwrite the context manager __exit as it doesn't forward the information whether an exception was thrown or not otherwise
|
|
632
|
+
# which may lead to a deadlock if an exception is thrown within a cluster executor with statement, because self.jobs_empty_cond.wait()
|
|
633
|
+
# never succeeds.
|
|
634
|
+
def __exit__(
|
|
635
|
+
self,
|
|
636
|
+
exc_type: type[BaseException] | None,
|
|
637
|
+
_exc_val: BaseException | None,
|
|
638
|
+
_exc_tb: TracebackType | None,
|
|
639
|
+
) -> Literal[False]:
|
|
640
|
+
# Don't wait if an exception was thrown
|
|
641
|
+
self.shutdown(wait=exc_type is None)
|
|
642
|
+
return False
|
|
643
|
+
|
|
594
644
|
def shutdown(self, wait: bool = True, cancel_futures: bool = True) -> None:
|
|
595
645
|
"""Close the pool."""
|
|
646
|
+
if self.is_shutting_down:
|
|
647
|
+
return
|
|
648
|
+
|
|
649
|
+
self.is_shutting_down = True
|
|
596
650
|
if not cancel_futures:
|
|
597
651
|
logging.warning(
|
|
598
652
|
"The provided cancel_futures argument is ignored by ClusterExecutor."
|
|
599
653
|
)
|
|
600
|
-
self.was_requested_to_shutdown = True
|
|
601
654
|
if wait:
|
|
602
655
|
with self.jobs_lock:
|
|
603
656
|
if self.jobs and self.wait_thread.is_alive():
|
|
604
657
|
self.jobs_empty_cond.wait()
|
|
658
|
+
else:
|
|
659
|
+
self.inner_handle_kill()
|
|
605
660
|
|
|
606
661
|
self.wait_thread.stop()
|
|
607
662
|
self.wait_thread.join()
|
|
@@ -617,6 +672,7 @@ class ClusterExecutor(futures.Executor):
|
|
|
617
672
|
f"Could not delete file during clean up. Path: {file_to_clean_up} Exception: {exc}. Continuing..."
|
|
618
673
|
)
|
|
619
674
|
self.files_to_clean_up = []
|
|
675
|
+
self._deregister_shutdown_hook(self.handle_kill)
|
|
620
676
|
|
|
621
677
|
def map( # type: ignore[override]
|
|
622
678
|
self,
|
|
@@ -113,11 +113,7 @@ class KubernetesExecutor(ClusterExecutor):
|
|
|
113
113
|
return job_id
|
|
114
114
|
return cls.get_jobid_with_index(job_id, job_index)
|
|
115
115
|
|
|
116
|
-
def inner_handle_kill(
|
|
117
|
-
self,
|
|
118
|
-
*args: Any, # noqa: ARG002 Unused method argument: `args`
|
|
119
|
-
**kwargs: Any, # noqa: ARG002 Unused method argument: `kwargs`
|
|
120
|
-
) -> None:
|
|
116
|
+
def inner_handle_kill(self) -> None:
|
|
121
117
|
job_ids = ",".join(str(job_id) for job_id in self.jobs.keys())
|
|
122
118
|
|
|
123
119
|
print(
|
|
@@ -4,7 +4,7 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
import re
|
|
6
6
|
from concurrent.futures import Future
|
|
7
|
-
from typing import
|
|
7
|
+
from typing import Literal
|
|
8
8
|
|
|
9
9
|
from cluster_tools._utils.call import call, chcall
|
|
10
10
|
from cluster_tools._utils.string_ import random_string
|
|
@@ -56,7 +56,7 @@ class PBSExecutor(ClusterExecutor):
|
|
|
56
56
|
def get_job_id_string(cls) -> str:
|
|
57
57
|
return cls.get_current_job_id()
|
|
58
58
|
|
|
59
|
-
def inner_handle_kill(self
|
|
59
|
+
def inner_handle_kill(self) -> None:
|
|
60
60
|
scheduled_job_ids: list[int | str] = list(self.jobs.keys())
|
|
61
61
|
|
|
62
62
|
if len(scheduled_job_ids):
|
|
@@ -244,7 +244,7 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
244
244
|
|
|
245
245
|
return str(int(job_id)) # int() ensures coherent parsing
|
|
246
246
|
|
|
247
|
-
def inner_handle_kill(self
|
|
247
|
+
def inner_handle_kill(self) -> None:
|
|
248
248
|
for submit_thread in self.submit_threads:
|
|
249
249
|
submit_thread.stop()
|
|
250
250
|
|
|
@@ -260,10 +260,15 @@ class SlurmExecutor(ClusterExecutor):
|
|
|
260
260
|
# but can be canceled together using the job_id.
|
|
261
261
|
unique_job_ids = set(map(lambda x: str(x).split("_")[0], scheduled_job_ids))
|
|
262
262
|
job_id_string = " ".join(unique_job_ids)
|
|
263
|
+
# Allow to speed up the shutdown, for example, when running voxelytics locally
|
|
264
|
+
sigterm_wait_in_s_env = float(os.environ.get("SIGTERM_WAIT_IN_S", 5))
|
|
263
265
|
# Send SIGINT signal to running jobs instead of terminating the jobs right away. This way, the jobs can
|
|
264
266
|
# react to the signal, safely shutdown and signal (cancel) jobs they possibly scheduled, recursively.
|
|
267
|
+
# After a short waiting time kill all jobs that are still running (due to race conditions or because they
|
|
268
|
+
# didn't react to the SIGINT signal for some reason).
|
|
265
269
|
_, stderr, _ = call(
|
|
266
|
-
f"scancel --state=PENDING {job_id_string}; scancel -s SIGINT --state=RUNNING {job_id_string};
|
|
270
|
+
f"scancel --state=PENDING {job_id_string}; scancel -s SIGINT --state=RUNNING {job_id_string};"
|
|
271
|
+
+ f"scancel --state=SUSPENDED {job_id_string}; sleep {sigterm_wait_in_s_env}; scancel {job_id_string}"
|
|
267
272
|
)
|
|
268
273
|
|
|
269
274
|
maybe_error_or_warning = (
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.4.6'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cluster_tools
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.6
|
|
4
4
|
Summary: Utility library for easily distributing code execution on clusters
|
|
5
5
|
Author-email: scalable minds <hello@scalableminds.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,10 +10,10 @@ Requires-Python: >=3.10
|
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: typing-extensions~=4.12.0
|
|
13
|
-
Provides-Extra: kubernetes
|
|
14
|
-
Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
|
|
15
13
|
Provides-Extra: dask
|
|
16
|
-
Requires-Dist:
|
|
14
|
+
Requires-Dist: distributed~=2023.9.1; extra == "dask"
|
|
15
|
+
Provides-Extra: kubernetes
|
|
16
|
+
Requires-Dist: kubernetes~=27.2.0; extra == "kubernetes"
|
|
17
17
|
Provides-Extra: all
|
|
18
18
|
Requires-Dist: cluster_tools[kubernetes]; extra == "all"
|
|
19
19
|
Requires-Dist: cluster_tools[dask]; extra == "all"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cluster_tools"
|
|
3
|
-
version = "2.4.
|
|
3
|
+
version = "2.4.6" # filled by dunamai
|
|
4
4
|
description = "Utility library for easily distributing code execution on clusters"
|
|
5
5
|
authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
|
|
6
6
|
readme = "README.md"
|
|
@@ -17,8 +17,8 @@ Changelog = "https://github.com/scalableminds/webknossos-libs/blob/master/cluste
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
[project.optional-dependencies]
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
dask = ["distributed ~=2023.9.1"]
|
|
21
|
+
kubernetes = ["kubernetes ~=27.2.0"]
|
|
22
22
|
all=["cluster_tools[kubernetes]", "cluster_tools[dask]"]
|
|
23
23
|
|
|
24
24
|
[tool.uv]
|
|
@@ -26,6 +26,7 @@ dev-dependencies = [
|
|
|
26
26
|
"icecream ~=2.1.1",
|
|
27
27
|
"mypy ~=1.15.0",
|
|
28
28
|
"pytest ~=8.3.3",
|
|
29
|
+
"pytest-timeout>=2.4.0",
|
|
29
30
|
"ruff ~=0.11.0",
|
|
30
31
|
]
|
|
31
32
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import concurrent.futures
|
|
2
2
|
import contextlib
|
|
3
|
+
import gc
|
|
3
4
|
import io
|
|
4
5
|
import logging
|
|
5
6
|
import multiprocessing as mp
|
|
@@ -168,10 +169,12 @@ def wait_until_first_job_was_submitted(
|
|
|
168
169
|
time.sleep(0.1)
|
|
169
170
|
|
|
170
171
|
|
|
171
|
-
|
|
172
|
+
@pytest.mark.timeout(30)
|
|
173
|
+
def test_slurm_deferred_submit_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
172
174
|
# Test that the SlurmExecutor stops scheduling jobs in a separate thread
|
|
173
175
|
# once it was killed even if the executor was used multiple times and
|
|
174
176
|
# therefore started multiple job submission threads
|
|
177
|
+
monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
|
|
175
178
|
max_submit_jobs = 1
|
|
176
179
|
|
|
177
180
|
# Only one job can be scheduled at a time
|
|
@@ -179,43 +182,48 @@ def test_slurm_deferred_submit_shutdown() -> None:
|
|
|
179
182
|
executor = cluster_tools.get_executor("slurm", debug=True)
|
|
180
183
|
|
|
181
184
|
try:
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
+
with executor:
|
|
186
|
+
# Use the executor twice to start multiple job submission threads
|
|
187
|
+
executor.map_to_futures(sleep, [0.5] * 10)
|
|
188
|
+
executor.map_to_futures(sleep, [0.5] * 10)
|
|
185
189
|
|
|
186
|
-
|
|
190
|
+
wait_until_first_job_was_submitted(executor)
|
|
187
191
|
|
|
188
|
-
|
|
189
|
-
|
|
192
|
+
for submit_thread in executor.submit_threads:
|
|
193
|
+
assert submit_thread.is_alive()
|
|
190
194
|
|
|
191
|
-
|
|
195
|
+
sigint_handler = signal.getsignal(signal.SIGINT)
|
|
196
|
+
assert callable(sigint_handler) # Mainly for typechecking
|
|
197
|
+
sigint_handler(signal.SIGINT, None)
|
|
192
198
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
199
|
+
# Wait for the threads to die down, but less than it would take to submit all jobs
|
|
200
|
+
# which would take ~5 seconds since only one job is scheduled at a time
|
|
201
|
+
for submit_thread in executor.submit_threads:
|
|
202
|
+
submit_thread.join(1)
|
|
203
|
+
assert not submit_thread.is_alive()
|
|
198
204
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
205
|
+
# Wait for scheduled jobs to be canceled, so that the queue is empty again
|
|
206
|
+
while executor.get_number_of_submitted_jobs() > 0:
|
|
207
|
+
time.sleep(0.5)
|
|
202
208
|
|
|
203
209
|
finally:
|
|
204
210
|
call("echo y | sacctmgr modify qos normal set MaxSubmitJobs=-1")
|
|
205
211
|
|
|
206
212
|
|
|
207
|
-
|
|
213
|
+
@pytest.mark.timeout(30)
|
|
214
|
+
def test_slurm_job_canceling_on_shutdown(
|
|
215
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
216
|
+
) -> None:
|
|
208
217
|
# Test that scheduled jobs are canceled on shutdown, regardless
|
|
209
218
|
# of whether they are pending or running.
|
|
210
|
-
|
|
219
|
+
monkeypatch.setenv("SLURM_MAX_RUNNING_SIZE", "2")
|
|
220
|
+
monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
|
|
211
221
|
|
|
212
222
|
executor = cluster_tools.get_executor("slurm", debug=True)
|
|
213
|
-
# Only two jobs can run at once, so that some of the jobs will be
|
|
214
|
-
# running and some will be pending.
|
|
215
|
-
original_max_running_size = os.environ.get("SLURM_MAX_RUNNING_SIZE")
|
|
216
|
-
os.environ["SLURM_MAX_RUNNING_SIZE"] = str(max_running_size)
|
|
217
223
|
|
|
218
|
-
|
|
224
|
+
with executor:
|
|
225
|
+
# Only two jobs can run at once, so that some of the jobs will be
|
|
226
|
+
# running and some will be pending.
|
|
219
227
|
executor.map_to_futures(sleep, [10] * 4)
|
|
220
228
|
|
|
221
229
|
# Wait until first job is running
|
|
@@ -223,18 +231,77 @@ def test_slurm_job_canceling_on_shutdown() -> None:
|
|
|
223
231
|
|
|
224
232
|
job_start_time = time.time()
|
|
225
233
|
|
|
226
|
-
|
|
227
|
-
#
|
|
228
|
-
|
|
229
|
-
time.sleep(1)
|
|
230
|
-
|
|
231
|
-
executor.handle_kill(signal.default_int_handler, None, None)
|
|
234
|
+
sigint_handler = signal.getsignal(signal.SIGINT)
|
|
235
|
+
assert callable(sigint_handler) # Mainly for typechecking
|
|
236
|
+
sigint_handler(signal.SIGINT, None)
|
|
232
237
|
|
|
233
238
|
# Wait for scheduled jobs to be canceled, so that the queue is empty again
|
|
234
239
|
# and measure how long the cancellation takes
|
|
235
240
|
while executor.get_number_of_submitted_jobs() > 0:
|
|
236
241
|
time.sleep(0.5)
|
|
237
242
|
|
|
243
|
+
job_cancellation_duration = time.time() - job_start_time
|
|
244
|
+
|
|
245
|
+
# Killing the executor should have canceled all submitted jobs, regardless
|
|
246
|
+
# of whether they were running or pending in much less time than it would
|
|
247
|
+
# have taken the jobs to finish on their own
|
|
248
|
+
assert job_cancellation_duration < 5
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@pytest.mark.timeout(30)
|
|
252
|
+
def test_slurm_signal_handling(
|
|
253
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
254
|
+
) -> None:
|
|
255
|
+
monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
|
|
256
|
+
original_sigint_handler_was_called = False
|
|
257
|
+
|
|
258
|
+
def original_sigint_handler(_signum: int | None, _frame: Any) -> None:
|
|
259
|
+
nonlocal original_sigint_handler_was_called
|
|
260
|
+
original_sigint_handler_was_called = True
|
|
261
|
+
|
|
262
|
+
signal.signal(
|
|
263
|
+
signal.SIGINT,
|
|
264
|
+
original_sigint_handler,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 0
|
|
268
|
+
# Reset class variable after changing signal handler so it will be set again when the next cluster executor is instantiated
|
|
269
|
+
cluster_tools.SlurmExecutor._installed_signal_handler = False
|
|
270
|
+
|
|
271
|
+
with cluster_tools.get_executor("slurm", debug=True) as executor1:
|
|
272
|
+
assert executor1.get_number_of_submitted_jobs() == 0
|
|
273
|
+
assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 1
|
|
274
|
+
|
|
275
|
+
futures = executor1.map_to_futures(square, [2])
|
|
276
|
+
concurrent.futures.wait(futures)
|
|
277
|
+
|
|
278
|
+
# Let the first executor be no longer referenced to provoke potential bugs in the signal handler chaining
|
|
279
|
+
# See https://github.com/scalableminds/webknossos-libs/pull/1317
|
|
280
|
+
del executor1
|
|
281
|
+
gc.collect()
|
|
282
|
+
|
|
283
|
+
with cluster_tools.get_executor("slurm", debug=True) as executor2:
|
|
284
|
+
assert executor2.get_number_of_submitted_jobs() == 0
|
|
285
|
+
assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 1
|
|
286
|
+
|
|
287
|
+
executor2.map_to_futures(sleep, [10] * 4)
|
|
288
|
+
|
|
289
|
+
# Wait until first job is running
|
|
290
|
+
wait_until_first_job_was_submitted(executor2, "RUNNING")
|
|
291
|
+
|
|
292
|
+
job_start_time = time.time()
|
|
293
|
+
|
|
294
|
+
sigint_handler = signal.getsignal(signal.SIGINT)
|
|
295
|
+
assert callable(sigint_handler) # Mainly for typechecking
|
|
296
|
+
sigint_handler(signal.SIGINT, None)
|
|
297
|
+
|
|
298
|
+
assert original_sigint_handler_was_called
|
|
299
|
+
|
|
300
|
+
# Wait for scheduled jobs to be canceled, so that the queue is empty again
|
|
301
|
+
# and measure how long the cancellation takes
|
|
302
|
+
while executor2.get_number_of_submitted_jobs() > 0:
|
|
303
|
+
time.sleep(0.5)
|
|
304
|
+
|
|
238
305
|
job_cancellation_duration = time.time() - job_start_time
|
|
239
306
|
|
|
240
307
|
# Killing the executor should have canceled all submitted jobs, regardless
|
|
@@ -242,11 +309,9 @@ def test_slurm_job_canceling_on_shutdown() -> None:
|
|
|
242
309
|
# have taken the jobs to finish on their own
|
|
243
310
|
assert job_cancellation_duration < 5
|
|
244
311
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
else:
|
|
249
|
-
del os.environ["SLURM_MAX_RUNNING_SIZE"]
|
|
312
|
+
assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 0, (
|
|
313
|
+
"The cluster executor shutdown hooks should have been cleaned up"
|
|
314
|
+
)
|
|
250
315
|
|
|
251
316
|
|
|
252
317
|
def test_slurm_number_of_submitted_jobs() -> None:
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.4.4'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/multiprocessing_logging_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/multiprocessing_pickle.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|