cluster-tools 2.4.4__tar.gz → 2.4.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/PKG-INFO +4 -4
  2. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/cluster_executor.py +92 -36
  3. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/kube.py +1 -5
  4. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/pbs.py +2 -2
  5. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/slurm.py +7 -2
  6. cluster_tools-2.4.6/cluster_tools/version.py +1 -0
  7. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/PKG-INFO +4 -4
  8. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/pyproject.toml +4 -3
  9. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_slurm.py +99 -34
  10. cluster_tools-2.4.4/cluster_tools/version.py +0 -1
  11. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/LICENSE +0 -0
  12. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/README.md +0 -0
  13. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/__init__.py +0 -0
  14. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/call.py +0 -0
  15. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/file_wait_thread.py +0 -0
  16. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/multiprocessing_logging_handler.py +0 -0
  17. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/pickling.py +0 -0
  18. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/reflection.py +0 -0
  19. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/string_.py +0 -0
  20. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/tailf.py +0 -0
  21. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/_utils/warning.py +0 -0
  22. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executor_protocol.py +0 -0
  23. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/dask.py +0 -0
  24. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/multiprocessing_.py +0 -0
  25. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
  26. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/sequential.py +0 -0
  27. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/executors/sequential_pickle.py +0 -0
  28. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/remote.py +0 -0
  29. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools/schedulers/__init__.py +0 -0
  30. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/SOURCES.txt +0 -0
  31. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/dependency_links.txt +0 -0
  32. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/requires.txt +2 -2
  33. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/cluster_tools.egg-info/top_level.txt +0 -0
  34. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/setup.cfg +0 -0
  35. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_all.py +0 -0
  36. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_dask.py +0 -0
  37. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_deref_main.py +0 -0
  38. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_kubernetes.py +0 -0
  39. {cluster_tools-2.4.4 → cluster_tools-2.4.6}/tests/test_multiprocessing.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 2.4.4
3
+ Version: 2.4.6
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -10,10 +10,10 @@ Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: typing-extensions~=4.12.0
13
- Provides-Extra: kubernetes
14
- Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
15
13
  Provides-Extra: dask
16
- Requires-Dist: kubernetes~=27.2.0; extra == "dask"
14
+ Requires-Dist: distributed~=2023.9.1; extra == "dask"
15
+ Provides-Extra: kubernetes
16
+ Requires-Dist: kubernetes~=27.2.0; extra == "kubernetes"
17
17
  Provides-Extra: all
18
18
  Requires-Dist: cluster_tools[kubernetes]; extra == "all"
19
19
  Requires-Dist: cluster_tools[dask]; extra == "all"
@@ -9,13 +9,13 @@ from collections.abc import Callable, Iterable, Iterator
9
9
  from concurrent import futures
10
10
  from concurrent.futures import Future
11
11
  from functools import partial
12
+ from types import FrameType, TracebackType
12
13
  from typing import (
13
14
  Any,
14
15
  Literal,
15
16
  TypeVar,
16
17
  cast,
17
18
  )
18
- from weakref import ReferenceType, ref
19
19
 
20
20
  from typing_extensions import ParamSpec
21
21
 
@@ -38,18 +38,6 @@ _P = ParamSpec("_P")
38
38
  _S = TypeVar("_S")
39
39
 
40
40
 
41
- def _handle_kill_through_weakref(
42
- executor_ref: "ReferenceType[ClusterExecutor]",
43
- existing_sigint_handler: Any,
44
- signum: int | None,
45
- frame: Any,
46
- ) -> None:
47
- executor = executor_ref()
48
- if executor is None:
49
- return
50
- executor.handle_kill(existing_sigint_handler, signum, frame)
51
-
52
-
53
41
  def join_messages(strings: list[str]) -> str:
54
42
  return " ".join(x.strip() for x in strings if x.strip())
55
43
 
@@ -79,6 +67,9 @@ class RemoteTimeLimitException(RemoteResourceLimitException):
79
67
  class ClusterExecutor(futures.Executor):
80
68
  """Futures executor for executing jobs on a cluster."""
81
69
 
70
+ _shutdown_hooks: list[Callable[[], None]] = []
71
+ _installed_signal_handler: bool = False
72
+
82
73
  def __init__(
83
74
  self,
84
75
  debug: bool = False,
@@ -103,7 +94,6 @@ class ClusterExecutor(futures.Executor):
103
94
  self.job_resources = job_resources
104
95
  self.additional_setup_lines = additional_setup_lines or []
105
96
  self.job_name = job_name
106
- self.was_requested_to_shutdown = False
107
97
  self.cfut_dir = (
108
98
  cfut_dir if cfut_dir is not None else os.getenv("CFUT_DIR", ".cfut")
109
99
  )
@@ -130,15 +120,7 @@ class ClusterExecutor(futures.Executor):
130
120
 
131
121
  os.makedirs(self.cfut_dir, exist_ok=True)
132
122
 
133
- # Clean up if a SIGINT signal is received. However, do not interfere with the
134
- # existing signal handler of the process or the
135
- # shutdown of the main process which sends SIGTERM signals to terminate all
136
- # child processes.
137
- existing_sigint_handler = signal.getsignal(signal.SIGINT)
138
- signal.signal(
139
- signal.SIGINT,
140
- partial(_handle_kill_through_weakref, ref(self), existing_sigint_handler),
141
- )
123
+ self._register_shutdown_hook(self.handle_kill)
142
124
 
143
125
  self.metadata = {}
144
126
  assert not ("logging_config" in kwargs and "logging_setup_fn" in kwargs), (
@@ -158,26 +140,81 @@ class ClusterExecutor(futures.Executor):
158
140
  def executor_key(cls) -> str:
159
141
  pass
160
142
 
161
- def handle_kill(
162
- self, existing_sigint_handler: Any, signum: int | None, frame: Any
143
+ @classmethod
144
+ def _ensure_signal_handlers_are_installed(cls) -> None:
145
+ # Only overwrite the signal handler once
146
+ if cls._installed_signal_handler:
147
+ return
148
+
149
+ # Clean up if a SIGINT or SIGTERM signal is received. However, do not
150
+ # interfere with the existing signal handler of the process and execute
151
+ # it afterwards.
152
+ existing_sigint_handler = signal.getsignal(signal.SIGINT)
153
+ signal.signal(
154
+ signal.SIGINT,
155
+ partial(cls._handle_shutdown, existing_sigint_handler),
156
+ )
157
+ existing_sigterm_handler = signal.getsignal(signal.SIGTERM)
158
+ signal.signal(
159
+ signal.SIGTERM,
160
+ partial(cls._handle_shutdown, existing_sigterm_handler),
161
+ )
162
+
163
+ cls._installed_signal_handler = True
164
+
165
+ @classmethod
166
+ def _register_shutdown_hook(cls, hook: Callable[[], None]) -> None:
167
+ cls._shutdown_hooks.append(hook)
168
+ cls._ensure_signal_handlers_are_installed()
169
+
170
+ @classmethod
171
+ def _deregister_shutdown_hook(cls, hook: Callable[[], None]) -> None:
172
+ if hook in cls._shutdown_hooks:
173
+ cls._shutdown_hooks.remove(hook)
174
+ else:
175
+ logging.warning(
176
+ "Cannot deregister executors shutdown hook since it's not registered."
177
+ )
178
+
179
+ @classmethod
180
+ def _handle_shutdown(
181
+ cls,
182
+ existing_signal_handler: Callable[[int, FrameType | None], None] | int | None,
183
+ signum: int,
184
+ frame: Any,
163
185
  ) -> None:
186
+ logging.critical(
187
+ f"[{cls.__name__}] Caught signal {signal.Signals(signum).name}, running shutdown hooks"
188
+ )
189
+ try:
190
+ for hook in cls._shutdown_hooks:
191
+ hook()
192
+ except Exception as e:
193
+ print(f"Error during shutdown: {e}")
194
+
195
+ if (
196
+ callable(existing_signal_handler)
197
+ and existing_signal_handler
198
+ not in (
199
+ signal.SIG_DFL, # For completeness sake (since it's not callable anyways). The system's default signal handler
200
+ signal.SIG_IGN, # For completeness sake (since it's not callable anyways). The instruction to ignore a signal
201
+ signal.default_int_handler, # Python's default SIGINT handler
202
+ )
203
+ ):
204
+ existing_signal_handler(signum, frame)
205
+
206
+ def handle_kill(self) -> None:
164
207
  if self.is_shutting_down:
165
208
  return
166
209
 
167
210
  self.is_shutting_down = True
168
211
 
169
- self.inner_handle_kill(signum, frame)
170
212
  self.wait_thread.stop()
213
+ self.inner_handle_kill()
171
214
  self.clean_up()
172
215
 
173
- if (
174
- existing_sigint_handler != signal.default_int_handler
175
- and callable(existing_sigint_handler) # Could also be signal.SIG_IGN
176
- ):
177
- existing_sigint_handler(signum, frame)
178
-
179
216
  @abstractmethod
180
- def inner_handle_kill(self, _signum: Any, _frame: Any) -> None:
217
+ def inner_handle_kill(self) -> None:
181
218
  pass
182
219
 
183
220
  @abstractmethod
@@ -363,9 +400,9 @@ class ClusterExecutor(futures.Executor):
363
400
  self._maybe_mark_logs_for_cleanup(jobid)
364
401
 
365
402
  def ensure_not_shutdown(self) -> None:
366
- if self.was_requested_to_shutdown:
403
+ if self.is_shutting_down:
367
404
  raise RuntimeError(
368
- "submit() was invoked on a ClusterExecutor instance even though shutdown() was executed for that instance."
405
+ "submit() was invoked on a ClusterExecutor instance even though shutdown() or handle_kill() was executed for that instance."
369
406
  )
370
407
 
371
408
  def create_enriched_future(self) -> Future:
@@ -591,17 +628,35 @@ class ClusterExecutor(futures.Executor):
591
628
  should_keep_output,
592
629
  )
593
630
 
631
+ # Overwrite the context manager __exit as it doesn't forward the information whether an exception was thrown or not otherwise
632
+ # which may lead to a deadlock if an exception is thrown within a cluster executor with statement, because self.jobs_empty_cond.wait()
633
+ # never succeeds.
634
+ def __exit__(
635
+ self,
636
+ exc_type: type[BaseException] | None,
637
+ _exc_val: BaseException | None,
638
+ _exc_tb: TracebackType | None,
639
+ ) -> Literal[False]:
640
+ # Don't wait if an exception was thrown
641
+ self.shutdown(wait=exc_type is None)
642
+ return False
643
+
594
644
  def shutdown(self, wait: bool = True, cancel_futures: bool = True) -> None:
595
645
  """Close the pool."""
646
+ if self.is_shutting_down:
647
+ return
648
+
649
+ self.is_shutting_down = True
596
650
  if not cancel_futures:
597
651
  logging.warning(
598
652
  "The provided cancel_futures argument is ignored by ClusterExecutor."
599
653
  )
600
- self.was_requested_to_shutdown = True
601
654
  if wait:
602
655
  with self.jobs_lock:
603
656
  if self.jobs and self.wait_thread.is_alive():
604
657
  self.jobs_empty_cond.wait()
658
+ else:
659
+ self.inner_handle_kill()
605
660
 
606
661
  self.wait_thread.stop()
607
662
  self.wait_thread.join()
@@ -617,6 +672,7 @@ class ClusterExecutor(futures.Executor):
617
672
  f"Could not delete file during clean up. Path: {file_to_clean_up} Exception: {exc}. Continuing..."
618
673
  )
619
674
  self.files_to_clean_up = []
675
+ self._deregister_shutdown_hook(self.handle_kill)
620
676
 
621
677
  def map( # type: ignore[override]
622
678
  self,
@@ -113,11 +113,7 @@ class KubernetesExecutor(ClusterExecutor):
113
113
  return job_id
114
114
  return cls.get_jobid_with_index(job_id, job_index)
115
115
 
116
- def inner_handle_kill(
117
- self,
118
- *args: Any, # noqa: ARG002 Unused method argument: `args`
119
- **kwargs: Any, # noqa: ARG002 Unused method argument: `kwargs`
120
- ) -> None:
116
+ def inner_handle_kill(self) -> None:
121
117
  job_ids = ",".join(str(job_id) for job_id in self.jobs.keys())
122
118
 
123
119
  print(
@@ -4,7 +4,7 @@ import logging
4
4
  import os
5
5
  import re
6
6
  from concurrent.futures import Future
7
- from typing import Any, Literal
7
+ from typing import Literal
8
8
 
9
9
  from cluster_tools._utils.call import call, chcall
10
10
  from cluster_tools._utils.string_ import random_string
@@ -56,7 +56,7 @@ class PBSExecutor(ClusterExecutor):
56
56
  def get_job_id_string(cls) -> str:
57
57
  return cls.get_current_job_id()
58
58
 
59
- def inner_handle_kill(self, *args: Any, **kwargs: Any) -> None: # noqa: ARG002 Unused method argument: `args`, kwargs
59
+ def inner_handle_kill(self) -> None:
60
60
  scheduled_job_ids: list[int | str] = list(self.jobs.keys())
61
61
 
62
62
  if len(scheduled_job_ids):
@@ -244,7 +244,7 @@ class SlurmExecutor(ClusterExecutor):
244
244
 
245
245
  return str(int(job_id)) # int() ensures coherent parsing
246
246
 
247
- def inner_handle_kill(self, *args: Any, **kwargs: Any) -> None: # noqa ARG002 Unused method argument: `args`, kwargs
247
+ def inner_handle_kill(self) -> None:
248
248
  for submit_thread in self.submit_threads:
249
249
  submit_thread.stop()
250
250
 
@@ -260,10 +260,15 @@ class SlurmExecutor(ClusterExecutor):
260
260
  # but can be canceled together using the job_id.
261
261
  unique_job_ids = set(map(lambda x: str(x).split("_")[0], scheduled_job_ids))
262
262
  job_id_string = " ".join(unique_job_ids)
263
+ # Allow to speed up the shutdown, for example, when running voxelytics locally
264
+ sigterm_wait_in_s_env = float(os.environ.get("SIGTERM_WAIT_IN_S", 5))
263
265
  # Send SIGINT signal to running jobs instead of terminating the jobs right away. This way, the jobs can
264
266
  # react to the signal, safely shutdown and signal (cancel) jobs they possibly scheduled, recursively.
267
+ # After a short waiting time kill all jobs that are still running (due to race conditions or because they
268
+ # didn't react to the SIGINT signal for some reason).
265
269
  _, stderr, _ = call(
266
- f"scancel --state=PENDING {job_id_string}; scancel -s SIGINT --state=RUNNING {job_id_string}; scancel --state=SUSPENDED {job_id_string}"
270
+ f"scancel --state=PENDING {job_id_string}; scancel -s SIGINT --state=RUNNING {job_id_string};"
271
+ + f"scancel --state=SUSPENDED {job_id_string}; sleep {sigterm_wait_in_s_env}; scancel {job_id_string}"
267
272
  )
268
273
 
269
274
  maybe_error_or_warning = (
@@ -0,0 +1 @@
1
+ __version__ = '2.4.6'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 2.4.4
3
+ Version: 2.4.6
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -10,10 +10,10 @@ Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: typing-extensions~=4.12.0
13
- Provides-Extra: kubernetes
14
- Requires-Dist: distributed~=2023.9.1; extra == "kubernetes"
15
13
  Provides-Extra: dask
16
- Requires-Dist: kubernetes~=27.2.0; extra == "dask"
14
+ Requires-Dist: distributed~=2023.9.1; extra == "dask"
15
+ Provides-Extra: kubernetes
16
+ Requires-Dist: kubernetes~=27.2.0; extra == "kubernetes"
17
17
  Provides-Extra: all
18
18
  Requires-Dist: cluster_tools[kubernetes]; extra == "all"
19
19
  Requires-Dist: cluster_tools[dask]; extra == "all"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cluster_tools"
3
- version = "2.4.4" # filled by dunamai
3
+ version = "2.4.6" # filled by dunamai
4
4
  description = "Utility library for easily distributing code execution on clusters"
5
5
  authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
6
6
  readme = "README.md"
@@ -17,8 +17,8 @@ Changelog = "https://github.com/scalableminds/webknossos-libs/blob/master/cluste
17
17
 
18
18
 
19
19
  [project.optional-dependencies]
20
- kubernetes = ["distributed ~=2023.9.1"]
21
- dask = ["kubernetes ~=27.2.0"]
20
+ dask = ["distributed ~=2023.9.1"]
21
+ kubernetes = ["kubernetes ~=27.2.0"]
22
22
  all=["cluster_tools[kubernetes]", "cluster_tools[dask]"]
23
23
 
24
24
  [tool.uv]
@@ -26,6 +26,7 @@ dev-dependencies = [
26
26
  "icecream ~=2.1.1",
27
27
  "mypy ~=1.15.0",
28
28
  "pytest ~=8.3.3",
29
+ "pytest-timeout>=2.4.0",
29
30
  "ruff ~=0.11.0",
30
31
  ]
31
32
 
@@ -1,5 +1,6 @@
1
1
  import concurrent.futures
2
2
  import contextlib
3
+ import gc
3
4
  import io
4
5
  import logging
5
6
  import multiprocessing as mp
@@ -168,10 +169,12 @@ def wait_until_first_job_was_submitted(
168
169
  time.sleep(0.1)
169
170
 
170
171
 
171
- def test_slurm_deferred_submit_shutdown() -> None:
172
+ @pytest.mark.timeout(30)
173
+ def test_slurm_deferred_submit_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
172
174
  # Test that the SlurmExecutor stops scheduling jobs in a separate thread
173
175
  # once it was killed even if the executor was used multiple times and
174
176
  # therefore started multiple job submission threads
177
+ monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
175
178
  max_submit_jobs = 1
176
179
 
177
180
  # Only one job can be scheduled at a time
@@ -179,43 +182,48 @@ def test_slurm_deferred_submit_shutdown() -> None:
179
182
  executor = cluster_tools.get_executor("slurm", debug=True)
180
183
 
181
184
  try:
182
- # Use the executor twice to start multiple job submission threads
183
- executor.map_to_futures(sleep, [0.5] * 10)
184
- executor.map_to_futures(sleep, [0.5] * 10)
185
+ with executor:
186
+ # Use the executor twice to start multiple job submission threads
187
+ executor.map_to_futures(sleep, [0.5] * 10)
188
+ executor.map_to_futures(sleep, [0.5] * 10)
185
189
 
186
- wait_until_first_job_was_submitted(executor)
190
+ wait_until_first_job_was_submitted(executor)
187
191
 
188
- for submit_thread in executor.submit_threads:
189
- assert submit_thread.is_alive()
192
+ for submit_thread in executor.submit_threads:
193
+ assert submit_thread.is_alive()
190
194
 
191
- executor.handle_kill(signal.default_int_handler, None, None)
195
+ sigint_handler = signal.getsignal(signal.SIGINT)
196
+ assert callable(sigint_handler) # Mainly for typechecking
197
+ sigint_handler(signal.SIGINT, None)
192
198
 
193
- # Wait for the threads to die down, but less than it would take to submit all jobs
194
- # which would take ~5 seconds since only one job is scheduled at a time
195
- for submit_thread in executor.submit_threads:
196
- submit_thread.join(1)
197
- assert not submit_thread.is_alive()
199
+ # Wait for the threads to die down, but less than it would take to submit all jobs
200
+ # which would take ~5 seconds since only one job is scheduled at a time
201
+ for submit_thread in executor.submit_threads:
202
+ submit_thread.join(1)
203
+ assert not submit_thread.is_alive()
198
204
 
199
- # Wait for scheduled jobs to be canceled, so that the queue is empty again
200
- while executor.get_number_of_submitted_jobs() > 0:
201
- time.sleep(0.5)
205
+ # Wait for scheduled jobs to be canceled, so that the queue is empty again
206
+ while executor.get_number_of_submitted_jobs() > 0:
207
+ time.sleep(0.5)
202
208
 
203
209
  finally:
204
210
  call("echo y | sacctmgr modify qos normal set MaxSubmitJobs=-1")
205
211
 
206
212
 
207
- def test_slurm_job_canceling_on_shutdown() -> None:
213
+ @pytest.mark.timeout(30)
214
+ def test_slurm_job_canceling_on_shutdown(
215
+ monkeypatch: pytest.MonkeyPatch,
216
+ ) -> None:
208
217
  # Test that scheduled jobs are canceled on shutdown, regardless
209
218
  # of whether they are pending or running.
210
- max_running_size = 2
219
+ monkeypatch.setenv("SLURM_MAX_RUNNING_SIZE", "2")
220
+ monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
211
221
 
212
222
  executor = cluster_tools.get_executor("slurm", debug=True)
213
- # Only two jobs can run at once, so that some of the jobs will be
214
- # running and some will be pending.
215
- original_max_running_size = os.environ.get("SLURM_MAX_RUNNING_SIZE")
216
- os.environ["SLURM_MAX_RUNNING_SIZE"] = str(max_running_size)
217
223
 
218
- try:
224
+ with executor:
225
+ # Only two jobs can run at once, so that some of the jobs will be
226
+ # running and some will be pending.
219
227
  executor.map_to_futures(sleep, [10] * 4)
220
228
 
221
229
  # Wait until first job is running
@@ -223,18 +231,77 @@ def test_slurm_job_canceling_on_shutdown() -> None:
223
231
 
224
232
  job_start_time = time.time()
225
233
 
226
- # The job cancellation is flaky if jobs just switched from PENDING to RUNNING,
227
- # probably because Python is not yet able to react to the SIGINT signal.
228
- # This is an unsolved issue as of now.
229
- time.sleep(1)
230
-
231
- executor.handle_kill(signal.default_int_handler, None, None)
234
+ sigint_handler = signal.getsignal(signal.SIGINT)
235
+ assert callable(sigint_handler) # Mainly for typechecking
236
+ sigint_handler(signal.SIGINT, None)
232
237
 
233
238
  # Wait for scheduled jobs to be canceled, so that the queue is empty again
234
239
  # and measure how long the cancellation takes
235
240
  while executor.get_number_of_submitted_jobs() > 0:
236
241
  time.sleep(0.5)
237
242
 
243
+ job_cancellation_duration = time.time() - job_start_time
244
+
245
+ # Killing the executor should have canceled all submitted jobs, regardless
246
+ # of whether they were running or pending in much less time than it would
247
+ # have taken the jobs to finish on their own
248
+ assert job_cancellation_duration < 5
249
+
250
+
251
+ @pytest.mark.timeout(30)
252
+ def test_slurm_signal_handling(
253
+ monkeypatch: pytest.MonkeyPatch,
254
+ ) -> None:
255
+ monkeypatch.setenv("SIGTERM_WAIT_IN_S", "0")
256
+ original_sigint_handler_was_called = False
257
+
258
+ def original_sigint_handler(_signum: int | None, _frame: Any) -> None:
259
+ nonlocal original_sigint_handler_was_called
260
+ original_sigint_handler_was_called = True
261
+
262
+ signal.signal(
263
+ signal.SIGINT,
264
+ original_sigint_handler,
265
+ )
266
+
267
+ assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 0
268
+ # Reset class variable after changing signal handler so it will be set again when the next cluster executor is instantiated
269
+ cluster_tools.SlurmExecutor._installed_signal_handler = False
270
+
271
+ with cluster_tools.get_executor("slurm", debug=True) as executor1:
272
+ assert executor1.get_number_of_submitted_jobs() == 0
273
+ assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 1
274
+
275
+ futures = executor1.map_to_futures(square, [2])
276
+ concurrent.futures.wait(futures)
277
+
278
+ # Let the first executor be no longer referenced to provoke potential bugs in the signal handler chaining
279
+ # See https://github.com/scalableminds/webknossos-libs/pull/1317
280
+ del executor1
281
+ gc.collect()
282
+
283
+ with cluster_tools.get_executor("slurm", debug=True) as executor2:
284
+ assert executor2.get_number_of_submitted_jobs() == 0
285
+ assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 1
286
+
287
+ executor2.map_to_futures(sleep, [10] * 4)
288
+
289
+ # Wait until first job is running
290
+ wait_until_first_job_was_submitted(executor2, "RUNNING")
291
+
292
+ job_start_time = time.time()
293
+
294
+ sigint_handler = signal.getsignal(signal.SIGINT)
295
+ assert callable(sigint_handler) # Mainly for typechecking
296
+ sigint_handler(signal.SIGINT, None)
297
+
298
+ assert original_sigint_handler_was_called
299
+
300
+ # Wait for scheduled jobs to be canceled, so that the queue is empty again
301
+ # and measure how long the cancellation takes
302
+ while executor2.get_number_of_submitted_jobs() > 0:
303
+ time.sleep(0.5)
304
+
238
305
  job_cancellation_duration = time.time() - job_start_time
239
306
 
240
307
  # Killing the executor should have canceled all submitted jobs, regardless
@@ -242,11 +309,9 @@ def test_slurm_job_canceling_on_shutdown() -> None:
242
309
  # have taken the jobs to finish on their own
243
310
  assert job_cancellation_duration < 5
244
311
 
245
- finally:
246
- if original_max_running_size is not None:
247
- os.environ["SLURM_MAX_RUNNING_SIZE"] = original_max_running_size
248
- else:
249
- del os.environ["SLURM_MAX_RUNNING_SIZE"]
312
+ assert len(cluster_tools.SlurmExecutor._shutdown_hooks) == 0, (
313
+ "The cluster executor shutdown hooks should have been cleaned up"
314
+ )
250
315
 
251
316
 
252
317
  def test_slurm_number_of_submitted_jobs() -> None:
@@ -1 +0,0 @@
1
- __version__ = '2.4.4'
File without changes
File without changes
@@ -5,7 +5,7 @@ cluster_tools[kubernetes]
5
5
  cluster_tools[dask]
6
6
 
7
7
  [dask]
8
- kubernetes~=27.2.0
8
+ distributed~=2023.9.1
9
9
 
10
10
  [kubernetes]
11
- distributed~=2023.9.1
11
+ kubernetes~=27.2.0
File without changes