cluster-tools 3.5.0__tar.gz → 3.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/PKG-INFO +1 -1
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/multiprocessing_logging_handler.py +55 -30
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/cluster_executor.py +65 -8
- cluster_tools-3.5.2/cluster_tools/version.py +1 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/PKG-INFO +1 -1
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/pyproject.toml +1 -1
- cluster_tools-3.5.0/cluster_tools/version.py +0 -1
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/LICENSE +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/README.md +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/__init__.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/call.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/file_wait_thread.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/pickling.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/reflection.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/string_.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/tailf.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/warning.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executor_protocol.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/batching.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/dask.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/multiprocessing_.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/sequential.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/sequential_pickle.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/remote.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/__init__.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/kube.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/pbs.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/slurm.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/SOURCES.txt +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/dependency_links.txt +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/requires.txt +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/top_level.txt +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/setup.cfg +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_all.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_batching.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_dask.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_deref_main.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_kubernetes.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_multiprocessing.py +0 -0
- {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_slurm.py +0 -0
{cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/multiprocessing_logging_handler.py
RENAMED
|
@@ -15,6 +15,10 @@ from typing import Any
|
|
|
15
15
|
|
|
16
16
|
# Inspired by https://stackoverflow.com/a/894284
|
|
17
17
|
|
|
18
|
+
# Since the root logger object is global, accessing it from multiple threads can be problematic
|
|
19
|
+
# To be thread safe, we use a lock.
|
|
20
|
+
LOGGER_LOCK = threading.Lock()
|
|
21
|
+
|
|
18
22
|
|
|
19
23
|
class _MultiprocessingLoggingHandler(logging.Handler):
|
|
20
24
|
"""This class wraps a logging handler and instantiates a multiprocessing queue.
|
|
@@ -84,17 +88,22 @@ class _MultiprocessingLoggingHandler(logging.Handler):
|
|
|
84
88
|
self._usage_counter += 1
|
|
85
89
|
|
|
86
90
|
def decrement_usage(self) -> None:
|
|
87
|
-
|
|
91
|
+
with LOGGER_LOCK:
|
|
92
|
+
self._usage_counter -= 1
|
|
93
|
+
if self._usage_counter == 0:
|
|
94
|
+
# unwrap inner handler:
|
|
95
|
+
root_logger = getLogger()
|
|
96
|
+
root_logger.removeHandler(self)
|
|
97
|
+
root_logger.addHandler(self.wrapped_handler)
|
|
98
|
+
|
|
99
|
+
# Thread-owned handlers (e.g. task-specific file handlers) are closed by their
|
|
100
|
+
# owner (attach_logging_handler), not by the pool.
|
|
101
|
+
if not getattr(self.wrapped_handler, "_owner_thread_id", None):
|
|
102
|
+
self.wrapped_handler.close()
|
|
88
103
|
if self._usage_counter == 0:
|
|
89
|
-
# unwrap inner handler:
|
|
90
|
-
root_logger = getLogger()
|
|
91
|
-
root_logger.removeHandler(self)
|
|
92
|
-
root_logger.addHandler(self.wrapped_handler)
|
|
93
|
-
|
|
94
104
|
self._is_closed = True
|
|
95
105
|
self._queue_thread.join(30)
|
|
96
106
|
self._manager.shutdown()
|
|
97
|
-
self.wrapped_handler.close()
|
|
98
107
|
super().close()
|
|
99
108
|
|
|
100
109
|
def close(self) -> None:
|
|
@@ -115,35 +124,51 @@ def _setup_logging_multiprocessing(
|
|
|
115
124
|
"""
|
|
116
125
|
warnings.filters = filters
|
|
117
126
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
root_logger.
|
|
127
|
+
with LOGGER_LOCK:
|
|
128
|
+
root_logger = getLogger()
|
|
129
|
+
for handler in root_logger.handlers:
|
|
130
|
+
root_logger.removeHandler(handler)
|
|
121
131
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
132
|
+
root_logger.setLevel(min(levels) if len(levels) else logging.DEBUG)
|
|
133
|
+
for queue, level in zip(queues, levels):
|
|
134
|
+
handler = QueueHandler(queue)
|
|
135
|
+
handler.setLevel(level)
|
|
136
|
+
root_logger.addHandler(handler)
|
|
127
137
|
|
|
128
138
|
|
|
129
139
|
class _MultiprocessingLoggingHandlerPool:
|
|
130
140
|
def __init__(self) -> None:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
141
|
+
with LOGGER_LOCK:
|
|
142
|
+
root_logger = getLogger()
|
|
143
|
+
current_thread_id = threading.get_ident()
|
|
144
|
+
|
|
145
|
+
self.handlers = []
|
|
146
|
+
for i, handler in enumerate(list(root_logger.handlers)):
|
|
147
|
+
# Resolve the underlying handler to check for thread ownership.
|
|
148
|
+
# An already-wrapped handler exposes its original via .wrapped_handler.
|
|
149
|
+
underlying = (
|
|
150
|
+
handler.wrapped_handler
|
|
151
|
+
if isinstance(handler, _MultiprocessingLoggingHandler)
|
|
152
|
+
else handler
|
|
140
153
|
)
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
154
|
+
owner_thread = getattr(underlying, "_owner_thread_id", None)
|
|
155
|
+
# Skip handlers owned by a different thread: relevant when starting multiprocessing from multiple threads
|
|
156
|
+
# where each thread should only handle the logging for its own multiprocessing executor
|
|
157
|
+
if owner_thread is not None and owner_thread != current_thread_id:
|
|
158
|
+
continue
|
|
159
|
+
|
|
160
|
+
# Wrap logging handlers in _MultiprocessingLoggingHandlers to make them work in a multiprocessing setup
|
|
161
|
+
# when using start_methods other than fork, for example, spawn or forkserver
|
|
162
|
+
if not isinstance(handler, _MultiprocessingLoggingHandler):
|
|
163
|
+
mp_handler = _MultiprocessingLoggingHandler(
|
|
164
|
+
f"multi-processing-handler-{i}", handler
|
|
165
|
+
)
|
|
166
|
+
root_logger.removeHandler(handler)
|
|
167
|
+
root_logger.addHandler(mp_handler)
|
|
168
|
+
self.handlers.append(mp_handler)
|
|
169
|
+
else:
|
|
170
|
+
handler.increment_usage()
|
|
171
|
+
self.handlers.append(handler)
|
|
147
172
|
|
|
148
173
|
def get_multiprocessing_logging_setup_fn(self) -> Callable[[], None]:
|
|
149
174
|
# Return a logging setup function that when called will setup QueueHandler loggers
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import atexit
|
|
1
2
|
import logging
|
|
2
3
|
import os
|
|
3
4
|
import signal
|
|
@@ -69,6 +70,10 @@ class ClusterExecutor(futures.Executor):
|
|
|
69
70
|
|
|
70
71
|
_shutdown_hooks: list[Callable[[], None]] = []
|
|
71
72
|
_installed_signal_handler: bool = False
|
|
73
|
+
_installed_atexit_handler: bool = False
|
|
74
|
+
_shutdown_hooks_ran: bool = False
|
|
75
|
+
# We keep a lock to guard critical parts against race conditions when being called from multiple threads
|
|
76
|
+
_state_lock: threading.Lock = threading.Lock()
|
|
72
77
|
|
|
73
78
|
def __init__(
|
|
74
79
|
self,
|
|
@@ -142,9 +147,25 @@ class ClusterExecutor(futures.Executor):
|
|
|
142
147
|
|
|
143
148
|
@classmethod
|
|
144
149
|
def _ensure_signal_handlers_are_installed(cls) -> None:
|
|
145
|
-
|
|
150
|
+
with cls._state_lock:
|
|
151
|
+
if not cls._installed_atexit_handler:
|
|
152
|
+
atexit.register(cls._run_shutdown_hooks)
|
|
153
|
+
cls._installed_atexit_handler = True
|
|
154
|
+
|
|
155
|
+
# signal.signal() only works from the main thread. If we're on a worker
|
|
156
|
+
# thread, skip but don't mark as installed — a later main-thread
|
|
157
|
+
# instantiation will still be able to install the handlers.
|
|
146
158
|
if cls._installed_signal_handler:
|
|
147
159
|
return
|
|
160
|
+
if threading.current_thread() is not threading.main_thread():
|
|
161
|
+
logging.warning(
|
|
162
|
+
f"[{cls.__name__}] Cannot install signal handlers because the executor "
|
|
163
|
+
"was instantiated from a non-main thread. Cleanup on SIGTERM will not "
|
|
164
|
+
"work; SIGINT and normal exits are covered via atexit. "
|
|
165
|
+
f"Call {cls.__name__}.install_signal_handlers() from the main thread "
|
|
166
|
+
"at startup to enable full signal handling."
|
|
167
|
+
)
|
|
168
|
+
return
|
|
148
169
|
|
|
149
170
|
# Clean up if a SIGINT or SIGTERM signal is received. However, do not
|
|
150
171
|
# interfere with the existing signal handler of the process and execute
|
|
@@ -162,9 +183,32 @@ class ClusterExecutor(futures.Executor):
|
|
|
162
183
|
|
|
163
184
|
cls._installed_signal_handler = True
|
|
164
185
|
|
|
186
|
+
@classmethod
|
|
187
|
+
def install_signal_handlers(cls) -> None:
|
|
188
|
+
"""Install SIGINT/SIGTERM handlers for cluster job cleanup.
|
|
189
|
+
|
|
190
|
+
Must be called from the main thread. Call this once at program startup
|
|
191
|
+
before using ClusterExecutor from worker threads to ensure that
|
|
192
|
+
SIGINT and SIGTERM trigger proper job cancellation and cleanup.
|
|
193
|
+
"""
|
|
194
|
+
if threading.current_thread() is not threading.main_thread():
|
|
195
|
+
raise RuntimeError(
|
|
196
|
+
f"{cls.__name__}.install_signal_handlers() must be called from the main thread."
|
|
197
|
+
)
|
|
198
|
+
cls._ensure_signal_handlers_are_installed()
|
|
199
|
+
|
|
165
200
|
@classmethod
|
|
166
201
|
def _register_shutdown_hook(cls, hook: Callable[[], None]) -> None:
|
|
167
|
-
cls.
|
|
202
|
+
with cls._state_lock:
|
|
203
|
+
cls._shutdown_hooks.append(hook)
|
|
204
|
+
# New active executors exist; allow hooks to run again if signaled.
|
|
205
|
+
# This handles the case where a previous signal already ran hooks
|
|
206
|
+
# (setting _shutdown_hooks_ran=True) but new executors are now active.
|
|
207
|
+
# IMPORTANT: This should not happen during normal execution!
|
|
208
|
+
# However, it might happen during tests (since multiple tests run in the same process).
|
|
209
|
+
# So we keep it to allow the tests to work and also to make the code more robust against weird edge cases
|
|
210
|
+
# where new shutdown hooks are registered after a first shutdown was already triggered.
|
|
211
|
+
cls._shutdown_hooks_ran = False
|
|
168
212
|
cls._ensure_signal_handlers_are_installed()
|
|
169
213
|
|
|
170
214
|
@classmethod
|
|
@@ -176,6 +220,23 @@ class ClusterExecutor(futures.Executor):
|
|
|
176
220
|
"Cannot deregister executors shutdown hook since it's not registered."
|
|
177
221
|
)
|
|
178
222
|
|
|
223
|
+
@classmethod
|
|
224
|
+
def _run_shutdown_hooks(cls) -> None:
|
|
225
|
+
# Make sure shutdown hooks run only once
|
|
226
|
+
with cls._state_lock:
|
|
227
|
+
if cls._shutdown_hooks_ran:
|
|
228
|
+
return
|
|
229
|
+
cls._shutdown_hooks_ran = True
|
|
230
|
+
# Copying the list of hooks since hooks might unregister themselves when executed.
|
|
231
|
+
# Without the copy, this would cause the cls._shutdown_hooks list to be modified
|
|
232
|
+
# while we iterate over it which is dangerous!
|
|
233
|
+
hooks = list(cls._shutdown_hooks)
|
|
234
|
+
for hook in hooks:
|
|
235
|
+
try:
|
|
236
|
+
hook()
|
|
237
|
+
except Exception as e: # noqa: PERF203
|
|
238
|
+
logging.error(f"Error during running shutdown hook {hook}: {e}")
|
|
239
|
+
|
|
179
240
|
@classmethod
|
|
180
241
|
def _handle_shutdown(
|
|
181
242
|
cls,
|
|
@@ -184,13 +245,9 @@ class ClusterExecutor(futures.Executor):
|
|
|
184
245
|
frame: Any,
|
|
185
246
|
) -> None:
|
|
186
247
|
logging.critical(
|
|
187
|
-
f"[{cls.__name__}] Caught signal {signal.Signals(signum).name},
|
|
248
|
+
f"[{cls.__name__}] Caught signal {signal.Signals(signum).name}, starting shutdown hooks"
|
|
188
249
|
)
|
|
189
|
-
|
|
190
|
-
for hook in cls._shutdown_hooks:
|
|
191
|
-
hook()
|
|
192
|
-
except Exception as e:
|
|
193
|
-
print(f"Error during shutdown: {e}")
|
|
250
|
+
cls._run_shutdown_hooks()
|
|
194
251
|
|
|
195
252
|
if (
|
|
196
253
|
callable(existing_signal_handler)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '3.5.2'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cluster_tools"
|
|
3
|
-
version = "3.5.
|
|
3
|
+
version = "3.5.2" # filled by dunamai
|
|
4
4
|
description = "Utility library for easily distributing code execution on clusters"
|
|
5
5
|
authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
|
|
6
6
|
readme = "README.md"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '3.5.0'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/multiprocessing_pickle.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|