cluster-tools 3.5.0__tar.gz → 3.5.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/PKG-INFO +1 -1
  2. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/multiprocessing_logging_handler.py +55 -30
  3. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/cluster_executor.py +65 -8
  4. cluster_tools-3.5.2/cluster_tools/version.py +1 -0
  5. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/PKG-INFO +1 -1
  6. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/pyproject.toml +1 -1
  7. cluster_tools-3.5.0/cluster_tools/version.py +0 -1
  8. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/LICENSE +0 -0
  9. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/README.md +0 -0
  10. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/__init__.py +0 -0
  11. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/call.py +0 -0
  12. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/file_wait_thread.py +0 -0
  13. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/pickling.py +0 -0
  14. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/reflection.py +0 -0
  15. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/string_.py +0 -0
  16. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/tailf.py +0 -0
  17. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/_utils/warning.py +0 -0
  18. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executor_protocol.py +0 -0
  19. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/batching.py +0 -0
  20. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/dask.py +0 -0
  21. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/multiprocessing_.py +0 -0
  22. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/multiprocessing_pickle.py +0 -0
  23. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/sequential.py +0 -0
  24. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/executors/sequential_pickle.py +0 -0
  25. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/remote.py +0 -0
  26. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/__init__.py +0 -0
  27. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/kube.py +0 -0
  28. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/pbs.py +0 -0
  29. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools/schedulers/slurm.py +0 -0
  30. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/SOURCES.txt +0 -0
  31. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/dependency_links.txt +0 -0
  32. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/requires.txt +0 -0
  33. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/cluster_tools.egg-info/top_level.txt +0 -0
  34. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/setup.cfg +0 -0
  35. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_all.py +0 -0
  36. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_batching.py +0 -0
  37. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_dask.py +0 -0
  38. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_deref_main.py +0 -0
  39. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_kubernetes.py +0 -0
  40. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_multiprocessing.py +0 -0
  41. {cluster_tools-3.5.0 → cluster_tools-3.5.2}/tests/test_slurm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 3.5.0
3
+ Version: 3.5.2
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -15,6 +15,10 @@ from typing import Any
15
15
 
16
16
  # Inspired by https://stackoverflow.com/a/894284
17
17
 
18
+ # Since the root logger object is global, accessing it from multiple threads can be problematic
19
+ # To be thread safe, we use a lock.
20
+ LOGGER_LOCK = threading.Lock()
21
+
18
22
 
19
23
  class _MultiprocessingLoggingHandler(logging.Handler):
20
24
  """This class wraps a logging handler and instantiates a multiprocessing queue.
@@ -84,17 +88,22 @@ class _MultiprocessingLoggingHandler(logging.Handler):
84
88
  self._usage_counter += 1
85
89
 
86
90
  def decrement_usage(self) -> None:
87
- self._usage_counter -= 1
91
+ with LOGGER_LOCK:
92
+ self._usage_counter -= 1
93
+ if self._usage_counter == 0:
94
+ # unwrap inner handler:
95
+ root_logger = getLogger()
96
+ root_logger.removeHandler(self)
97
+ root_logger.addHandler(self.wrapped_handler)
98
+
99
+ # Thread-owned handlers (e.g. task-specific file handlers) are closed by their
100
+ # owner (attach_logging_handler), not by the pool.
101
+ if not getattr(self.wrapped_handler, "_owner_thread_id", None):
102
+ self.wrapped_handler.close()
88
103
  if self._usage_counter == 0:
89
- # unwrap inner handler:
90
- root_logger = getLogger()
91
- root_logger.removeHandler(self)
92
- root_logger.addHandler(self.wrapped_handler)
93
-
94
104
  self._is_closed = True
95
105
  self._queue_thread.join(30)
96
106
  self._manager.shutdown()
97
- self.wrapped_handler.close()
98
107
  super().close()
99
108
 
100
109
  def close(self) -> None:
@@ -115,35 +124,51 @@ def _setup_logging_multiprocessing(
115
124
  """
116
125
  warnings.filters = filters
117
126
 
118
- root_logger = getLogger()
119
- for handler in root_logger.handlers:
120
- root_logger.removeHandler(handler)
127
+ with LOGGER_LOCK:
128
+ root_logger = getLogger()
129
+ for handler in root_logger.handlers:
130
+ root_logger.removeHandler(handler)
121
131
 
122
- root_logger.setLevel(min(levels) if len(levels) else logging.DEBUG)
123
- for queue, level in zip(queues, levels):
124
- handler = QueueHandler(queue)
125
- handler.setLevel(level)
126
- root_logger.addHandler(handler)
132
+ root_logger.setLevel(min(levels) if len(levels) else logging.DEBUG)
133
+ for queue, level in zip(queues, levels):
134
+ handler = QueueHandler(queue)
135
+ handler.setLevel(level)
136
+ root_logger.addHandler(handler)
127
137
 
128
138
 
129
139
  class _MultiprocessingLoggingHandlerPool:
130
140
  def __init__(self) -> None:
131
- root_logger = getLogger()
132
-
133
- self.handlers = []
134
- for i, handler in enumerate(list(root_logger.handlers)):
135
- # Wrap logging handlers in _MultiprocessingLoggingHandlers to make them work in a multiprocessing setup
136
- # when using start_methods other than fork, for example, spawn or forkserver
137
- if not isinstance(handler, _MultiprocessingLoggingHandler):
138
- mp_handler = _MultiprocessingLoggingHandler(
139
- f"multi-processing-handler-{i}", handler
141
+ with LOGGER_LOCK:
142
+ root_logger = getLogger()
143
+ current_thread_id = threading.get_ident()
144
+
145
+ self.handlers = []
146
+ for i, handler in enumerate(list(root_logger.handlers)):
147
+ # Resolve the underlying handler to check for thread ownership.
148
+ # An already-wrapped handler exposes its original via .wrapped_handler.
149
+ underlying = (
150
+ handler.wrapped_handler
151
+ if isinstance(handler, _MultiprocessingLoggingHandler)
152
+ else handler
140
153
  )
141
- root_logger.removeHandler(handler)
142
- root_logger.addHandler(mp_handler)
143
- self.handlers.append(mp_handler)
144
- else:
145
- handler.increment_usage()
146
- self.handlers.append(handler)
154
+ owner_thread = getattr(underlying, "_owner_thread_id", None)
155
+ # Skip handlers owned by a different thread: relevant when starting multiprocessing from multiple threads
156
+ # where each thread should only handle the logging for its own multiprocessing executor
157
+ if owner_thread is not None and owner_thread != current_thread_id:
158
+ continue
159
+
160
+ # Wrap logging handlers in _MultiprocessingLoggingHandlers to make them work in a multiprocessing setup
161
+ # when using start_methods other than fork, for example, spawn or forkserver
162
+ if not isinstance(handler, _MultiprocessingLoggingHandler):
163
+ mp_handler = _MultiprocessingLoggingHandler(
164
+ f"multi-processing-handler-{i}", handler
165
+ )
166
+ root_logger.removeHandler(handler)
167
+ root_logger.addHandler(mp_handler)
168
+ self.handlers.append(mp_handler)
169
+ else:
170
+ handler.increment_usage()
171
+ self.handlers.append(handler)
147
172
 
148
173
  def get_multiprocessing_logging_setup_fn(self) -> Callable[[], None]:
149
174
  # Return a logging setup function that when called will setup QueueHandler loggers
@@ -1,3 +1,4 @@
1
+ import atexit
1
2
  import logging
2
3
  import os
3
4
  import signal
@@ -69,6 +70,10 @@ class ClusterExecutor(futures.Executor):
69
70
 
70
71
  _shutdown_hooks: list[Callable[[], None]] = []
71
72
  _installed_signal_handler: bool = False
73
+ _installed_atexit_handler: bool = False
74
+ _shutdown_hooks_ran: bool = False
75
+ # We keep a lock to guard critical parts against race conditions when being called from multiple threads
76
+ _state_lock: threading.Lock = threading.Lock()
72
77
 
73
78
  def __init__(
74
79
  self,
@@ -142,9 +147,25 @@ class ClusterExecutor(futures.Executor):
142
147
 
143
148
  @classmethod
144
149
  def _ensure_signal_handlers_are_installed(cls) -> None:
145
- # Only overwrite the signal handler once
150
+ with cls._state_lock:
151
+ if not cls._installed_atexit_handler:
152
+ atexit.register(cls._run_shutdown_hooks)
153
+ cls._installed_atexit_handler = True
154
+
155
+ # signal.signal() only works from the main thread. If we're on a worker
156
+ # thread, skip but don't mark as installed — a later main-thread
157
+ # instantiation will still be able to install the handlers.
146
158
  if cls._installed_signal_handler:
147
159
  return
160
+ if threading.current_thread() is not threading.main_thread():
161
+ logging.warning(
162
+ f"[{cls.__name__}] Cannot install signal handlers because the executor "
163
+ "was instantiated from a non-main thread. Cleanup on SIGTERM will not "
164
+ "work; SIGINT and normal exits are covered via atexit. "
165
+ f"Call {cls.__name__}.install_signal_handlers() from the main thread "
166
+ "at startup to enable full signal handling."
167
+ )
168
+ return
148
169
 
149
170
  # Clean up if a SIGINT or SIGTERM signal is received. However, do not
150
171
  # interfere with the existing signal handler of the process and execute
@@ -162,9 +183,32 @@ class ClusterExecutor(futures.Executor):
162
183
 
163
184
  cls._installed_signal_handler = True
164
185
 
186
+ @classmethod
187
+ def install_signal_handlers(cls) -> None:
188
+ """Install SIGINT/SIGTERM handlers for cluster job cleanup.
189
+
190
+ Must be called from the main thread. Call this once at program startup
191
+ before using ClusterExecutor from worker threads to ensure that
192
+ SIGINT and SIGTERM trigger proper job cancellation and cleanup.
193
+ """
194
+ if threading.current_thread() is not threading.main_thread():
195
+ raise RuntimeError(
196
+ f"{cls.__name__}.install_signal_handlers() must be called from the main thread."
197
+ )
198
+ cls._ensure_signal_handlers_are_installed()
199
+
165
200
  @classmethod
166
201
  def _register_shutdown_hook(cls, hook: Callable[[], None]) -> None:
167
- cls._shutdown_hooks.append(hook)
202
+ with cls._state_lock:
203
+ cls._shutdown_hooks.append(hook)
204
+ # New active executors exist; allow hooks to run again if signaled.
205
+ # This handles the case where a previous signal already ran hooks
206
+ # (setting _shutdown_hooks_ran=True) but new executors are now active.
207
+ # IMPORTANT: This should not happen during normal execution!
208
+ # However, it might happen during tests (since multiple tests run in the same process).
209
+ # So we keep it to allow the tests to work and also to make the code more robust against weird edge cases
210
+ # where new shutdown hooks are registered after a first shutdown was already triggered.
211
+ cls._shutdown_hooks_ran = False
168
212
  cls._ensure_signal_handlers_are_installed()
169
213
 
170
214
  @classmethod
@@ -176,6 +220,23 @@ class ClusterExecutor(futures.Executor):
176
220
  "Cannot deregister executors shutdown hook since it's not registered."
177
221
  )
178
222
 
223
+ @classmethod
224
+ def _run_shutdown_hooks(cls) -> None:
225
+ # Make sure shutdown hooks run only once
226
+ with cls._state_lock:
227
+ if cls._shutdown_hooks_ran:
228
+ return
229
+ cls._shutdown_hooks_ran = True
230
+ # Copying the list of hooks since hooks might unregister themselves when executed.
231
+ # Without the copy, this would cause the cls._shutdown_hooks list to be modified
232
+ # while we iterate over it which is dangerous!
233
+ hooks = list(cls._shutdown_hooks)
234
+ for hook in hooks:
235
+ try:
236
+ hook()
237
+ except Exception as e: # noqa: PERF203
238
+ logging.error(f"Error during running shutdown hook {hook}: {e}")
239
+
179
240
  @classmethod
180
241
  def _handle_shutdown(
181
242
  cls,
@@ -184,13 +245,9 @@ class ClusterExecutor(futures.Executor):
184
245
  frame: Any,
185
246
  ) -> None:
186
247
  logging.critical(
187
- f"[{cls.__name__}] Caught signal {signal.Signals(signum).name}, running shutdown hooks"
248
+ f"[{cls.__name__}] Caught signal {signal.Signals(signum).name}, starting shutdown hooks"
188
249
  )
189
- try:
190
- for hook in cls._shutdown_hooks:
191
- hook()
192
- except Exception as e:
193
- print(f"Error during shutdown: {e}")
250
+ cls._run_shutdown_hooks()
194
251
 
195
252
  if (
196
253
  callable(existing_signal_handler)
@@ -0,0 +1 @@
1
+ __version__ = '3.5.2'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cluster_tools
3
- Version: 3.5.0
3
+ Version: 3.5.2
4
4
  Summary: Utility library for easily distributing code execution on clusters
5
5
  Author-email: scalable minds <hello@scalableminds.com>
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cluster_tools"
3
- version = "3.5.0" # filled by dunamai
3
+ version = "3.5.2" # filled by dunamai
4
4
  description = "Utility library for easily distributing code execution on clusters"
5
5
  authors = [{name= "scalable minds", email="hello@scalableminds.com"}]
6
6
  readme = "README.md"
@@ -1 +0,0 @@
1
- __version__ = '3.5.0'
File without changes
File without changes
File without changes