deadpool-executor 2026.3.2__tar.gz → 2026.4.1__tar.gz

{deadpool_executor-2026.3.2 → deadpool_executor-2026.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deadpool-executor
-Version: 2026.3.2
+Version: 2026.4.1
 Summary: Deadpool
 Author-email: Caleb Hattingh <caleb.hattingh@gmail.com>
 Description-Content-Type: text/x-rst

{deadpool_executor-2026.3.2 → deadpool_executor-2026.4.1}/deadpool.py

@@ -44,7 +44,7 @@ from functools import partial
 import psutil
 from setproctitle import setproctitle
 
-__version__ = "2026.3.2"
+__version__ = "2026.4.1"
 __all__ = [
     "Deadpool",
     "Future",
@@ -320,6 +320,9 @@ class Deadpool(Executor):
         self.running_jobs = Queue(maxsize=self.pool_size)
         self.running_futs = weakref.WeakSet()
         self.existing_workers = weakref.WeakSet()
+        # Lock protecting busy_workers, existing_workers, and
+        # running_futs for thread-safety without the GIL.
+        self._workers_lock = threading.Lock()
         self.closed = False
         self.shutdown_wait = shutdown_wait
         self.shutdown_cancel_futures = shutdown_cancel_futures
@@ -352,9 +355,10 @@ class Deadpool(Executor):
 
         # These are not counters; they are determined at the time of the
         # call based on the state of the worker processes.
-        stats["worker_processes_still_alive"] = len(self.existing_workers)
+        with self._workers_lock:
+            stats["worker_processes_still_alive"] = len(self.existing_workers)
+            stats["worker_processes_busy"] = len(self.busy_workers)
         stats["worker_processes_idle"] = self.workers.qsize()
-        stats["worker_processes_busy"] = len(self.busy_workers)
 
         return stats
 
@@ -388,7 +392,8 @@ class Deadpool(Executor):
         )
         self.workers.put(worker)
         self._statistics.worker_processes_created.increment()
-        self.existing_workers.add(worker)
+        with self._workers_lock:
+            self.existing_workers.add(worker)
 
     def clear_workers(self):
         """Clear all workers from the pool.
@@ -434,7 +439,8 @@ class Deadpool(Executor):
             t.start()
 
     def get_process(self) -> WorkerProcess:
-        bw = len(self.busy_workers)
+        with self._workers_lock:
+            bw = len(self.busy_workers)
         mw = self.pool_size
         qs = self.workers.qsize()
 
@@ -443,22 +449,20 @@ class Deadpool(Executor):
             self.add_worker_to_pool()
 
         wp = self.workers.get()
-        self.busy_workers.add(wp)
-        if (
-            len(self.busy_workers)
-            > self._statistics.max_workers_busy_concurrently.value
-        ):
-            self._statistics.max_workers_busy_concurrently.value = len(
-                self.busy_workers
-            )
+        with self._workers_lock:
+            self.busy_workers.add(wp)
+            busy_count = len(self.busy_workers)
+        with self._statistics.max_workers_busy_concurrently.lock:
+            if busy_count > self._statistics.max_workers_busy_concurrently.value:
+                self._statistics.max_workers_busy_concurrently.value = busy_count
 
         return wp
 
     def done_with_process(self, wp: WorkerProcess):
         # This worker is done with its job and is no longer busy.
-        self.busy_workers.remove(wp)
-
-        count_workers_busy = len(self.busy_workers)
+        with self._workers_lock:
+            self.busy_workers.remove(wp)
+            count_workers_busy = len(self.busy_workers)
         count_workers_idle = self.workers.qsize()
         backlog_size = self.submitted_jobs.qsize()
 
@@ -544,7 +548,8 @@ class Deadpool(Executor):
                 return
 
             fut.pid = worker.pid
-            self.running_futs.add(fut)
+            with self._workers_lock:
+                self.running_futs.add(fut)
 
             while True:
                 if worker.results_are_available():
@@ -552,19 +557,35 @@ class Deadpool(Executor):
                         results = worker.get_results()
                     except EOFError:
                         self._statistics.tasks_failed.increment()
-                        fut.set_exception(
-                            ProcessError("Worker process died unexpectedly")
-                        )
+                        if not fut.done():
+                            try:
+                                fut.set_exception(
+                                    ProcessError("Worker process died unexpectedly")
+                                )
+                            except InvalidStateError:
+                                pass
                     except BaseException as e:
                         self._statistics.tasks_failed.increment()
                         logger.debug(f"Unexpected exception from worker: {e}")
-                        fut.set_exception(e)
+                        if not fut.done():
+                            try:
+                                fut.set_exception(e)
+                            except InvalidStateError:
+                                pass
                     else:
                         if isinstance(results, BaseException):
                             self._statistics.tasks_failed.increment()
-                            fut.set_exception(results)
+                            if not fut.done():
+                                try:
+                                    fut.set_exception(results)
+                                except InvalidStateError:
+                                    pass
                         else:
-                            fut.set_result(results)
+                            if not fut.done():
+                                try:
+                                    fut.set_result(results)
+                                except InvalidStateError:
+                                    pass
 
                         if isinstance(results, TimeoutError):
                             self._statistics.tasks_failed.increment()
@@ -680,7 +701,8 @@ class Deadpool(Executor):
         # want to wait, that she probably wants us to also stop
         # running processes.
         if (not wait) and cancel_futures:
-            running_futs = list(self.running_futs)
+            with self._workers_lock:
+                running_futs = list(self.running_futs)
             for fut in running_futs:
                 fut.cancel_and_kill_if_running()
 
@@ -700,8 +722,10 @@ class Deadpool(Executor):
 
         # There may be a few processes left in the
         # `busy_workers` queue. Shut them down too.
-        while self.busy_workers:
-            worker = self.busy_workers.pop()
+        with self._workers_lock:
+            remaining = list(self.busy_workers)
+            self.busy_workers.clear()
+        for worker in remaining:
             worker.shutdown()
 
     def __enter__(self):

{deadpool_executor-2026.3.2 → deadpool_executor-2026.4.1}/noxfile.py

@@ -11,6 +11,7 @@ import nox
         "3.12",
         "3.13",
         "3.14",
+        "3.14t",
     ]
 )
 def test(session):
@@ -27,6 +28,7 @@ def test(session):
         "3.12",
         "3.13",
         "3.14",
+        "3.14t",
     ]
 )
 def testcov(session):

{deadpool_executor-2026.3.2 → deadpool_executor-2026.4.1}/tests/test_deadpool.py

@@ -7,7 +7,8 @@ import signal
 import sys
 import time
 import multiprocessing as mp
-from concurrent.futures import CancelledError, as_completed
+import threading
+from concurrent.futures import CancelledError, InvalidStateError, as_completed
 from contextlib import contextmanager
 from functools import partial
 
@@ -634,6 +635,47 @@ def test_cancel_and_kill():
             fut.result()
 
 
+def delayed_error(delay=0.5):
+    time.sleep(delay)
+    raise ValueError("delayed error from worker")
+
+
+def test_set_exception_on_cancelled_future():
+    """Cancelling a future whose worker raises an exception must not
+    cause an InvalidStateError in the run_task thread.
+
+    Reproduces the bug: deadpool futures stay PENDING (never set to
+    RUNNING), so fut.cancel() succeeds even while a worker is
+    processing. When the worker finishes with an exception, run_task
+    calls fut.set_exception() on the already-cancelled future, raising
+    InvalidStateError.
+    """
+    thread_exceptions = []
+    original_hook = threading.excepthook
+
+    def capture_hook(args):
+        thread_exceptions.append(args)
+
+    threading.excepthook = capture_hook
+    try:
+        exe = deadpool.Deadpool(max_workers=1)
+        fut = exe.submit(delayed_error, 0.5)
+        time.sleep(0.2)  # let worker start
+        fut.cancel()  # cancel while worker is still running
+        time.sleep(1.0)  # let worker finish and run_task process results
+        exe.shutdown(wait=True)
+
+        assert fut.cancelled()
+        invalid_state_errors = [
+            e for e in thread_exceptions if isinstance(e.exc_value, InvalidStateError)
+        ]
+        assert (
+            not invalid_state_errors
+        ), f"InvalidStateError raised in background thread: {invalid_state_errors}"
+    finally:
+        threading.excepthook = original_hook
+
+
 def test_trim_memory():
     """Just testing it doesn't fail."""
     deadpool.trim_memory()
@@ -654,13 +696,15 @@ def leaker(n):
 def test_max_memory(logging_initializer):
     # Verify that the memory threshold feature in deadpool
     # works as expected. This test will run 20 functions, 10
-    # of which will consume 1MB of memory. The other 10 will
+    # of which will consume 150MB of memory. The other 10 will
     # consume 0.1MB of memory. The memory threshold is set to
-    # 1.5MB, so the first 10 functions should cause their workers
-    # to be replaced by new workers, while the other 10 functions
+    # 100MB, so the large functions should cause their workers
+    # to be replaced by new workers, while the small functions
     # should be able to run without requiring their workers to be
-    # replaced. So we'll count the total number of subprocess PID
-    # values seen by a task, and verify the result.
+    # replaced. We verify that more unique PIDs were seen than
+    # the pool size, proving that worker replacement occurred.
+    # We use >= 10 rather than == 11 because PID recycling can
+    # cause a replaced worker's PID to be reused by its successor.
 
     leak_test_accumulator.clear()
     with deadpool.Deadpool(
@@ -675,11 +719,10 @@ def test_max_memory(logging_initializer):
 
         pids = set(f.result() for f in deadpool.as_completed(futs))
 
-    # We should see 11 unique PIDs, because the first 10 functions
-    # should have caused their workers to be replaced, while their
-    # replacements should have been able to run the remaining 10
-    # functions without being replaced.
-    assert len(pids) == 11
+    # We expect ~11 unique PIDs (1 initial + 10 replacements), but
+    # PID recycling may reduce this slightly. At minimum we should
+    # see substantially more than max_workers (1).
+    assert len(pids) >= 10
 
 
 def test_can_pickle_nested_function():