deadpool-executor 2025.1.2__tar.gz → 2025.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/PKG-INFO +115 -1
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/README.rst +113 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/deadpool.py +102 -14
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/examples/leftover.py +3 -1
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/pyproject.toml +1 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/tests/test_deadpool.py +91 -8
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/.coveragerc +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/LICENSE-AGPL +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/LICENSE-Apache +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/covstart.pth +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/examples/callbacks.py +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/examples/entrypoint.py +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/examples/priorities.py +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/img1.jpg +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/noxfile.py +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/tests/conftest.py +0 -0
- {deadpool_executor-2025.1.2 → deadpool_executor-2025.2.2}/tests/test_oom.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: deadpool-executor
|
|
3
|
-
Version: 2025.
|
|
3
|
+
Version: 2025.2.2
|
|
4
4
|
Summary: Deadpool
|
|
5
5
|
Author-email: Caleb Hattingh <caleb.hattingh@gmail.com>
|
|
6
6
|
Description-Content-Type: text/x-rst
|
|
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
17
|
Classifier: Programming Language :: Python :: Implementation
|
|
17
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
19
|
Requires-Dist: psutil
|
|
@@ -247,6 +248,119 @@ stdlib pool:
|
|
|
247
248
|
that modification will get the new vars. One example use-case
|
|
248
249
|
is dynamically changing the logging level within subprocesses.
|
|
249
250
|
|
|
251
|
+
Minimum and Maximum Workers
|
|
252
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
253
|
+
|
|
254
|
+
``Deadpool`` has a ``min_workers`` and ``max_workers`` parameter.
|
|
255
|
+
While ``max_workers`` is the same as the stdlib pool, ``min_workers``
|
|
256
|
+
is a new feature.
|
|
257
|
+
|
|
258
|
+
The ``min_workers`` parameter allows deadpool to "scale down" the
|
|
259
|
+
pool when it is idle. This is another strategy alongside other
|
|
260
|
+
features like ``max_tasks_per_child`` and ``max_worker_memory_bytes``
|
|
261
|
+
to help deal with memory bloat in long-running pools.
|
|
262
|
+
|
|
263
|
+
Statistics
|
|
264
|
+
~~~~~~~~~~
|
|
265
|
+
|
|
266
|
+
Here is a very simple example of how to get statistics from the
|
|
267
|
+
executor:
|
|
268
|
+
|
|
269
|
+
.. code-block:: python
|
|
270
|
+
|
|
271
|
+
with deadpool.Deadpool() as exe:
|
|
272
|
+
fut = exe.submit(...)
|
|
273
|
+
stats = exe.get_statistics()
|
|
274
|
+
|
|
275
|
+
The call must be made while the executor is still alive. It
|
|
276
|
+
will succeed after the executor is shut down or closed, but
|
|
277
|
+
some of the statistics will be zeroed out.
|
|
278
|
+
|
|
279
|
+
The call to ``get_statistics`` will return a dictionary with the
|
|
280
|
+
following keys:
|
|
281
|
+
|
|
282
|
+
- ``tasks_received``: The total number of tasks submitted to the
|
|
283
|
+
executor. Does not mean that they started running, only that they
|
|
284
|
+
were successfully submitted.
|
|
285
|
+
- ``tasks_launched``: The total number of tasks that were launched
|
|
286
|
+
on a subprocess. This records the count of all tasks that were
|
|
287
|
+
successfully scheduled to run. These tasks were picked up from
|
|
288
|
+
the submit backlog and given to a worker process to execute.
|
|
289
|
+
- ``tasks_failed``: The total number of tasks that failed. This
|
|
290
|
+
includes tasks that raised an exception, and tasks that were
|
|
291
|
+
killed due to a timeout, and really any other reason that a task
|
|
292
|
+
failed.
|
|
293
|
+
- ``worker_processes_created``: The total number of subprocesses that
|
|
294
|
+
were ever created by the executor. This can be, and often will be
|
|
295
|
+
greater than the `max_workers` setting because there are many options
|
|
296
|
+
that can cause workers to be discarded and replaced. Examples of these
|
|
297
|
+
might be the ``max_tasks_per_child`` setting, or the ``min_workers``
|
|
298
|
+
setting, or the memory thresholds and so on.
|
|
299
|
+
- ``max_workers_busy_concurrently``: The maximum number of workers that
|
|
300
|
+
were ever busy at the same time. This is a useful statistic to
|
|
301
|
+
decide whether you might consider increasing or decreasing the size
|
|
302
|
+
of the pool. For example, if your ``max_workers`` is set to 100, but
|
|
303
|
+
after running for, say, a week, you see that ``max_workers_busy_concurrently``
|
|
304
|
+
is only 50, then you might consider reducing the pool size to 50.
|
|
305
|
+
The system memory manager on linux likes to hold onto heap memory.
|
|
306
|
+
If your have more workers than you need, you'll see that the system
|
|
307
|
+
memory usage over time is going to be higher than it needs to be
|
|
308
|
+
because even when the pool is fully idle, you will still observe
|
|
309
|
+
the persistent worker processes having a large memory allocation
|
|
310
|
+
even though no jobs are running. This is a symptom of malloc
|
|
311
|
+
retention behaviour.
|
|
312
|
+
- ``worker_processes_still_alive``: The number of worker processes that
|
|
313
|
+
are still alive. This includes both idle and busy worker processes.
|
|
314
|
+
This is mainly a debugging statistic that I can use to check whether
|
|
315
|
+
worker processes are "leaking" somehow and not being cleaned up
|
|
316
|
+
correctly. This number should not be greater than the ``max_workers``.
|
|
317
|
+
(It could be, temporarily, depending on the exact timing and strategy
|
|
318
|
+
in the inner workings of the executor, but on average it should not)
|
|
319
|
+
- ``worker_processes_idle``: The number of worker processes that are idle.
|
|
320
|
+
- ``worker_processes_busy``: The number of worker processes that are busy.
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
Here is an example from the tests to explain what each of the
|
|
324
|
+
statistics mean:
|
|
325
|
+
|
|
326
|
+
.. code-block:: python
|
|
327
|
+
|
|
328
|
+
with deadpool.Deadpool(min_workers=5, max_workers=10) as exe:
|
|
329
|
+
futs = []
|
|
330
|
+
for _ in range(50):
|
|
331
|
+
futs.append(exe.submit(t, 0.05))
|
|
332
|
+
futs.append(exe.submit(f_err, Exception))
|
|
333
|
+
|
|
334
|
+
results = []
|
|
335
|
+
for fut in deadpool.as_completed(futs):
|
|
336
|
+
try:
|
|
337
|
+
results.append(fut.result())
|
|
338
|
+
except Exception:
|
|
339
|
+
pass
|
|
340
|
+
|
|
341
|
+
time.sleep(0.5)
|
|
342
|
+
stats = exe.get_statistics()
|
|
343
|
+
|
|
344
|
+
assert results == [0.05] * 50
|
|
345
|
+
print(f"{stats=}")
|
|
346
|
+
assert stats == {
|
|
347
|
+
"tasks_received": 100,
|
|
348
|
+
"tasks_launched": 100,
|
|
349
|
+
"tasks_failed": 50,
|
|
350
|
+
"worker_processes_created": 10,
|
|
351
|
+
"max_workers_busy_concurrently": 10,
|
|
352
|
+
"worker_processes_still_alive": 5,
|
|
353
|
+
"worker_processes_idle": 5,
|
|
354
|
+
"worker_processes_busy": 0,
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
In this example, we submit 100 tasks, 50 of which will raise an
|
|
358
|
+
exception. The executor will create 10 worker processes, and
|
|
359
|
+
will have a maximum of 10 workers busy at the same time. After
|
|
360
|
+
all the tasks are completed, we wait for a short time to allow
|
|
361
|
+
the executor to clean up any worker processes that are no longer
|
|
362
|
+
needed. The statistics should show that 5 worker processes are
|
|
363
|
+
still alive, and all of them are idle.
|
|
250
364
|
|
|
251
365
|
Show me some code
|
|
252
366
|
=================
|
|
@@ -220,6 +220,119 @@ stdlib pool:
|
|
|
220
220
|
that modification will get the new vars. One example use-case
|
|
221
221
|
is dynamically changing the logging level within subprocesses.
|
|
222
222
|
|
|
223
|
+
Minimum and Maximum Workers
|
|
224
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
225
|
+
|
|
226
|
+
``Deadpool`` has a ``min_workers`` and ``max_workers`` parameter.
|
|
227
|
+
While ``max_workers`` is the same as the stdlib pool, ``min_workers``
|
|
228
|
+
is a new feature.
|
|
229
|
+
|
|
230
|
+
The ``min_workers`` parameter allows deadpool to "scale down" the
|
|
231
|
+
pool when it is idle. This is another strategy alongside other
|
|
232
|
+
features like ``max_tasks_per_child`` and ``max_worker_memory_bytes``
|
|
233
|
+
to help deal with memory bloat in long-running pools.
|
|
234
|
+
|
|
235
|
+
Statistics
|
|
236
|
+
~~~~~~~~~~
|
|
237
|
+
|
|
238
|
+
Here is a very simple example of how to get statistics from the
|
|
239
|
+
executor:
|
|
240
|
+
|
|
241
|
+
.. code-block:: python
|
|
242
|
+
|
|
243
|
+
with deadpool.Deadpool() as exe:
|
|
244
|
+
fut = exe.submit(...)
|
|
245
|
+
stats = exe.get_statistics()
|
|
246
|
+
|
|
247
|
+
The call must be made while the executor is still alive. It
|
|
248
|
+
will succeed after the executor is shut down or closed, but
|
|
249
|
+
some of the statistics will be zeroed out.
|
|
250
|
+
|
|
251
|
+
The call to ``get_statistics`` will return a dictionary with the
|
|
252
|
+
following keys:
|
|
253
|
+
|
|
254
|
+
- ``tasks_received``: The total number of tasks submitted to the
|
|
255
|
+
executor. Does not mean that they started running, only that they
|
|
256
|
+
were successfully submitted.
|
|
257
|
+
- ``tasks_launched``: The total number of tasks that were launched
|
|
258
|
+
on a subprocess. This records the count of all tasks that were
|
|
259
|
+
successfully scheduled to run. These tasks were picked up from
|
|
260
|
+
the submit backlog and given to a worker process to execute.
|
|
261
|
+
- ``tasks_failed``: The total number of tasks that failed. This
|
|
262
|
+
includes tasks that raised an exception, and tasks that were
|
|
263
|
+
killed due to a timeout, and really any other reason that a task
|
|
264
|
+
failed.
|
|
265
|
+
- ``worker_processes_created``: The total number of subprocesses that
|
|
266
|
+
were ever created by the executor. This can be, and often will be
|
|
267
|
+
greater than the `max_workers` setting because there are many options
|
|
268
|
+
that can cause workers to be discarded and replaced. Examples of these
|
|
269
|
+
might be the ``max_tasks_per_child`` setting, or the ``min_workers``
|
|
270
|
+
setting, or the memory thresholds and so on.
|
|
271
|
+
- ``max_workers_busy_concurrently``: The maximum number of workers that
|
|
272
|
+
were ever busy at the same time. This is a useful statistic to
|
|
273
|
+
decide whether you might consider increasing or decreasing the size
|
|
274
|
+
of the pool. For example, if your ``max_workers`` is set to 100, but
|
|
275
|
+
after running for, say, a week, you see that ``max_workers_busy_concurrently``
|
|
276
|
+
is only 50, then you might consider reducing the pool size to 50.
|
|
277
|
+
The system memory manager on linux likes to hold onto heap memory.
|
|
278
|
+
If your have more workers than you need, you'll see that the system
|
|
279
|
+
memory usage over time is going to be higher than it needs to be
|
|
280
|
+
because even when the pool is fully idle, you will still observe
|
|
281
|
+
the persistent worker processes having a large memory allocation
|
|
282
|
+
even though no jobs are running. This is a symptom of malloc
|
|
283
|
+
retention behaviour.
|
|
284
|
+
- ``worker_processes_still_alive``: The number of worker processes that
|
|
285
|
+
are still alive. This includes both idle and busy worker processes.
|
|
286
|
+
This is mainly a debugging statistic that I can use to check whether
|
|
287
|
+
worker processes are "leaking" somehow and not being cleaned up
|
|
288
|
+
correctly. This number should not be greater than the ``max_workers``.
|
|
289
|
+
(It could be, temporarily, depending on the exact timing and strategy
|
|
290
|
+
in the inner workings of the executor, but on average it should not)
|
|
291
|
+
- ``worker_processes_idle``: The number of worker processes that are idle.
|
|
292
|
+
- ``worker_processes_busy``: The number of worker processes that are busy.
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
Here is an example from the tests to explain what each of the
|
|
296
|
+
statistics mean:
|
|
297
|
+
|
|
298
|
+
.. code-block:: python
|
|
299
|
+
|
|
300
|
+
with deadpool.Deadpool(min_workers=5, max_workers=10) as exe:
|
|
301
|
+
futs = []
|
|
302
|
+
for _ in range(50):
|
|
303
|
+
futs.append(exe.submit(t, 0.05))
|
|
304
|
+
futs.append(exe.submit(f_err, Exception))
|
|
305
|
+
|
|
306
|
+
results = []
|
|
307
|
+
for fut in deadpool.as_completed(futs):
|
|
308
|
+
try:
|
|
309
|
+
results.append(fut.result())
|
|
310
|
+
except Exception:
|
|
311
|
+
pass
|
|
312
|
+
|
|
313
|
+
time.sleep(0.5)
|
|
314
|
+
stats = exe.get_statistics()
|
|
315
|
+
|
|
316
|
+
assert results == [0.05] * 50
|
|
317
|
+
print(f"{stats=}")
|
|
318
|
+
assert stats == {
|
|
319
|
+
"tasks_received": 100,
|
|
320
|
+
"tasks_launched": 100,
|
|
321
|
+
"tasks_failed": 50,
|
|
322
|
+
"worker_processes_created": 10,
|
|
323
|
+
"max_workers_busy_concurrently": 10,
|
|
324
|
+
"worker_processes_still_alive": 5,
|
|
325
|
+
"worker_processes_idle": 5,
|
|
326
|
+
"worker_processes_busy": 0,
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
In this example, we submit 100 tasks, 50 of which will raise an
|
|
330
|
+
exception. The executor will create 10 worker processes, and
|
|
331
|
+
will have a maximum of 10 workers busy at the same time. After
|
|
332
|
+
all the tasks are completed, we wait for a short time to allow
|
|
333
|
+
the executor to clean up any worker processes that are no longer
|
|
334
|
+
needed. The statistics should show that 5 worker processes are
|
|
335
|
+
still alive, and all of them are idle.
|
|
223
336
|
|
|
224
337
|
Show me some code
|
|
225
338
|
=================
|
|
@@ -32,6 +32,7 @@ import traceback
|
|
|
32
32
|
import typing
|
|
33
33
|
import weakref
|
|
34
34
|
import atexit
|
|
35
|
+
import json
|
|
35
36
|
from concurrent.futures import CancelledError, Executor, InvalidStateError, as_completed
|
|
36
37
|
from dataclasses import dataclass, field
|
|
37
38
|
from multiprocessing.connection import Connection
|
|
@@ -42,7 +43,7 @@ from functools import partial
|
|
|
42
43
|
|
|
43
44
|
import psutil
|
|
44
45
|
|
|
45
|
-
__version__ = "2025.
|
|
46
|
+
__version__ = "2025.2.2"
|
|
46
47
|
__all__ = [
|
|
47
48
|
"Deadpool",
|
|
48
49
|
"Future",
|
|
@@ -71,6 +72,46 @@ logger = logging.getLogger("deadpool")
|
|
|
71
72
|
# logger.info("Error stopping the multiprocessing resource tracker")
|
|
72
73
|
|
|
73
74
|
|
|
75
|
+
@dataclass
|
|
76
|
+
class Stat:
|
|
77
|
+
lock: threading.Lock
|
|
78
|
+
value: int = 0
|
|
79
|
+
|
|
80
|
+
def increment(self, value: int = 1):
|
|
81
|
+
with self.lock:
|
|
82
|
+
self.value += value
|
|
83
|
+
|
|
84
|
+
def set(self, value: int = 0):
|
|
85
|
+
self.value = value
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Statistics:
|
|
89
|
+
def __init__(self):
|
|
90
|
+
self._lock = threading.Lock()
|
|
91
|
+
|
|
92
|
+
self.tasks_received = Stat(self._lock, 0)
|
|
93
|
+
self.tasks_launched = Stat(self._lock, 0)
|
|
94
|
+
self.tasks_failed = Stat(self._lock, 0)
|
|
95
|
+
self.worker_processes_created = Stat(self._lock, 0)
|
|
96
|
+
self.max_workers_busy_concurrently = Stat(self._lock, 0)
|
|
97
|
+
|
|
98
|
+
def reset_counters(self):
|
|
99
|
+
self.tasks_received.set()
|
|
100
|
+
self.tasks_launched.set()
|
|
101
|
+
self.tasks_failed.set()
|
|
102
|
+
self.worker_processes_created.set()
|
|
103
|
+
self.max_workers_busy_concurrently.set()
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> dict[str, typing.Any]:
|
|
106
|
+
return {
|
|
107
|
+
"tasks_received": self.tasks_received.value,
|
|
108
|
+
"tasks_launched": self.tasks_launched.value,
|
|
109
|
+
"tasks_failed": self.tasks_failed.value,
|
|
110
|
+
"worker_processes_created": self.worker_processes_created.value,
|
|
111
|
+
"max_workers_busy_concurrently": self.max_workers_busy_concurrently.value,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
74
115
|
@dataclass(order=True)
|
|
75
116
|
class PrioritizedItem:
|
|
76
117
|
priority: int
|
|
@@ -147,10 +188,10 @@ class WorkerProcess:
|
|
|
147
188
|
|
|
148
189
|
self.connection_receive_msgs_from_process.close()
|
|
149
190
|
|
|
150
|
-
if self.connection_send_msgs_to_process.writable:
|
|
191
|
+
if self.connection_send_msgs_to_process.writable: # pragma: no branch
|
|
151
192
|
try:
|
|
152
193
|
self.connection_send_msgs_to_process.send(None)
|
|
153
|
-
except BrokenPipeError:
|
|
194
|
+
except BrokenPipeError: # pragma: no cover
|
|
154
195
|
pass
|
|
155
196
|
else:
|
|
156
197
|
self.connection_send_msgs_to_process.close()
|
|
@@ -265,7 +306,11 @@ class Deadpool(Executor):
|
|
|
265
306
|
self.finitializer = finalizer
|
|
266
307
|
self.finitargs = finalargs
|
|
267
308
|
self.pool_size = max_workers or len(os.sched_getaffinity(0))
|
|
268
|
-
|
|
309
|
+
if min_workers is None:
|
|
310
|
+
self.min_workers = self.pool_size
|
|
311
|
+
else:
|
|
312
|
+
self.min_workers = min_workers
|
|
313
|
+
|
|
269
314
|
self.max_tasks_per_child = max_tasks_per_child
|
|
270
315
|
self.max_worker_memory_bytes = max_worker_memory_bytes
|
|
271
316
|
self.submitted_jobs: PriorityQueue[PrioritizedItem] = PriorityQueue(
|
|
@@ -273,6 +318,7 @@ class Deadpool(Executor):
|
|
|
273
318
|
)
|
|
274
319
|
self.running_jobs = Queue(maxsize=self.pool_size)
|
|
275
320
|
self.running_futs = weakref.WeakSet()
|
|
321
|
+
self.existing_workers = weakref.WeakSet()
|
|
276
322
|
self.closed = False
|
|
277
323
|
self.shutdown_wait = shutdown_wait
|
|
278
324
|
self.shutdown_cancel_futures = shutdown_cancel_futures
|
|
@@ -280,6 +326,7 @@ class Deadpool(Executor):
|
|
|
280
326
|
self.malloc_trim_rss_memory_threshold_bytes = (
|
|
281
327
|
malloc_trim_rss_memory_threshold_bytes
|
|
282
328
|
)
|
|
329
|
+
self._statistics = Statistics()
|
|
283
330
|
|
|
284
331
|
# TODO: overcommit
|
|
285
332
|
self.workers: SimpleQueue[WorkerProcess] = SimpleQueue()
|
|
@@ -299,6 +346,17 @@ class Deadpool(Executor):
|
|
|
299
346
|
)
|
|
300
347
|
self.runner_thread.start()
|
|
301
348
|
|
|
349
|
+
def get_statistics(self) -> dict[str, typing.Any]:
|
|
350
|
+
stats = self._statistics.to_dict()
|
|
351
|
+
|
|
352
|
+
# These are not counters; they are determined at the time of the
|
|
353
|
+
# call based on the state of the worker processes.
|
|
354
|
+
stats["worker_processes_still_alive"] = len(self.existing_workers)
|
|
355
|
+
stats["worker_processes_idle"] = self.workers.qsize()
|
|
356
|
+
stats["worker_processes_busy"] = len(self.busy_workers)
|
|
357
|
+
|
|
358
|
+
return stats
|
|
359
|
+
|
|
302
360
|
def add_worker_to_pool(self):
|
|
303
361
|
if self.propagate_environ:
|
|
304
362
|
# By constructing here, late, we allow the user to make
|
|
@@ -328,6 +386,8 @@ class Deadpool(Executor):
|
|
|
328
386
|
malloc_trim_rss_memory_threshold_bytes=self.malloc_trim_rss_memory_threshold_bytes,
|
|
329
387
|
)
|
|
330
388
|
self.workers.put(worker)
|
|
389
|
+
self._statistics.worker_processes_created.increment()
|
|
390
|
+
self.existing_workers.add(worker)
|
|
331
391
|
|
|
332
392
|
def clear_workers(self):
|
|
333
393
|
"""Clear all workers from the pool.
|
|
@@ -369,6 +429,7 @@ class Deadpool(Executor):
|
|
|
369
429
|
continue
|
|
370
430
|
|
|
371
431
|
t = threading.Thread(target=self.run_task, args=job, daemon=True)
|
|
432
|
+
self._statistics.tasks_launched.increment()
|
|
372
433
|
t.start()
|
|
373
434
|
|
|
374
435
|
def get_process(self) -> WorkerProcess:
|
|
@@ -382,17 +443,35 @@ class Deadpool(Executor):
|
|
|
382
443
|
|
|
383
444
|
wp = self.workers.get()
|
|
384
445
|
self.busy_workers.add(wp)
|
|
446
|
+
if (
|
|
447
|
+
len(self.busy_workers)
|
|
448
|
+
> self._statistics.max_workers_busy_concurrently.value
|
|
449
|
+
):
|
|
450
|
+
self._statistics.max_workers_busy_concurrently.value = len(
|
|
451
|
+
self.busy_workers
|
|
452
|
+
)
|
|
453
|
+
|
|
385
454
|
return wp
|
|
386
455
|
|
|
387
456
|
def done_with_process(self, wp: WorkerProcess):
|
|
457
|
+
# This worker is done with its job and is no longer busy.
|
|
388
458
|
self.busy_workers.remove(wp)
|
|
389
459
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
460
|
+
count_workers_busy = len(self.busy_workers)
|
|
461
|
+
count_workers_idle = self.workers.qsize()
|
|
462
|
+
backlog_size = self.submitted_jobs.qsize()
|
|
463
|
+
|
|
464
|
+
# The `1` is for `wp` itself.
|
|
465
|
+
total_workers = count_workers_busy + count_workers_idle + 1
|
|
466
|
+
there_are_more_workers_than_min = total_workers > self.min_workers
|
|
467
|
+
task_backlog_is_empty = backlog_size == 0
|
|
468
|
+
|
|
469
|
+
# if there_are_more_workers_than_min and (there_are_idle_workers or task_backlog_is_empty):
|
|
470
|
+
if there_are_more_workers_than_min and task_backlog_is_empty:
|
|
471
|
+
# We have more workers than the minimum, and there is no backlog of
|
|
472
|
+
# tasks. This implies any tasks currently in play have already been picked
|
|
473
|
+
# up by workers in the pool, or the pool is idle. We can safely remove
|
|
474
|
+
# this worker from the pool.
|
|
396
475
|
wp.shutdown(wait=False)
|
|
397
476
|
return
|
|
398
477
|
|
|
@@ -450,9 +529,12 @@ class Deadpool(Executor):
|
|
|
450
529
|
# where the worker process often OOMs. As such, not sure
|
|
451
530
|
# whether logging at warning level is appropriate.
|
|
452
531
|
logger.warning(f"BrokenPipeError on {worker.pid}, retrying.")
|
|
532
|
+
worker.ok = False
|
|
453
533
|
self.done_with_process(worker)
|
|
534
|
+
# TODO: probably this should be moved into the `done_with_process`
|
|
535
|
+
# and can act on the `worker.ok` flag.
|
|
454
536
|
kill_proc_tree(worker.pid, sig=signal.SIGKILL)
|
|
455
|
-
else:
|
|
537
|
+
else: # pragma: no cover
|
|
456
538
|
# If we get here, we've tried to submit the job to a worker
|
|
457
539
|
# process multiple times and failed each time. We're giving
|
|
458
540
|
# up.
|
|
@@ -468,19 +550,23 @@ class Deadpool(Executor):
|
|
|
468
550
|
try:
|
|
469
551
|
results = worker.get_results()
|
|
470
552
|
except EOFError:
|
|
553
|
+
self._statistics.tasks_failed.increment()
|
|
471
554
|
fut.set_exception(
|
|
472
555
|
ProcessError("Worker process died unexpectedly")
|
|
473
556
|
)
|
|
474
557
|
except BaseException as e:
|
|
558
|
+
self._statistics.tasks_failed.increment()
|
|
475
559
|
logger.debug(f"Unexpected exception from worker: {e}")
|
|
476
560
|
fut.set_exception(e)
|
|
477
561
|
else:
|
|
478
562
|
if isinstance(results, BaseException):
|
|
563
|
+
self._statistics.tasks_failed.increment()
|
|
479
564
|
fut.set_exception(results)
|
|
480
565
|
else:
|
|
481
566
|
fut.set_result(results)
|
|
482
567
|
|
|
483
568
|
if isinstance(results, TimeoutError):
|
|
569
|
+
self._statistics.tasks_failed.increment()
|
|
484
570
|
logger.debug(
|
|
485
571
|
f"TimeoutError on {worker.pid}, setting ok=False"
|
|
486
572
|
)
|
|
@@ -488,6 +574,7 @@ class Deadpool(Executor):
|
|
|
488
574
|
finally:
|
|
489
575
|
break
|
|
490
576
|
elif not worker.is_alive():
|
|
577
|
+
self._statistics.tasks_failed.increment()
|
|
491
578
|
logger.debug(f"p is no longer alive: {worker.process}")
|
|
492
579
|
try:
|
|
493
580
|
signame = signal.strsignal(-worker.process.exitcode)
|
|
@@ -550,6 +637,7 @@ class Deadpool(Executor):
|
|
|
550
637
|
item=(fn, args, kwargs, deadpool_timeout, fut),
|
|
551
638
|
)
|
|
552
639
|
)
|
|
640
|
+
self._statistics.tasks_received.increment()
|
|
553
641
|
return fut
|
|
554
642
|
|
|
555
643
|
def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
|
|
@@ -604,7 +692,7 @@ class Deadpool(Executor):
|
|
|
604
692
|
try:
|
|
605
693
|
worker = self.workers.get_nowait()
|
|
606
694
|
worker.shutdown()
|
|
607
|
-
except Empty:
|
|
695
|
+
except Empty: # pragma: no cover
|
|
608
696
|
break
|
|
609
697
|
|
|
610
698
|
# There may be a few processes left in the
|
|
@@ -673,7 +761,7 @@ def kill_proc_tree(
|
|
|
673
761
|
for p in children:
|
|
674
762
|
try:
|
|
675
763
|
p.send_signal(sig)
|
|
676
|
-
except psutil.NoSuchProcess:
|
|
764
|
+
except psutil.NoSuchProcess: # pragma: no cover
|
|
677
765
|
pass
|
|
678
766
|
|
|
679
767
|
gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate)
|
|
@@ -726,7 +814,7 @@ def raw_runner2(
|
|
|
726
814
|
conn.send(obj)
|
|
727
815
|
except BrokenPipeError: # pragma: no cover
|
|
728
816
|
logger.debug("Pipe not usable")
|
|
729
|
-
except BaseException:
|
|
817
|
+
except BaseException: # pragma: no cover
|
|
730
818
|
logger.exception("Unexpected pipe error")
|
|
731
819
|
|
|
732
820
|
def timed_out():
|
|
@@ -28,7 +28,9 @@ def main():
|
|
|
28
28
|
time.sleep(random.randrange(0, 10) / 10)
|
|
29
29
|
w = random.randrange(0, 50) / 10
|
|
30
30
|
exe.submit(worker, results, w, deadpool_timeout=4)
|
|
31
|
-
print(
|
|
31
|
+
print(
|
|
32
|
+
f"{exe.workers.qsize()=} {len(exe.busy_workers)=} {len(exe.existing_workers)=}"
|
|
33
|
+
)
|
|
32
34
|
|
|
33
35
|
outcome = []
|
|
34
36
|
while not results.empty():
|
|
@@ -20,6 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Programming Language :: Python :: 3.10",
|
|
21
21
|
"Programming Language :: Python :: 3.11",
|
|
22
22
|
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
23
24
|
"Programming Language :: Python :: Implementation",
|
|
24
25
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
25
26
|
]
|
|
@@ -5,6 +5,7 @@ import queue
|
|
|
5
5
|
import signal
|
|
6
6
|
import sys
|
|
7
7
|
import time
|
|
8
|
+
import multiprocessing as mp
|
|
8
9
|
from concurrent.futures import CancelledError, as_completed
|
|
9
10
|
from contextlib import contextmanager
|
|
10
11
|
from functools import partial
|
|
@@ -19,11 +20,6 @@ def logging_initializer():
|
|
|
19
20
|
return partial(logging.basicConfig, level=logging.DEBUG)
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
async def test_func():
|
|
23
|
-
await asyncio.sleep(0.1)
|
|
24
|
-
return 42
|
|
25
|
-
|
|
26
|
-
|
|
27
23
|
def f():
|
|
28
24
|
return 123
|
|
29
25
|
|
|
@@ -53,14 +49,36 @@ def test_cancel_all_futures():
|
|
|
53
49
|
|
|
54
50
|
|
|
55
51
|
@pytest.mark.parametrize("malloc_threshold", [None, 0, 1_000_000])
|
|
56
|
-
|
|
52
|
+
@pytest.mark.parametrize("daemon", [True, False])
|
|
53
|
+
@pytest.mark.parametrize("min_workers", [None, 10])
|
|
54
|
+
def test_simple(malloc_threshold, daemon, min_workers):
|
|
57
55
|
with deadpool.Deadpool(
|
|
58
|
-
malloc_trim_rss_memory_threshold_bytes=malloc_threshold
|
|
56
|
+
malloc_trim_rss_memory_threshold_bytes=malloc_threshold,
|
|
57
|
+
daemon=daemon,
|
|
58
|
+
min_workers=min_workers,
|
|
59
|
+
max_workers=10,
|
|
59
60
|
) as exe:
|
|
60
61
|
fut = exe.submit(t, 0.05)
|
|
61
62
|
result = fut.result()
|
|
62
63
|
|
|
64
|
+
time.sleep(0.5)
|
|
65
|
+
stats = exe.get_statistics()
|
|
66
|
+
# To exercise the reset path
|
|
67
|
+
exe._statistics.reset_counters()
|
|
68
|
+
|
|
63
69
|
assert result == 0.05
|
|
70
|
+
print(f"{stats=}")
|
|
71
|
+
|
|
72
|
+
assert stats == {
|
|
73
|
+
"tasks_received": 1,
|
|
74
|
+
"tasks_launched": 1,
|
|
75
|
+
"tasks_failed": 0,
|
|
76
|
+
"worker_processes_created": 10,
|
|
77
|
+
"max_workers_busy_concurrently": 1,
|
|
78
|
+
"worker_processes_still_alive": 10,
|
|
79
|
+
"worker_processes_idle": 10,
|
|
80
|
+
"worker_processes_busy": 0,
|
|
81
|
+
}
|
|
64
82
|
|
|
65
83
|
# Outside the context manager, no new tasks
|
|
66
84
|
# can be submitted.
|
|
@@ -68,6 +86,68 @@ def test_simple(malloc_threshold):
|
|
|
68
86
|
exe.submit(f)
|
|
69
87
|
|
|
70
88
|
|
|
89
|
+
def test_stats():
|
|
90
|
+
with deadpool.Deadpool(
|
|
91
|
+
min_workers=5,
|
|
92
|
+
max_workers=10,
|
|
93
|
+
) as exe:
|
|
94
|
+
futs = []
|
|
95
|
+
for _ in range(6):
|
|
96
|
+
futs.append(exe.submit(t, 0.05))
|
|
97
|
+
|
|
98
|
+
results = [fut.result() for fut in deadpool.as_completed(futs)]
|
|
99
|
+
time.sleep(0.5)
|
|
100
|
+
stats = exe.get_statistics()
|
|
101
|
+
|
|
102
|
+
assert results == [0.05] * 6
|
|
103
|
+
print(f"{stats=}")
|
|
104
|
+
assert stats == {
|
|
105
|
+
"tasks_received": 6,
|
|
106
|
+
"tasks_launched": 6,
|
|
107
|
+
"tasks_failed": 0,
|
|
108
|
+
"worker_processes_created": 10,
|
|
109
|
+
# This is an important one.
|
|
110
|
+
"max_workers_busy_concurrently": 6,
|
|
111
|
+
"worker_processes_still_alive": 5,
|
|
112
|
+
"worker_processes_idle": 5,
|
|
113
|
+
"worker_processes_busy": 0,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_stats_with_errors():
|
|
118
|
+
with deadpool.Deadpool(
|
|
119
|
+
min_workers=5,
|
|
120
|
+
max_workers=10,
|
|
121
|
+
) as exe:
|
|
122
|
+
futs = []
|
|
123
|
+
for _ in range(50):
|
|
124
|
+
futs.append(exe.submit(t, 0.05))
|
|
125
|
+
futs.append(exe.submit(f_err, Exception))
|
|
126
|
+
|
|
127
|
+
results = []
|
|
128
|
+
for fut in deadpool.as_completed(futs):
|
|
129
|
+
try:
|
|
130
|
+
results.append(fut.result())
|
|
131
|
+
except Exception:
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
time.sleep(0.5)
|
|
135
|
+
stats = exe.get_statistics()
|
|
136
|
+
|
|
137
|
+
assert results == [0.05] * 50
|
|
138
|
+
print(f"{stats=}")
|
|
139
|
+
assert stats == {
|
|
140
|
+
"tasks_received": 100,
|
|
141
|
+
"tasks_launched": 100,
|
|
142
|
+
"tasks_failed": 50,
|
|
143
|
+
"worker_processes_created": 10,
|
|
144
|
+
"max_workers_busy_concurrently": 10,
|
|
145
|
+
"worker_processes_still_alive": 5,
|
|
146
|
+
"worker_processes_idle": 5,
|
|
147
|
+
"worker_processes_busy": 0,
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
71
151
|
##### Env var propagation #####
|
|
72
152
|
|
|
73
153
|
|
|
@@ -217,7 +297,10 @@ def test_simple_batch(logging_initializer):
|
|
|
217
297
|
assert results == [0.1] * 2
|
|
218
298
|
|
|
219
299
|
|
|
220
|
-
@pytest.mark.parametrize(
|
|
300
|
+
@pytest.mark.parametrize(
|
|
301
|
+
"ctx",
|
|
302
|
+
["spawn", "forkserver", mp.get_context("spawn"), mp.get_context("forkserver")],
|
|
303
|
+
)
|
|
221
304
|
def test_ctx(logging_initializer, ctx):
|
|
222
305
|
with deadpool.Deadpool(mp_context=ctx, initializer=logging_initializer) as exe:
|
|
223
306
|
fut = exe.submit(t, 0.05)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|