deadpool-executor 2025.1.2__tar.gz → 2025.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: deadpool-executor
3
- Version: 2025.1.2
3
+ Version: 2025.2.2
4
4
  Summary: Deadpool
5
5
  Author-email: Caleb Hattingh <caleb.hattingh@gmail.com>
6
6
  Description-Content-Type: text/x-rst
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
13
13
  Classifier: Programming Language :: Python :: 3.10
14
14
  Classifier: Programming Language :: Python :: 3.11
15
15
  Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
16
17
  Classifier: Programming Language :: Python :: Implementation
17
18
  Classifier: Programming Language :: Python :: Implementation :: CPython
18
19
  Requires-Dist: psutil
@@ -247,6 +248,119 @@ stdlib pool:
247
248
  that modification will get the new vars. One example use-case
248
249
  is dynamically changing the logging level within subprocesses.
249
250
 
251
+ Minimum and Maximum Workers
252
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
253
+
254
+ ``Deadpool`` has a ``min_workers`` and ``max_workers`` parameter.
255
+ While ``max_workers`` is the same as the stdlib pool, ``min_workers``
256
+ is a new feature.
257
+
258
+ The ``min_workers`` parameter allows deadpool to "scale down" the
259
+ pool when it is idle. This is another strategy alongside other
260
+ features like ``max_tasks_per_child`` and ``max_worker_memory_bytes``
261
+ to help deal with memory bloat in long-running pools.
262
+
263
+ Statistics
264
+ ~~~~~~~~~~
265
+
266
+ Here is a very simple example of how to get statistics from the
267
+ executor:
268
+
269
+ .. code-block:: python
270
+
271
+ with deadpool.Deadpool() as exe:
272
+ fut = exe.submit(...)
273
+ stats = exe.get_statistics()
274
+
275
+ The call must be made while the executor is still alive. It
276
+ will succeed after the executor is shut down or closed, but
277
+ some of the statistics will be zeroed out.
278
+
279
+ The call to ``get_statistics`` will return a dictionary with the
280
+ following keys:
281
+
282
+ - ``tasks_received``: The total number of tasks submitted to the
283
+ executor. Does not mean that they started running, only that they
284
+ were successfully submitted.
285
+ - ``tasks_launched``: The total number of tasks that were launched
286
+ on a subprocess. This records the count of all tasks that were
287
+ successfully scheduled to run. These tasks were picked up from
288
+ the submit backlog and given to a worker process to execute.
289
+ - ``tasks_failed``: The total number of tasks that failed. This
290
+ includes tasks that raised an exception, and tasks that were
291
+ killed due to a timeout, and really any other reason that a task
292
+ failed.
293
+ - ``worker_processes_created``: The total number of subprocesses that
294
+ were ever created by the executor. This can be, and often will be
295
+ greater than the `max_workers` setting because there are many options
296
+ that can cause workers to be discarded and replaced. Examples of these
297
+ might be the ``max_tasks_per_child`` setting, or the ``min_workers``
298
+ setting, or the memory thresholds and so on.
299
+ - ``max_workers_busy_concurrently``: The maximum number of workers that
300
+ were ever busy at the same time. This is a useful statistic to
301
+ decide whether you might consider increasing or decreasing the size
302
+ of the pool. For example, if your ``max_workers`` is set to 100, but
303
+ after running for, say, a week, you see that ``max_workers_busy_concurrently``
304
+ is only 50, then you might consider reducing the pool size to 50.
305
+ The system memory manager on linux likes to hold onto heap memory.
306
+ If your have more workers than you need, you'll see that the system
307
+ memory usage over time is going to be higher than it needs to be
308
+ because even when the pool is fully idle, you will still observe
309
+ the persistent worker processes having a large memory allocation
310
+ even though no jobs are running. This is a symptom of malloc
311
+ retention behaviour.
312
+ - ``worker_processes_still_alive``: The number of worker processes that
313
+ are still alive. This includes both idle and busy worker processes.
314
+ This is mainly a debugging statistic that I can use to check whether
315
+ worker processes are "leaking" somehow and not being cleaned up
316
+ correctly. This number should not be greater than the ``max_workers``.
317
+ (It could be, temporarily, depending on the exact timing and strategy
318
+ in the inner workings of the executor, but on average it should not)
319
+ - ``worker_processes_idle``: The number of worker processes that are idle.
320
+ - ``worker_processes_busy``: The number of worker processes that are busy.
321
+
322
+
323
+ Here is an example from the tests to explain what each of the
324
+ statistics mean:
325
+
326
+ .. code-block:: python
327
+
328
+ with deadpool.Deadpool(min_workers=5, max_workers=10) as exe:
329
+ futs = []
330
+ for _ in range(50):
331
+ futs.append(exe.submit(t, 0.05))
332
+ futs.append(exe.submit(f_err, Exception))
333
+
334
+ results = []
335
+ for fut in deadpool.as_completed(futs):
336
+ try:
337
+ results.append(fut.result())
338
+ except Exception:
339
+ pass
340
+
341
+ time.sleep(0.5)
342
+ stats = exe.get_statistics()
343
+
344
+ assert results == [0.05] * 50
345
+ print(f"{stats=}")
346
+ assert stats == {
347
+ "tasks_received": 100,
348
+ "tasks_launched": 100,
349
+ "tasks_failed": 50,
350
+ "worker_processes_created": 10,
351
+ "max_workers_busy_concurrently": 10,
352
+ "worker_processes_still_alive": 5,
353
+ "worker_processes_idle": 5,
354
+ "worker_processes_busy": 0,
355
+ }
356
+
357
+ In this example, we submit 100 tasks, 50 of which will raise an
358
+ exception. The executor will create 10 worker processes, and
359
+ will have a maximum of 10 workers busy at the same time. After
360
+ all the tasks are completed, we wait for a short time to allow
361
+ the executor to clean up any worker processes that are no longer
362
+ needed. The statistics should show that 5 worker processes are
363
+ still alive, and all of them are idle.
250
364
 
251
365
  Show me some code
252
366
  =================
@@ -220,6 +220,119 @@ stdlib pool:
220
220
  that modification will get the new vars. One example use-case
221
221
  is dynamically changing the logging level within subprocesses.
222
222
 
223
+ Minimum and Maximum Workers
224
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
225
+
226
+ ``Deadpool`` has a ``min_workers`` and ``max_workers`` parameter.
227
+ While ``max_workers`` is the same as the stdlib pool, ``min_workers``
228
+ is a new feature.
229
+
230
+ The ``min_workers`` parameter allows deadpool to "scale down" the
231
+ pool when it is idle. This is another strategy alongside other
232
+ features like ``max_tasks_per_child`` and ``max_worker_memory_bytes``
233
+ to help deal with memory bloat in long-running pools.
234
+
235
+ Statistics
236
+ ~~~~~~~~~~
237
+
238
+ Here is a very simple example of how to get statistics from the
239
+ executor:
240
+
241
+ .. code-block:: python
242
+
243
+ with deadpool.Deadpool() as exe:
244
+ fut = exe.submit(...)
245
+ stats = exe.get_statistics()
246
+
247
+ The call must be made while the executor is still alive. It
248
+ will succeed after the executor is shut down or closed, but
249
+ some of the statistics will be zeroed out.
250
+
251
+ The call to ``get_statistics`` will return a dictionary with the
252
+ following keys:
253
+
254
+ - ``tasks_received``: The total number of tasks submitted to the
255
+ executor. Does not mean that they started running, only that they
256
+ were successfully submitted.
257
+ - ``tasks_launched``: The total number of tasks that were launched
258
+ on a subprocess. This records the count of all tasks that were
259
+ successfully scheduled to run. These tasks were picked up from
260
+ the submit backlog and given to a worker process to execute.
261
+ - ``tasks_failed``: The total number of tasks that failed. This
262
+ includes tasks that raised an exception, and tasks that were
263
+ killed due to a timeout, and really any other reason that a task
264
+ failed.
265
+ - ``worker_processes_created``: The total number of subprocesses that
266
+ were ever created by the executor. This can be, and often will be
267
+ greater than the `max_workers` setting because there are many options
268
+ that can cause workers to be discarded and replaced. Examples of these
269
+ might be the ``max_tasks_per_child`` setting, or the ``min_workers``
270
+ setting, or the memory thresholds and so on.
271
+ - ``max_workers_busy_concurrently``: The maximum number of workers that
272
+ were ever busy at the same time. This is a useful statistic to
273
+ decide whether you might consider increasing or decreasing the size
274
+ of the pool. For example, if your ``max_workers`` is set to 100, but
275
+ after running for, say, a week, you see that ``max_workers_busy_concurrently``
276
+ is only 50, then you might consider reducing the pool size to 50.
277
+ The system memory manager on linux likes to hold onto heap memory.
278
+ If your have more workers than you need, you'll see that the system
279
+ memory usage over time is going to be higher than it needs to be
280
+ because even when the pool is fully idle, you will still observe
281
+ the persistent worker processes having a large memory allocation
282
+ even though no jobs are running. This is a symptom of malloc
283
+ retention behaviour.
284
+ - ``worker_processes_still_alive``: The number of worker processes that
285
+ are still alive. This includes both idle and busy worker processes.
286
+ This is mainly a debugging statistic that I can use to check whether
287
+ worker processes are "leaking" somehow and not being cleaned up
288
+ correctly. This number should not be greater than the ``max_workers``.
289
+ (It could be, temporarily, depending on the exact timing and strategy
290
+ in the inner workings of the executor, but on average it should not)
291
+ - ``worker_processes_idle``: The number of worker processes that are idle.
292
+ - ``worker_processes_busy``: The number of worker processes that are busy.
293
+
294
+
295
+ Here is an example from the tests to explain what each of the
296
+ statistics mean:
297
+
298
+ .. code-block:: python
299
+
300
+ with deadpool.Deadpool(min_workers=5, max_workers=10) as exe:
301
+ futs = []
302
+ for _ in range(50):
303
+ futs.append(exe.submit(t, 0.05))
304
+ futs.append(exe.submit(f_err, Exception))
305
+
306
+ results = []
307
+ for fut in deadpool.as_completed(futs):
308
+ try:
309
+ results.append(fut.result())
310
+ except Exception:
311
+ pass
312
+
313
+ time.sleep(0.5)
314
+ stats = exe.get_statistics()
315
+
316
+ assert results == [0.05] * 50
317
+ print(f"{stats=}")
318
+ assert stats == {
319
+ "tasks_received": 100,
320
+ "tasks_launched": 100,
321
+ "tasks_failed": 50,
322
+ "worker_processes_created": 10,
323
+ "max_workers_busy_concurrently": 10,
324
+ "worker_processes_still_alive": 5,
325
+ "worker_processes_idle": 5,
326
+ "worker_processes_busy": 0,
327
+ }
328
+
329
+ In this example, we submit 100 tasks, 50 of which will raise an
330
+ exception. The executor will create 10 worker processes, and
331
+ will have a maximum of 10 workers busy at the same time. After
332
+ all the tasks are completed, we wait for a short time to allow
333
+ the executor to clean up any worker processes that are no longer
334
+ needed. The statistics should show that 5 worker processes are
335
+ still alive, and all of them are idle.
223
336
 
224
337
  Show me some code
225
338
  =================
@@ -32,6 +32,7 @@ import traceback
32
32
  import typing
33
33
  import weakref
34
34
  import atexit
35
+ import json
35
36
  from concurrent.futures import CancelledError, Executor, InvalidStateError, as_completed
36
37
  from dataclasses import dataclass, field
37
38
  from multiprocessing.connection import Connection
@@ -42,7 +43,7 @@ from functools import partial
42
43
 
43
44
  import psutil
44
45
 
45
- __version__ = "2025.1.2"
46
+ __version__ = "2025.2.2"
46
47
  __all__ = [
47
48
  "Deadpool",
48
49
  "Future",
@@ -71,6 +72,46 @@ logger = logging.getLogger("deadpool")
71
72
  # logger.info("Error stopping the multiprocessing resource tracker")
72
73
 
73
74
 
75
+ @dataclass
76
+ class Stat:
77
+ lock: threading.Lock
78
+ value: int = 0
79
+
80
+ def increment(self, value: int = 1):
81
+ with self.lock:
82
+ self.value += value
83
+
84
+ def set(self, value: int = 0):
85
+ self.value = value
86
+
87
+
88
+ class Statistics:
89
+ def __init__(self):
90
+ self._lock = threading.Lock()
91
+
92
+ self.tasks_received = Stat(self._lock, 0)
93
+ self.tasks_launched = Stat(self._lock, 0)
94
+ self.tasks_failed = Stat(self._lock, 0)
95
+ self.worker_processes_created = Stat(self._lock, 0)
96
+ self.max_workers_busy_concurrently = Stat(self._lock, 0)
97
+
98
+ def reset_counters(self):
99
+ self.tasks_received.set()
100
+ self.tasks_launched.set()
101
+ self.tasks_failed.set()
102
+ self.worker_processes_created.set()
103
+ self.max_workers_busy_concurrently.set()
104
+
105
+ def to_dict(self) -> dict[str, typing.Any]:
106
+ return {
107
+ "tasks_received": self.tasks_received.value,
108
+ "tasks_launched": self.tasks_launched.value,
109
+ "tasks_failed": self.tasks_failed.value,
110
+ "worker_processes_created": self.worker_processes_created.value,
111
+ "max_workers_busy_concurrently": self.max_workers_busy_concurrently.value,
112
+ }
113
+
114
+
74
115
  @dataclass(order=True)
75
116
  class PrioritizedItem:
76
117
  priority: int
@@ -147,10 +188,10 @@ class WorkerProcess:
147
188
 
148
189
  self.connection_receive_msgs_from_process.close()
149
190
 
150
- if self.connection_send_msgs_to_process.writable:
191
+ if self.connection_send_msgs_to_process.writable: # pragma: no branch
151
192
  try:
152
193
  self.connection_send_msgs_to_process.send(None)
153
- except BrokenPipeError:
194
+ except BrokenPipeError: # pragma: no cover
154
195
  pass
155
196
  else:
156
197
  self.connection_send_msgs_to_process.close()
@@ -265,7 +306,11 @@ class Deadpool(Executor):
265
306
  self.finitializer = finalizer
266
307
  self.finitargs = finalargs
267
308
  self.pool_size = max_workers or len(os.sched_getaffinity(0))
268
- self.min_workers = min_workers or self.pool_size
309
+ if min_workers is None:
310
+ self.min_workers = self.pool_size
311
+ else:
312
+ self.min_workers = min_workers
313
+
269
314
  self.max_tasks_per_child = max_tasks_per_child
270
315
  self.max_worker_memory_bytes = max_worker_memory_bytes
271
316
  self.submitted_jobs: PriorityQueue[PrioritizedItem] = PriorityQueue(
@@ -273,6 +318,7 @@ class Deadpool(Executor):
273
318
  )
274
319
  self.running_jobs = Queue(maxsize=self.pool_size)
275
320
  self.running_futs = weakref.WeakSet()
321
+ self.existing_workers = weakref.WeakSet()
276
322
  self.closed = False
277
323
  self.shutdown_wait = shutdown_wait
278
324
  self.shutdown_cancel_futures = shutdown_cancel_futures
@@ -280,6 +326,7 @@ class Deadpool(Executor):
280
326
  self.malloc_trim_rss_memory_threshold_bytes = (
281
327
  malloc_trim_rss_memory_threshold_bytes
282
328
  )
329
+ self._statistics = Statistics()
283
330
 
284
331
  # TODO: overcommit
285
332
  self.workers: SimpleQueue[WorkerProcess] = SimpleQueue()
@@ -299,6 +346,17 @@ class Deadpool(Executor):
299
346
  )
300
347
  self.runner_thread.start()
301
348
 
349
+ def get_statistics(self) -> dict[str, typing.Any]:
350
+ stats = self._statistics.to_dict()
351
+
352
+ # These are not counters; they are determined at the time of the
353
+ # call based on the state of the worker processes.
354
+ stats["worker_processes_still_alive"] = len(self.existing_workers)
355
+ stats["worker_processes_idle"] = self.workers.qsize()
356
+ stats["worker_processes_busy"] = len(self.busy_workers)
357
+
358
+ return stats
359
+
302
360
  def add_worker_to_pool(self):
303
361
  if self.propagate_environ:
304
362
  # By constructing here, late, we allow the user to make
@@ -328,6 +386,8 @@ class Deadpool(Executor):
328
386
  malloc_trim_rss_memory_threshold_bytes=self.malloc_trim_rss_memory_threshold_bytes,
329
387
  )
330
388
  self.workers.put(worker)
389
+ self._statistics.worker_processes_created.increment()
390
+ self.existing_workers.add(worker)
331
391
 
332
392
  def clear_workers(self):
333
393
  """Clear all workers from the pool.
@@ -369,6 +429,7 @@ class Deadpool(Executor):
369
429
  continue
370
430
 
371
431
  t = threading.Thread(target=self.run_task, args=job, daemon=True)
432
+ self._statistics.tasks_launched.increment()
372
433
  t.start()
373
434
 
374
435
  def get_process(self) -> WorkerProcess:
@@ -382,17 +443,35 @@ class Deadpool(Executor):
382
443
 
383
444
  wp = self.workers.get()
384
445
  self.busy_workers.add(wp)
446
+ if (
447
+ len(self.busy_workers)
448
+ > self._statistics.max_workers_busy_concurrently.value
449
+ ):
450
+ self._statistics.max_workers_busy_concurrently.value = len(
451
+ self.busy_workers
452
+ )
453
+
385
454
  return wp
386
455
 
387
456
  def done_with_process(self, wp: WorkerProcess):
457
+ # This worker is done with its job and is no longer busy.
388
458
  self.busy_workers.remove(wp)
389
459
 
390
- bw = len(self.busy_workers)
391
- mw = self.min_workers
392
- qs = self.workers.qsize()
393
-
394
- total_workers = bw + qs
395
- if total_workers > mw and qs > 0:
460
+ count_workers_busy = len(self.busy_workers)
461
+ count_workers_idle = self.workers.qsize()
462
+ backlog_size = self.submitted_jobs.qsize()
463
+
464
+ # The `1` is for `wp` itself.
465
+ total_workers = count_workers_busy + count_workers_idle + 1
466
+ there_are_more_workers_than_min = total_workers > self.min_workers
467
+ task_backlog_is_empty = backlog_size == 0
468
+
469
+ # if there_are_more_workers_than_min and (there_are_idle_workers or task_backlog_is_empty):
470
+ if there_are_more_workers_than_min and task_backlog_is_empty:
471
+ # We have more workers than the minimum, and there is no backlog of
472
+ # tasks. This implies any tasks currently in play have already been picked
473
+ # up by workers in the pool, or the pool is idle. We can safely remove
474
+ # this worker from the pool.
396
475
  wp.shutdown(wait=False)
397
476
  return
398
477
 
@@ -450,9 +529,12 @@ class Deadpool(Executor):
450
529
  # where the worker process often OOMs. As such, not sure
451
530
  # whether logging at warning level is appropriate.
452
531
  logger.warning(f"BrokenPipeError on {worker.pid}, retrying.")
532
+ worker.ok = False
453
533
  self.done_with_process(worker)
534
+ # TODO: probably this should be moved into the `done_with_process`
535
+ # and can act on the `worker.ok` flag.
454
536
  kill_proc_tree(worker.pid, sig=signal.SIGKILL)
455
- else:
537
+ else: # pragma: no cover
456
538
  # If we get here, we've tried to submit the job to a worker
457
539
  # process multiple times and failed each time. We're giving
458
540
  # up.
@@ -468,19 +550,23 @@ class Deadpool(Executor):
468
550
  try:
469
551
  results = worker.get_results()
470
552
  except EOFError:
553
+ self._statistics.tasks_failed.increment()
471
554
  fut.set_exception(
472
555
  ProcessError("Worker process died unexpectedly")
473
556
  )
474
557
  except BaseException as e:
558
+ self._statistics.tasks_failed.increment()
475
559
  logger.debug(f"Unexpected exception from worker: {e}")
476
560
  fut.set_exception(e)
477
561
  else:
478
562
  if isinstance(results, BaseException):
563
+ self._statistics.tasks_failed.increment()
479
564
  fut.set_exception(results)
480
565
  else:
481
566
  fut.set_result(results)
482
567
 
483
568
  if isinstance(results, TimeoutError):
569
+ self._statistics.tasks_failed.increment()
484
570
  logger.debug(
485
571
  f"TimeoutError on {worker.pid}, setting ok=False"
486
572
  )
@@ -488,6 +574,7 @@ class Deadpool(Executor):
488
574
  finally:
489
575
  break
490
576
  elif not worker.is_alive():
577
+ self._statistics.tasks_failed.increment()
491
578
  logger.debug(f"p is no longer alive: {worker.process}")
492
579
  try:
493
580
  signame = signal.strsignal(-worker.process.exitcode)
@@ -550,6 +637,7 @@ class Deadpool(Executor):
550
637
  item=(fn, args, kwargs, deadpool_timeout, fut),
551
638
  )
552
639
  )
640
+ self._statistics.tasks_received.increment()
553
641
  return fut
554
642
 
555
643
  def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
@@ -604,7 +692,7 @@ class Deadpool(Executor):
604
692
  try:
605
693
  worker = self.workers.get_nowait()
606
694
  worker.shutdown()
607
- except Empty:
695
+ except Empty: # pragma: no cover
608
696
  break
609
697
 
610
698
  # There may be a few processes left in the
@@ -673,7 +761,7 @@ def kill_proc_tree(
673
761
  for p in children:
674
762
  try:
675
763
  p.send_signal(sig)
676
- except psutil.NoSuchProcess:
764
+ except psutil.NoSuchProcess: # pragma: no cover
677
765
  pass
678
766
 
679
767
  gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate)
@@ -726,7 +814,7 @@ def raw_runner2(
726
814
  conn.send(obj)
727
815
  except BrokenPipeError: # pragma: no cover
728
816
  logger.debug("Pipe not usable")
729
- except BaseException:
817
+ except BaseException: # pragma: no cover
730
818
  logger.exception("Unexpected pipe error")
731
819
 
732
820
  def timed_out():
@@ -28,7 +28,9 @@ def main():
28
28
  time.sleep(random.randrange(0, 10) / 10)
29
29
  w = random.randrange(0, 50) / 10
30
30
  exe.submit(worker, results, w, deadpool_timeout=4)
31
- print(f"{exe.workers.qsize()=} {len(exe.busy_workers)=}")
31
+ print(
32
+ f"{exe.workers.qsize()=} {len(exe.busy_workers)=} {len(exe.existing_workers)=}"
33
+ )
32
34
 
33
35
  outcome = []
34
36
  while not results.empty():
@@ -20,6 +20,7 @@ classifiers = [
20
20
  "Programming Language :: Python :: 3.10",
21
21
  "Programming Language :: Python :: 3.11",
22
22
  "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
23
24
  "Programming Language :: Python :: Implementation",
24
25
  "Programming Language :: Python :: Implementation :: CPython",
25
26
  ]
@@ -5,6 +5,7 @@ import queue
5
5
  import signal
6
6
  import sys
7
7
  import time
8
+ import multiprocessing as mp
8
9
  from concurrent.futures import CancelledError, as_completed
9
10
  from contextlib import contextmanager
10
11
  from functools import partial
@@ -19,11 +20,6 @@ def logging_initializer():
19
20
  return partial(logging.basicConfig, level=logging.DEBUG)
20
21
 
21
22
 
22
- async def test_func():
23
- await asyncio.sleep(0.1)
24
- return 42
25
-
26
-
27
23
  def f():
28
24
  return 123
29
25
 
@@ -53,14 +49,36 @@ def test_cancel_all_futures():
53
49
 
54
50
 
55
51
  @pytest.mark.parametrize("malloc_threshold", [None, 0, 1_000_000])
56
- def test_simple(malloc_threshold):
52
+ @pytest.mark.parametrize("daemon", [True, False])
53
+ @pytest.mark.parametrize("min_workers", [None, 10])
54
+ def test_simple(malloc_threshold, daemon, min_workers):
57
55
  with deadpool.Deadpool(
58
- malloc_trim_rss_memory_threshold_bytes=malloc_threshold
56
+ malloc_trim_rss_memory_threshold_bytes=malloc_threshold,
57
+ daemon=daemon,
58
+ min_workers=min_workers,
59
+ max_workers=10,
59
60
  ) as exe:
60
61
  fut = exe.submit(t, 0.05)
61
62
  result = fut.result()
62
63
 
64
+ time.sleep(0.5)
65
+ stats = exe.get_statistics()
66
+ # To exercise the reset path
67
+ exe._statistics.reset_counters()
68
+
63
69
  assert result == 0.05
70
+ print(f"{stats=}")
71
+
72
+ assert stats == {
73
+ "tasks_received": 1,
74
+ "tasks_launched": 1,
75
+ "tasks_failed": 0,
76
+ "worker_processes_created": 10,
77
+ "max_workers_busy_concurrently": 1,
78
+ "worker_processes_still_alive": 10,
79
+ "worker_processes_idle": 10,
80
+ "worker_processes_busy": 0,
81
+ }
64
82
 
65
83
  # Outside the context manager, no new tasks
66
84
  # can be submitted.
@@ -68,6 +86,68 @@ def test_simple(malloc_threshold):
68
86
  exe.submit(f)
69
87
 
70
88
 
89
+ def test_stats():
90
+ with deadpool.Deadpool(
91
+ min_workers=5,
92
+ max_workers=10,
93
+ ) as exe:
94
+ futs = []
95
+ for _ in range(6):
96
+ futs.append(exe.submit(t, 0.05))
97
+
98
+ results = [fut.result() for fut in deadpool.as_completed(futs)]
99
+ time.sleep(0.5)
100
+ stats = exe.get_statistics()
101
+
102
+ assert results == [0.05] * 6
103
+ print(f"{stats=}")
104
+ assert stats == {
105
+ "tasks_received": 6,
106
+ "tasks_launched": 6,
107
+ "tasks_failed": 0,
108
+ "worker_processes_created": 10,
109
+ # This is an important one.
110
+ "max_workers_busy_concurrently": 6,
111
+ "worker_processes_still_alive": 5,
112
+ "worker_processes_idle": 5,
113
+ "worker_processes_busy": 0,
114
+ }
115
+
116
+
117
+ def test_stats_with_errors():
118
+ with deadpool.Deadpool(
119
+ min_workers=5,
120
+ max_workers=10,
121
+ ) as exe:
122
+ futs = []
123
+ for _ in range(50):
124
+ futs.append(exe.submit(t, 0.05))
125
+ futs.append(exe.submit(f_err, Exception))
126
+
127
+ results = []
128
+ for fut in deadpool.as_completed(futs):
129
+ try:
130
+ results.append(fut.result())
131
+ except Exception:
132
+ pass
133
+
134
+ time.sleep(0.5)
135
+ stats = exe.get_statistics()
136
+
137
+ assert results == [0.05] * 50
138
+ print(f"{stats=}")
139
+ assert stats == {
140
+ "tasks_received": 100,
141
+ "tasks_launched": 100,
142
+ "tasks_failed": 50,
143
+ "worker_processes_created": 10,
144
+ "max_workers_busy_concurrently": 10,
145
+ "worker_processes_still_alive": 5,
146
+ "worker_processes_idle": 5,
147
+ "worker_processes_busy": 0,
148
+ }
149
+
150
+
71
151
  ##### Env var propagation #####
72
152
 
73
153
 
@@ -217,7 +297,10 @@ def test_simple_batch(logging_initializer):
217
297
  assert results == [0.1] * 2
218
298
 
219
299
 
220
- @pytest.mark.parametrize("ctx", ["spawn", "forkserver"])
300
+ @pytest.mark.parametrize(
301
+ "ctx",
302
+ ["spawn", "forkserver", mp.get_context("spawn"), mp.get_context("forkserver")],
303
+ )
221
304
  def test_ctx(logging_initializer, ctx):
222
305
  with deadpool.Deadpool(mp_context=ctx, initializer=logging_initializer) as exe:
223
306
  fut = exe.submit(t, 0.05)