map-parallel 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
map_parallel/__init__.py
ADDED
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""A drop-in replacement of :func:`map` and :func:`itertools.starmap`, but in parallel with different backends.
|
|
2
|
+
|
|
3
|
+
>>> from map_parallel import map_parallel, starmap_parallel
|
|
4
|
+
>>> map_parallel(pow, [1, 2, 3], [2, 2, 2], mode="multithreading")
|
|
5
|
+
[1, 4, 9]
|
|
6
|
+
>>> starmap_parallel(pow, [(1, 2), (2, 2), (3, 2)], mode="multiprocessing", processes=2)
|
|
7
|
+
[1, 4, 9]
|
|
8
|
+
|
|
9
|
+
Exceptions raised by the mapped function propagate to the caller, whether the
|
|
10
|
+
results are returned or not:
|
|
11
|
+
|
|
12
|
+
>>> map_parallel(int, ["1", "x"], mode="multithreading", return_results=False)
|
|
13
|
+
Traceback (most recent call last):
|
|
14
|
+
...
|
|
15
|
+
ValueError: invalid literal for int() with base 10: 'x'
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import sys
|
|
21
|
+
from collections import deque
|
|
22
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
23
|
+
from functools import partial
|
|
24
|
+
from importlib.metadata import version
|
|
25
|
+
from itertools import chain, islice, starmap
|
|
26
|
+
from typing import Any, Literal, TypeVar
|
|
27
|
+
|
|
28
|
+
__version__ = version("map-parallel")
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"MODES",
|
|
32
|
+
"Mode",
|
|
33
|
+
"map_parallel",
|
|
34
|
+
"starmap_parallel",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
T = TypeVar("T")
|
|
38
|
+
|
|
39
|
+
Mode = Literal[
|
|
40
|
+
"multiprocessing",
|
|
41
|
+
"multithreading",
|
|
42
|
+
"multiinterpreter",
|
|
43
|
+
"dask",
|
|
44
|
+
"mpi",
|
|
45
|
+
"mpi_simple",
|
|
46
|
+
"serial",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
# helpers ######################################################################
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _starmap_chunk(f: Callable[..., T], chunk: Iterable[tuple[Any, ...]]) -> list[T]:
|
|
53
|
+
"""Apply ``f`` to each tuple of arguments in a chunk, in the worker.
|
|
54
|
+
|
|
55
|
+
Module-level, so that it pickles by reference.
|
|
56
|
+
"""
|
|
57
|
+
return list(starmap(f, chunk))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _batched(iterable: Iterable[T], n: int) -> Iterator[tuple[T, ...]]:
|
|
61
|
+
""":func:`itertools.batched`, which is new in Python 3.12."""
|
|
62
|
+
iterator = iter(iterable)
|
|
63
|
+
while batch := tuple(islice(iterator, n)):
|
|
64
|
+
yield batch
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _collect(results: Iterable[T], return_results: bool) -> list[T]:
|
|
68
|
+
"""Exhaust ``results``, so that an exception raised by any task propagates.
|
|
69
|
+
|
|
70
|
+
An executor raises the exception of a task only when its result is
|
|
71
|
+
retrieved, so the results are iterated even when they are not kept
|
|
72
|
+
(https://github.com/ickc/python-map_parallel/issues/2).
|
|
73
|
+
"""
|
|
74
|
+
if return_results:
|
|
75
|
+
return list(results)
|
|
76
|
+
deque(results, maxlen=0)
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _set_sys_path(path: tuple[str, ...]) -> None:
|
|
81
|
+
"""Initialize a worker interpreter with the main interpreter's ``sys.path``.
|
|
82
|
+
|
|
83
|
+
A new interpreter computes ``sys.path`` afresh, missing changes made at run
|
|
84
|
+
time (by pytest, or a script's own directory), so functions defined in such
|
|
85
|
+
modules would not unpickle in the workers.
|
|
86
|
+
"""
|
|
87
|
+
sys.path[:] = path
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# backends #####################################################################
|
|
91
|
+
# Each backend computes ``f(*args)`` for each ``args`` in ``iterable``, in order.
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _starmap_serial(
|
|
95
|
+
f: Callable[..., T],
|
|
96
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
97
|
+
*,
|
|
98
|
+
processes: int | None = None,
|
|
99
|
+
return_results: bool = True,
|
|
100
|
+
chunksize: int = 1,
|
|
101
|
+
) -> list[T]:
|
|
102
|
+
return _collect(starmap(f, iterable), return_results)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _starmap_executor(
|
|
106
|
+
executor: Any,
|
|
107
|
+
f: Callable[..., T],
|
|
108
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
109
|
+
*,
|
|
110
|
+
return_results: bool,
|
|
111
|
+
chunksize: int,
|
|
112
|
+
) -> list[T]:
|
|
113
|
+
"""Run on a :class:`concurrent.futures.Executor`, ``chunksize`` tasks per submission.
|
|
114
|
+
|
|
115
|
+
On an exception, the result iterator of ``Executor.map`` cancels the tasks
|
|
116
|
+
not yet started, and leaving the ``with`` block waits for those running.
|
|
117
|
+
"""
|
|
118
|
+
with executor:
|
|
119
|
+
chunks = executor.map(partial(_starmap_chunk, f), _batched(iterable, chunksize))
|
|
120
|
+
return _collect(chain.from_iterable(chunks), return_results)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _starmap_multiprocessing(
|
|
124
|
+
f: Callable[..., T],
|
|
125
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
126
|
+
*,
|
|
127
|
+
processes: int | None = None,
|
|
128
|
+
return_results: bool = True,
|
|
129
|
+
chunksize: int = 1,
|
|
130
|
+
) -> list[T]:
|
|
131
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
132
|
+
|
|
133
|
+
return _starmap_executor(
|
|
134
|
+
ProcessPoolExecutor(max_workers=processes),
|
|
135
|
+
f,
|
|
136
|
+
iterable,
|
|
137
|
+
return_results=return_results,
|
|
138
|
+
chunksize=chunksize,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _starmap_multithreading(
|
|
143
|
+
f: Callable[..., T],
|
|
144
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
145
|
+
*,
|
|
146
|
+
processes: int | None = None,
|
|
147
|
+
return_results: bool = True,
|
|
148
|
+
chunksize: int = 1,
|
|
149
|
+
) -> list[T]:
|
|
150
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
151
|
+
|
|
152
|
+
return _starmap_executor(
|
|
153
|
+
ThreadPoolExecutor(max_workers=processes),
|
|
154
|
+
f,
|
|
155
|
+
iterable,
|
|
156
|
+
return_results=return_results,
|
|
157
|
+
chunksize=chunksize,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _starmap_multiinterpreter(
|
|
162
|
+
f: Callable[..., T],
|
|
163
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
164
|
+
*,
|
|
165
|
+
processes: int | None = None,
|
|
166
|
+
return_results: bool = True,
|
|
167
|
+
chunksize: int = 1,
|
|
168
|
+
) -> list[T]:
|
|
169
|
+
try:
|
|
170
|
+
from concurrent.futures import InterpreterPoolExecutor
|
|
171
|
+
except ImportError:
|
|
172
|
+
raise ImportError(
|
|
173
|
+
"mode='multiinterpreter' needs concurrent.futures.InterpreterPoolExecutor, new in Python 3.14; "
|
|
174
|
+
f"this is Python {sys.version_info.major}.{sys.version_info.minor}."
|
|
175
|
+
) from None
|
|
176
|
+
|
|
177
|
+
return _starmap_executor(
|
|
178
|
+
InterpreterPoolExecutor(
|
|
179
|
+
max_workers=processes,
|
|
180
|
+
initializer=_set_sys_path,
|
|
181
|
+
initargs=(tuple(sys.path),),
|
|
182
|
+
),
|
|
183
|
+
f,
|
|
184
|
+
iterable,
|
|
185
|
+
return_results=return_results,
|
|
186
|
+
chunksize=chunksize,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _starmap_dask_client(
|
|
191
|
+
client: Any,
|
|
192
|
+
f: Callable[..., T],
|
|
193
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
194
|
+
*,
|
|
195
|
+
return_results: bool,
|
|
196
|
+
chunksize: int,
|
|
197
|
+
) -> list[T]:
|
|
198
|
+
from dask.utils import funcname
|
|
199
|
+
from distributed import wait
|
|
200
|
+
|
|
201
|
+
futures = client.map(
|
|
202
|
+
partial(_starmap_chunk, f),
|
|
203
|
+
list(_batched(iterable, chunksize)),
|
|
204
|
+
# f may be impure, so that equal arguments must not share one result
|
|
205
|
+
pure=False,
|
|
206
|
+
key=funcname(f),
|
|
207
|
+
)
|
|
208
|
+
try:
|
|
209
|
+
if return_results:
|
|
210
|
+
return [result for chunk in client.gather(futures) for result in chunk]
|
|
211
|
+
wait(futures)
|
|
212
|
+
for future in futures:
|
|
213
|
+
if future.status == "error":
|
|
214
|
+
future.result()
|
|
215
|
+
return []
|
|
216
|
+
except BaseException:
|
|
217
|
+
client.cancel(futures)
|
|
218
|
+
raise
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _starmap_dask(
|
|
222
|
+
f: Callable[..., T],
|
|
223
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
224
|
+
*,
|
|
225
|
+
processes: int | None = None,
|
|
226
|
+
return_results: bool = True,
|
|
227
|
+
chunksize: int = 1,
|
|
228
|
+
) -> list[T]:
|
|
229
|
+
from distributed import Client, LocalCluster, default_client
|
|
230
|
+
|
|
231
|
+
kwargs = {"return_results": return_results, "chunksize": chunksize}
|
|
232
|
+
try:
|
|
233
|
+
client = default_client()
|
|
234
|
+
except ValueError:
|
|
235
|
+
pass
|
|
236
|
+
else:
|
|
237
|
+
return _starmap_dask_client(client, f, iterable, **kwargs)
|
|
238
|
+
cluster_kwargs: dict[str, Any] = {"dashboard_address": None}
|
|
239
|
+
if processes is not None:
|
|
240
|
+
cluster_kwargs.update(n_workers=processes, threads_per_worker=1)
|
|
241
|
+
with LocalCluster(**cluster_kwargs) as cluster, Client(cluster) as client:
|
|
242
|
+
return _starmap_dask_client(client, f, iterable, **kwargs)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _starmap_mpi(
|
|
246
|
+
f: Callable[..., T],
|
|
247
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
248
|
+
*,
|
|
249
|
+
processes: int | None = None,
|
|
250
|
+
return_results: bool = True,
|
|
251
|
+
chunksize: int = 1,
|
|
252
|
+
) -> list[T]:
|
|
253
|
+
from mpi4py.futures import MPIPoolExecutor
|
|
254
|
+
|
|
255
|
+
with MPIPoolExecutor(max_workers=processes) as executor:
|
|
256
|
+
return _collect(executor.starmap(f, iterable, chunksize=chunksize), return_results)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class _RemoteTraceback(Exception):
|
|
260
|
+
"""The formatted traceback of an exception raised on another MPI rank."""
|
|
261
|
+
|
|
262
|
+
def __init__(self, tb: str) -> None:
|
|
263
|
+
self.tb = tb
|
|
264
|
+
|
|
265
|
+
def __str__(self) -> str:
|
|
266
|
+
return self.tb
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _exception_payload(exc: BaseException) -> tuple[str, bytes]:
|
|
270
|
+
"""The traceback of ``exc`` as text, and ``exc`` pickled (or, if it does not pickle, its repr)."""
|
|
271
|
+
import pickle
|
|
272
|
+
from traceback import format_exception
|
|
273
|
+
|
|
274
|
+
tb = "".join(format_exception(exc))
|
|
275
|
+
try:
|
|
276
|
+
pickled = pickle.dumps(exc)
|
|
277
|
+
except Exception:
|
|
278
|
+
pickled = pickle.dumps(RuntimeError(repr(exc)))
|
|
279
|
+
return tb, pickled
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _exception_from_payload(rank: int, payload: tuple[str, bytes]) -> BaseException:
|
|
283
|
+
import pickle
|
|
284
|
+
|
|
285
|
+
tb, pickled = payload
|
|
286
|
+
try:
|
|
287
|
+
exc = pickle.loads(pickled)
|
|
288
|
+
except Exception:
|
|
289
|
+
exc = RuntimeError(f"an exception that does not unpickle was raised on MPI rank {rank}")
|
|
290
|
+
exc.__cause__ = _RemoteTraceback(f'\n"""\nraised on MPI rank {rank}:\n{tb}"""')
|
|
291
|
+
return exc
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _starmap_mpi_simple(
|
|
295
|
+
f: Callable[..., T],
|
|
296
|
+
iterable: Iterable[tuple[Any, ...]],
|
|
297
|
+
*,
|
|
298
|
+
processes: int | None = None,
|
|
299
|
+
return_results: bool = True,
|
|
300
|
+
chunksize: int = 1,
|
|
301
|
+
) -> list[T]:
|
|
302
|
+
"""Split the arguments into one contiguous block per rank of ``MPI.COMM_WORLD``.
|
|
303
|
+
|
|
304
|
+
Every rank must call it, with the same arguments. If ``f`` raises on any
|
|
305
|
+
rank, it raises on every rank: on the ranks where ``f`` raised, the
|
|
306
|
+
exception itself, and on the others the first rank's exception, with that
|
|
307
|
+
rank's traceback as its cause, as :mod:`concurrent.futures` does. Otherwise
|
|
308
|
+
the ranks that succeeded would wait forever for the ones that failed.
|
|
309
|
+
"""
|
|
310
|
+
from mpi4py import MPI
|
|
311
|
+
|
|
312
|
+
comm = MPI.COMM_WORLD
|
|
313
|
+
size = comm.Get_size()
|
|
314
|
+
rank = comm.Get_rank()
|
|
315
|
+
|
|
316
|
+
args = list(iterable)
|
|
317
|
+
n = len(args)
|
|
318
|
+
local_args = args[(rank * n) // size : ((rank + 1) * n) // size]
|
|
319
|
+
del args
|
|
320
|
+
|
|
321
|
+
error: Exception | None = None
|
|
322
|
+
local_results: list[T] = []
|
|
323
|
+
try:
|
|
324
|
+
local_results = list(starmap(f, local_args))
|
|
325
|
+
except Exception as e:
|
|
326
|
+
error = e
|
|
327
|
+
|
|
328
|
+
# collective: every rank learns whether any rank failed
|
|
329
|
+
payloads = comm.allgather(None if error is None else _exception_payload(error))
|
|
330
|
+
if error is not None:
|
|
331
|
+
raise error
|
|
332
|
+
for other_rank, payload in enumerate(payloads):
|
|
333
|
+
if payload is not None:
|
|
334
|
+
raise _exception_from_payload(other_rank, payload)
|
|
335
|
+
|
|
336
|
+
if return_results:
|
|
337
|
+
gathered = comm.gather(local_results, root=0)
|
|
338
|
+
if rank == 0:
|
|
339
|
+
return [result for block in gathered for result in block]
|
|
340
|
+
return []
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
_BACKENDS: dict[str, Callable[..., list[Any]]] = {
|
|
344
|
+
"multiprocessing": _starmap_multiprocessing,
|
|
345
|
+
"multithreading": _starmap_multithreading,
|
|
346
|
+
"multiinterpreter": _starmap_multiinterpreter,
|
|
347
|
+
"dask": _starmap_dask,
|
|
348
|
+
"mpi": _starmap_mpi,
|
|
349
|
+
"mpi_simple": _starmap_mpi_simple,
|
|
350
|
+
"serial": _starmap_serial,
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
MODES: tuple[str, ...] = tuple(_BACKENDS)
|
|
354
|
+
"""The valid values of ``mode``."""
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _validate(mode: str, chunksize: int) -> Callable[..., list[Any]]:
|
|
358
|
+
try:
|
|
359
|
+
backend = _BACKENDS[mode]
|
|
360
|
+
except (KeyError, TypeError):
|
|
361
|
+
raise ValueError(f"mode must be one of {', '.join(map(repr, MODES))}; got {mode!r}.") from None
|
|
362
|
+
if not isinstance(chunksize, int) or chunksize < 1:
|
|
363
|
+
raise ValueError(f"chunksize must be a positive integer; got {chunksize!r}.")
|
|
364
|
+
return backend
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
# public API ###################################################################
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def map_parallel(
|
|
371
|
+
f: Callable[..., T],
|
|
372
|
+
*args: Iterable[Any],
|
|
373
|
+
processes: int | None = None,
|
|
374
|
+
mode: Mode = "multiprocessing",
|
|
375
|
+
return_results: bool = True,
|
|
376
|
+
chunksize: int = 1,
|
|
377
|
+
) -> list[T]:
|
|
378
|
+
"""Equivalent to ``list(map(f, *args))``, but in parallel.
|
|
379
|
+
|
|
380
|
+
:param f: the function to apply. Except for ``multithreading`` and
|
|
381
|
+
``serial``, it, its arguments and its results are sent to other
|
|
382
|
+
processes or interpreters, so they must be picklable. For
|
|
383
|
+
``multiprocessing`` and the MPI modes, ``f`` pickles by reference, so
|
|
384
|
+
it must be defined at the top level of a module: not a lambda nor a
|
|
385
|
+
closure. ``dask`` (through cloudpickle) and ``multiinterpreter`` also
|
|
386
|
+
take a lambda.
|
|
387
|
+
:param args: iterables of the arguments of ``f``, as in :func:`map`. They
|
|
388
|
+
are consumed in full before any result is returned.
|
|
389
|
+
:param processes: the number of workers; ``None`` for the backend's
|
|
390
|
+
default. If ``processes <= 1``, run serially (in every MPI rank, for
|
|
391
|
+
``mpi_simple``). For ``mpi``, the number of MPI processes is set by
|
|
392
|
+
``mpiexec`` if the script is run with ``python -m mpi4py.futures``,
|
|
393
|
+
else it is the number of processes to spawn. For ``mpi_simple``, it is
|
|
394
|
+
the number of MPI processes, set by ``mpiexec`` only.
|
|
395
|
+
:param mode: the backend to run on:
|
|
396
|
+
|
|
397
|
+
- ``multiprocessing``: :class:`concurrent.futures.ProcessPoolExecutor`,
|
|
398
|
+
for CPU-bound work in pure Python on a build with a GIL.
|
|
399
|
+
- ``multithreading``: :class:`concurrent.futures.ThreadPoolExecutor`,
|
|
400
|
+
for I/O-bound work, work that releases the GIL (much of NumPy), or
|
|
401
|
+
any work on a free-threaded build of Python.
|
|
402
|
+
- ``multiinterpreter``: :class:`concurrent.futures.InterpreterPoolExecutor`,
|
|
403
|
+
Python 3.14 or later: one interpreter per worker, each with its own
|
|
404
|
+
GIL, in one process.
|
|
405
|
+
- ``dask``: dask.distributed. Runs on the active
|
|
406
|
+
:class:`distributed.Client` if there is one (``processes`` is then
|
|
407
|
+
ignored), else on a :class:`distributed.LocalCluster` of
|
|
408
|
+
``processes`` single-threaded workers, closed afterwards.
|
|
409
|
+
- ``mpi``: :class:`mpi4py.futures.MPIPoolExecutor`. May not work,
|
|
410
|
+
depending on your MPI implementation; run with
|
|
411
|
+
``mpiexec -n N python -m mpi4py.futures script.py``.
|
|
412
|
+
- ``mpi_simple``: :mod:`mpi4py`, splitting the work into equal
|
|
413
|
+
contiguous blocks, one per MPI rank. Every rank must call it with the
|
|
414
|
+
same arguments; run with ``mpiexec -n N python script.py``.
|
|
415
|
+
- ``serial``: :func:`map`.
|
|
416
|
+
:param return_results: if ``False``, return ``[]`` instead of the results,
|
|
417
|
+
which are then not sent back (for ``mpi_simple``, not gathered to rank
|
|
418
|
+
0). Exceptions raised by ``f`` still propagate.
|
|
419
|
+
:param chunksize: the number of calls of ``f`` sent to a worker at a time.
|
|
420
|
+
Larger chunks cost less overhead when ``f`` is quick; smaller ones
|
|
421
|
+
balance the load better when its duration varies. Ignored by
|
|
422
|
+
``mpi_simple`` and ``serial``.
|
|
423
|
+
:return: the results, in order. For ``mpi_simple``, on rank 0 only; the
|
|
424
|
+
other ranks get ``[]``.
|
|
425
|
+
:raises ValueError: if ``mode`` or ``chunksize`` is invalid.
|
|
426
|
+
:raises ImportError: if the backend of ``mode`` is not available: dask or
|
|
427
|
+
mpi4py is not installed, or ``multiinterpreter`` on Python < 3.14.
|
|
428
|
+
|
|
429
|
+
Any exception that ``f`` raises is re-raised, with the remote traceback as
|
|
430
|
+
its cause where the backend provides it.
|
|
431
|
+
"""
|
|
432
|
+
backend = _validate(mode, chunksize)
|
|
433
|
+
if processes is not None and processes <= 1:
|
|
434
|
+
return _collect(map(f, *args), return_results)
|
|
435
|
+
# like map, stop at the shortest iterable
|
|
436
|
+
iterable = zip(*args, strict=False)
|
|
437
|
+
return backend(f, iterable, processes=processes, return_results=return_results, chunksize=chunksize)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def starmap_parallel(
|
|
441
|
+
f: Callable[..., T],
|
|
442
|
+
args: Iterable[Iterable[Any]],
|
|
443
|
+
processes: int | None = None,
|
|
444
|
+
mode: Mode = "multiprocessing",
|
|
445
|
+
return_results: bool = True,
|
|
446
|
+
chunksize: int = 1,
|
|
447
|
+
) -> list[T]:
|
|
448
|
+
"""Equivalent to ``list(itertools.starmap(f, args))``, but in parallel.
|
|
449
|
+
|
|
450
|
+
:param args: an iterable of iterables, each the arguments of one call of
|
|
451
|
+
``f``.
|
|
452
|
+
|
|
453
|
+
See :func:`map_parallel` for the other parameters.
|
|
454
|
+
"""
|
|
455
|
+
backend = _validate(mode, chunksize)
|
|
456
|
+
if processes is not None and processes <= 1:
|
|
457
|
+
return _collect(starmap(f, args), return_results)
|
|
458
|
+
return backend(f, map(tuple, args), processes=processes, return_results=return_results, chunksize=chunksize)
|
map_parallel/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: map-parallel
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A drop-in replacement of map/starmap but in parallel with different backends.
|
|
5
|
+
Keywords: map,starmap,parallel,concurrency,multiprocessing,multithreading,subinterpreters,free-threading,dask,mpi,mpi4py
|
|
6
|
+
Author: Kolen Cheung
|
|
7
|
+
Author-email: Kolen Cheung <christian.kolen@gmail.com>
|
|
8
|
+
License-Expression: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
|
+
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
23
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
24
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
25
|
+
Classifier: Topic :: Utilities
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Dist: distributed>=2023.1 ; extra == 'dask'
|
|
28
|
+
Requires-Dist: mpi4py>=3.1 ; extra == 'mpi'
|
|
29
|
+
Requires-Python: >=3.11
|
|
30
|
+
Project-URL: Homepage, https://github.com/ickc/python-map_parallel
|
|
31
|
+
Project-URL: Documentation, https://ickc.github.io/python-map_parallel
|
|
32
|
+
Project-URL: Repository, https://github.com/ickc/python-map_parallel
|
|
33
|
+
Project-URL: Issues, https://github.com/ickc/python-map_parallel/issues
|
|
34
|
+
Project-URL: Changelog, https://github.com/ickc/python-map_parallel/blob/main/CHANGELOG.md
|
|
35
|
+
Provides-Extra: dask
|
|
36
|
+
Provides-Extra: mpi
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# map_parallel
|
|
40
|
+
|
|
41
|
+
[](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
|
|
42
|
+
[](https://ickc.github.io/python-map_parallel)
|
|
43
|
+
[](https://pypi.org/project/map-parallel)
|
|
44
|
+
[](https://pypi.org/project/map-parallel)
|
|
45
|
+
[](https://anaconda.org/conda-forge/map_parallel)
|
|
46
|
+
[](https://doi.org/10.5281/zenodo.4641295)
|
|
47
|
+
[](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
|
|
48
|
+
|
|
49
|
+
A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
|
|
50
|
+
backends to choose from: processes, threads, subinterpreters, dask and MPI.
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from map_parallel import map_parallel, starmap_parallel
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def f(x, y):
|
|
57
|
+
return x**y
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == "__main__":
|
|
61
|
+
# list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
|
|
62
|
+
map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
|
|
63
|
+
# list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
|
|
64
|
+
starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The same code runs on any backend: change `mode`, and nothing else.
|
|
68
|
+
|
|
69
|
+
## Installation
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
pip install map-parallel
|
|
73
|
+
# with the optional backends
|
|
74
|
+
pip install "map-parallel[dask]"
|
|
75
|
+
pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
|
|
76
|
+
# or
|
|
77
|
+
conda install -c conda-forge map_parallel
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
map_parallel itself is pure Python with no dependencies, supports Python 3.11
|
|
81
|
+
and later, and is tested on free-threaded builds of Python (3.13t on).
|
|
82
|
+
|
|
83
|
+
## Backends
|
|
84
|
+
|
|
85
|
+
| `mode` | runs on | suits |
|
|
86
|
+
| --- | --- | --- |
|
|
87
|
+
| `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
|
|
88
|
+
| `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
|
|
89
|
+
| `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
|
|
90
|
+
| `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
|
|
91
|
+
| `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
|
|
92
|
+
| `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
|
|
93
|
+
| `serial` | `map` | debugging |
|
|
94
|
+
|
|
95
|
+
[Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
|
|
96
|
+
explains which to use when, and why.
|
|
97
|
+
|
|
98
|
+
Both functions take the same keyword arguments:
|
|
99
|
+
|
|
100
|
+
- `processes`: the number of workers, or `None` for the backend's default; `1`
|
|
101
|
+
runs serially.
|
|
102
|
+
- `chunksize`: the number of calls sent to a worker at a time. Raise it when
|
|
103
|
+
each call is quick, to cut the overhead per call.
|
|
104
|
+
- `return_results`: `False` to discard the results, e.g. when `f` writes its
|
|
105
|
+
output to files.
|
|
106
|
+
|
|
107
|
+
An exception raised by `f` propagates to the caller, whatever the backend, and
|
|
108
|
+
whether or not the results are returned. For `mpi_simple`, it is raised on every
|
|
109
|
+
rank, so that no rank waits forever for the one that failed.
|
|
110
|
+
|
|
111
|
+
Except for `multithreading` and `serial`, `f`, its arguments and its results go
|
|
112
|
+
to other processes or interpreters, so they must be picklable: define `f` at
|
|
113
|
+
the top level of a module, and guard the script's entry point with
|
|
114
|
+
`if __name__ == "__main__":`.
|
|
115
|
+
|
|
116
|
+
The MPI modes run under `mpiexec`:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
|
|
120
|
+
mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Documentation
|
|
124
|
+
|
|
125
|
+
<https://ickc.github.io/python-map_parallel>
|
|
126
|
+
|
|
127
|
+
## Citation
|
|
128
|
+
|
|
129
|
+
If map_parallel contributes to your research, please cite it through its
|
|
130
|
+
[Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
|
|
131
|
+
the latest version.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
map_parallel/__init__.py,sha256=xc4ksJlD5W2J-6tycroTPNXJZi4U7359QbSO4simLL4,15589
|
|
2
|
+
map_parallel/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
map_parallel-0.3.0.dist-info/licenses/LICENSE,sha256=pi20xUSVtptZkx-fV-iprBSzsNtyzxWbK0Y3153oOBM,1516
|
|
4
|
+
map_parallel-0.3.0.dist-info/WHEEL,sha256=G_-rrEibZr7OEM7zdH1dSO6s-VdjfnS5PA4CLAdI9Vw,80
|
|
5
|
+
map_parallel-0.3.0.dist-info/METADATA,sha256=Aeoi11BDjP48J4rShK1KhWhMc4IAY-TU7nB1Frbu9sI,6253
|
|
6
|
+
map_parallel-0.3.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020, Kolen Cheung. All rights reserved.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
|
6
|
+
|
|
7
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
8
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
9
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
10
|
+
|
|
11
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|