map-parallel 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,458 @@
1
+ """A drop-in replacement of :func:`map` and :func:`itertools.starmap`, but in parallel with different backends.
2
+
3
+ >>> from map_parallel import map_parallel, starmap_parallel
4
+ >>> map_parallel(pow, [1, 2, 3], [2, 2, 2], mode="multithreading")
5
+ [1, 4, 9]
6
+ >>> starmap_parallel(pow, [(1, 2), (2, 2), (3, 2)], mode="multiprocessing", processes=2)
7
+ [1, 4, 9]
8
+
9
+ Exceptions raised by the mapped function propagate to the caller, whether the
10
+ results are returned or not:
11
+
12
+ >>> map_parallel(int, ["1", "x"], mode="multithreading", return_results=False)
13
+ Traceback (most recent call last):
14
+ ...
15
+ ValueError: invalid literal for int() with base 10: 'x'
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import sys
21
+ from collections import deque
22
+ from collections.abc import Callable, Iterable, Iterator
23
+ from functools import partial
24
+ from importlib.metadata import version
25
+ from itertools import chain, islice, starmap
26
+ from typing import Any, Literal, TypeVar
27
+
28
+ __version__ = version("map-parallel")
29
+
30
+ __all__ = [
31
+ "MODES",
32
+ "Mode",
33
+ "map_parallel",
34
+ "starmap_parallel",
35
+ ]
36
+
37
+ T = TypeVar("T")
38
+
39
+ Mode = Literal[
40
+ "multiprocessing",
41
+ "multithreading",
42
+ "multiinterpreter",
43
+ "dask",
44
+ "mpi",
45
+ "mpi_simple",
46
+ "serial",
47
+ ]
48
+
49
+ # helpers ######################################################################
50
+
51
+
52
+ def _starmap_chunk(f: Callable[..., T], chunk: Iterable[tuple[Any, ...]]) -> list[T]:
53
+ """Apply ``f`` to each tuple of arguments in a chunk, in the worker.
54
+
55
+ Module-level, so that it pickles by reference.
56
+ """
57
+ return list(starmap(f, chunk))
58
+
59
+
60
+ def _batched(iterable: Iterable[T], n: int) -> Iterator[tuple[T, ...]]:
61
+ """:func:`itertools.batched`, which is new in Python 3.12."""
62
+ iterator = iter(iterable)
63
+ while batch := tuple(islice(iterator, n)):
64
+ yield batch
65
+
66
+
67
+ def _collect(results: Iterable[T], return_results: bool) -> list[T]:
68
+ """Exhaust ``results``, so that an exception raised by any task propagates.
69
+
70
+ An executor raises the exception of a task only when its result is
71
+ retrieved, so the results are iterated even when they are not kept
72
+ (https://github.com/ickc/python-map_parallel/issues/2).
73
+ """
74
+ if return_results:
75
+ return list(results)
76
+ deque(results, maxlen=0)
77
+ return []
78
+
79
+
80
+ def _set_sys_path(path: tuple[str, ...]) -> None:
81
+ """Initialize a worker interpreter with the main interpreter's ``sys.path``.
82
+
83
+ A new interpreter computes ``sys.path`` afresh, missing changes made at run
84
+ time (by pytest, or a script's own directory), so functions defined in such
85
+ modules would not unpickle in the workers.
86
+ """
87
+ sys.path[:] = path
88
+
89
+
90
+ # backends #####################################################################
91
+ # Each backend computes ``f(*args)`` for each ``args`` in ``iterable``, in order.
92
+
93
+
94
+ def _starmap_serial(
95
+ f: Callable[..., T],
96
+ iterable: Iterable[tuple[Any, ...]],
97
+ *,
98
+ processes: int | None = None,
99
+ return_results: bool = True,
100
+ chunksize: int = 1,
101
+ ) -> list[T]:
102
+ return _collect(starmap(f, iterable), return_results)
103
+
104
+
105
+ def _starmap_executor(
106
+ executor: Any,
107
+ f: Callable[..., T],
108
+ iterable: Iterable[tuple[Any, ...]],
109
+ *,
110
+ return_results: bool,
111
+ chunksize: int,
112
+ ) -> list[T]:
113
+ """Run on a :class:`concurrent.futures.Executor`, ``chunksize`` tasks per submission.
114
+
115
+ On an exception, the result iterator of ``Executor.map`` cancels the tasks
116
+ not yet started, and leaving the ``with`` block waits for those running.
117
+ """
118
+ with executor:
119
+ chunks = executor.map(partial(_starmap_chunk, f), _batched(iterable, chunksize))
120
+ return _collect(chain.from_iterable(chunks), return_results)
121
+
122
+
123
+ def _starmap_multiprocessing(
124
+ f: Callable[..., T],
125
+ iterable: Iterable[tuple[Any, ...]],
126
+ *,
127
+ processes: int | None = None,
128
+ return_results: bool = True,
129
+ chunksize: int = 1,
130
+ ) -> list[T]:
131
+ from concurrent.futures import ProcessPoolExecutor
132
+
133
+ return _starmap_executor(
134
+ ProcessPoolExecutor(max_workers=processes),
135
+ f,
136
+ iterable,
137
+ return_results=return_results,
138
+ chunksize=chunksize,
139
+ )
140
+
141
+
142
+ def _starmap_multithreading(
143
+ f: Callable[..., T],
144
+ iterable: Iterable[tuple[Any, ...]],
145
+ *,
146
+ processes: int | None = None,
147
+ return_results: bool = True,
148
+ chunksize: int = 1,
149
+ ) -> list[T]:
150
+ from concurrent.futures import ThreadPoolExecutor
151
+
152
+ return _starmap_executor(
153
+ ThreadPoolExecutor(max_workers=processes),
154
+ f,
155
+ iterable,
156
+ return_results=return_results,
157
+ chunksize=chunksize,
158
+ )
159
+
160
+
161
+ def _starmap_multiinterpreter(
162
+ f: Callable[..., T],
163
+ iterable: Iterable[tuple[Any, ...]],
164
+ *,
165
+ processes: int | None = None,
166
+ return_results: bool = True,
167
+ chunksize: int = 1,
168
+ ) -> list[T]:
169
+ try:
170
+ from concurrent.futures import InterpreterPoolExecutor
171
+ except ImportError:
172
+ raise ImportError(
173
+ "mode='multiinterpreter' needs concurrent.futures.InterpreterPoolExecutor, new in Python 3.14; "
174
+ f"this is Python {sys.version_info.major}.{sys.version_info.minor}."
175
+ ) from None
176
+
177
+ return _starmap_executor(
178
+ InterpreterPoolExecutor(
179
+ max_workers=processes,
180
+ initializer=_set_sys_path,
181
+ initargs=(tuple(sys.path),),
182
+ ),
183
+ f,
184
+ iterable,
185
+ return_results=return_results,
186
+ chunksize=chunksize,
187
+ )
188
+
189
+
190
+ def _starmap_dask_client(
191
+ client: Any,
192
+ f: Callable[..., T],
193
+ iterable: Iterable[tuple[Any, ...]],
194
+ *,
195
+ return_results: bool,
196
+ chunksize: int,
197
+ ) -> list[T]:
198
+ from dask.utils import funcname
199
+ from distributed import wait
200
+
201
+ futures = client.map(
202
+ partial(_starmap_chunk, f),
203
+ list(_batched(iterable, chunksize)),
204
+ # f may be impure, so that equal arguments must not share one result
205
+ pure=False,
206
+ key=funcname(f),
207
+ )
208
+ try:
209
+ if return_results:
210
+ return [result for chunk in client.gather(futures) for result in chunk]
211
+ wait(futures)
212
+ for future in futures:
213
+ if future.status == "error":
214
+ future.result()
215
+ return []
216
+ except BaseException:
217
+ client.cancel(futures)
218
+ raise
219
+
220
+
221
+ def _starmap_dask(
222
+ f: Callable[..., T],
223
+ iterable: Iterable[tuple[Any, ...]],
224
+ *,
225
+ processes: int | None = None,
226
+ return_results: bool = True,
227
+ chunksize: int = 1,
228
+ ) -> list[T]:
229
+ from distributed import Client, LocalCluster, default_client
230
+
231
+ kwargs = {"return_results": return_results, "chunksize": chunksize}
232
+ try:
233
+ client = default_client()
234
+ except ValueError:
235
+ pass
236
+ else:
237
+ return _starmap_dask_client(client, f, iterable, **kwargs)
238
+ cluster_kwargs: dict[str, Any] = {"dashboard_address": None}
239
+ if processes is not None:
240
+ cluster_kwargs.update(n_workers=processes, threads_per_worker=1)
241
+ with LocalCluster(**cluster_kwargs) as cluster, Client(cluster) as client:
242
+ return _starmap_dask_client(client, f, iterable, **kwargs)
243
+
244
+
245
+ def _starmap_mpi(
246
+ f: Callable[..., T],
247
+ iterable: Iterable[tuple[Any, ...]],
248
+ *,
249
+ processes: int | None = None,
250
+ return_results: bool = True,
251
+ chunksize: int = 1,
252
+ ) -> list[T]:
253
+ from mpi4py.futures import MPIPoolExecutor
254
+
255
+ with MPIPoolExecutor(max_workers=processes) as executor:
256
+ return _collect(executor.starmap(f, iterable, chunksize=chunksize), return_results)
257
+
258
+
259
+ class _RemoteTraceback(Exception):
260
+ """The formatted traceback of an exception raised on another MPI rank."""
261
+
262
+ def __init__(self, tb: str) -> None:
263
+ self.tb = tb
264
+
265
+ def __str__(self) -> str:
266
+ return self.tb
267
+
268
+
269
+ def _exception_payload(exc: BaseException) -> tuple[str, bytes]:
270
+ """The traceback of ``exc`` as text, and ``exc`` pickled (or, if it does not pickle, its repr)."""
271
+ import pickle
272
+ from traceback import format_exception
273
+
274
+ tb = "".join(format_exception(exc))
275
+ try:
276
+ pickled = pickle.dumps(exc)
277
+ except Exception:
278
+ pickled = pickle.dumps(RuntimeError(repr(exc)))
279
+ return tb, pickled
280
+
281
+
282
+ def _exception_from_payload(rank: int, payload: tuple[str, bytes]) -> BaseException:
283
+ import pickle
284
+
285
+ tb, pickled = payload
286
+ try:
287
+ exc = pickle.loads(pickled)
288
+ except Exception:
289
+ exc = RuntimeError(f"an exception that does not unpickle was raised on MPI rank {rank}")
290
+ exc.__cause__ = _RemoteTraceback(f'\n"""\nraised on MPI rank {rank}:\n{tb}"""')
291
+ return exc
292
+
293
+
294
+ def _starmap_mpi_simple(
295
+ f: Callable[..., T],
296
+ iterable: Iterable[tuple[Any, ...]],
297
+ *,
298
+ processes: int | None = None,
299
+ return_results: bool = True,
300
+ chunksize: int = 1,
301
+ ) -> list[T]:
302
+ """Split the arguments into one contiguous block per rank of ``MPI.COMM_WORLD``.
303
+
304
+ Every rank must call it, with the same arguments. If ``f`` raises on any
305
+ rank, it raises on every rank: on the ranks where ``f`` raised, the
306
+ exception itself, and on the others the first rank's exception, with that
307
+ rank's traceback as its cause, as :mod:`concurrent.futures` does. Otherwise
308
+ the ranks that succeeded would wait forever for the ones that failed.
309
+ """
310
+ from mpi4py import MPI
311
+
312
+ comm = MPI.COMM_WORLD
313
+ size = comm.Get_size()
314
+ rank = comm.Get_rank()
315
+
316
+ args = list(iterable)
317
+ n = len(args)
318
+ local_args = args[(rank * n) // size : ((rank + 1) * n) // size]
319
+ del args
320
+
321
+ error: Exception | None = None
322
+ local_results: list[T] = []
323
+ try:
324
+ local_results = list(starmap(f, local_args))
325
+ except Exception as e:
326
+ error = e
327
+
328
+ # collective: every rank learns whether any rank failed
329
+ payloads = comm.allgather(None if error is None else _exception_payload(error))
330
+ if error is not None:
331
+ raise error
332
+ for other_rank, payload in enumerate(payloads):
333
+ if payload is not None:
334
+ raise _exception_from_payload(other_rank, payload)
335
+
336
+ if return_results:
337
+ gathered = comm.gather(local_results, root=0)
338
+ if rank == 0:
339
+ return [result for block in gathered for result in block]
340
+ return []
341
+
342
+
343
+ _BACKENDS: dict[str, Callable[..., list[Any]]] = {
344
+ "multiprocessing": _starmap_multiprocessing,
345
+ "multithreading": _starmap_multithreading,
346
+ "multiinterpreter": _starmap_multiinterpreter,
347
+ "dask": _starmap_dask,
348
+ "mpi": _starmap_mpi,
349
+ "mpi_simple": _starmap_mpi_simple,
350
+ "serial": _starmap_serial,
351
+ }
352
+
353
+ MODES: tuple[str, ...] = tuple(_BACKENDS)
354
+ """The valid values of ``mode``."""
355
+
356
+
357
+ def _validate(mode: str, chunksize: int) -> Callable[..., list[Any]]:
358
+ try:
359
+ backend = _BACKENDS[mode]
360
+ except (KeyError, TypeError):
361
+ raise ValueError(f"mode must be one of {', '.join(map(repr, MODES))}; got {mode!r}.") from None
362
+ if not isinstance(chunksize, int) or chunksize < 1:
363
+ raise ValueError(f"chunksize must be a positive integer; got {chunksize!r}.")
364
+ return backend
365
+
366
+
367
+ # public API ###################################################################
368
+
369
+
370
+ def map_parallel(
371
+ f: Callable[..., T],
372
+ *args: Iterable[Any],
373
+ processes: int | None = None,
374
+ mode: Mode = "multiprocessing",
375
+ return_results: bool = True,
376
+ chunksize: int = 1,
377
+ ) -> list[T]:
378
+ """Equivalent to ``list(map(f, *args))``, but in parallel.
379
+
380
+ :param f: the function to apply. Except for ``multithreading`` and
381
+ ``serial``, it, its arguments and its results are sent to other
382
+ processes or interpreters, so they must be picklable. For
383
+ ``multiprocessing`` and the MPI modes, ``f`` pickles by reference, so
384
+ it must be defined at the top level of a module: not a lambda nor a
385
+ closure. ``dask`` (through cloudpickle) and ``multiinterpreter`` also
386
+ take a lambda.
387
+ :param args: iterables of the arguments of ``f``, as in :func:`map`. They
388
+ are consumed in full before any result is returned.
389
+ :param processes: the number of workers; ``None`` for the backend's
390
+ default. If ``processes <= 1``, run serially (in every MPI rank, for
391
+ ``mpi_simple``). For ``mpi``, the number of MPI processes is set by
392
+ ``mpiexec`` if the script is run with ``python -m mpi4py.futures``,
393
+ else it is the number of processes to spawn. For ``mpi_simple``, it is
394
+ the number of MPI processes, set by ``mpiexec`` only.
395
+ :param mode: the backend to run on:
396
+
397
+ - ``multiprocessing``: :class:`concurrent.futures.ProcessPoolExecutor`,
398
+ for CPU-bound work in pure Python on a build with a GIL.
399
+ - ``multithreading``: :class:`concurrent.futures.ThreadPoolExecutor`,
400
+ for I/O-bound work, work that releases the GIL (much of NumPy), or
401
+ any work on a free-threaded build of Python.
402
+ - ``multiinterpreter``: :class:`concurrent.futures.InterpreterPoolExecutor`,
403
+ Python 3.14 or later: one interpreter per worker, each with its own
404
+ GIL, in one process.
405
+ - ``dask``: dask.distributed. Runs on the active
406
+ :class:`distributed.Client` if there is one (``processes`` is then
407
+ ignored), else on a :class:`distributed.LocalCluster` of
408
+ ``processes`` single-threaded workers, closed afterwards.
409
+ - ``mpi``: :class:`mpi4py.futures.MPIPoolExecutor`. May not work,
410
+ depending on your MPI implementation; run with
411
+ ``mpiexec -n N python -m mpi4py.futures script.py``.
412
+ - ``mpi_simple``: :mod:`mpi4py`, splitting the work into equal
413
+ contiguous blocks, one per MPI rank. Every rank must call it with the
414
+ same arguments; run with ``mpiexec -n N python script.py``.
415
+ - ``serial``: :func:`map`.
416
+ :param return_results: if ``False``, return ``[]`` instead of the results,
417
+ which are then not sent back (for ``mpi_simple``, not gathered to rank
418
+ 0). Exceptions raised by ``f`` still propagate.
419
+ :param chunksize: the number of calls of ``f`` sent to a worker at a time.
420
+ Larger chunks cost less overhead when ``f`` is quick; smaller ones
421
+ balance the load better when its duration varies. Ignored by
422
+ ``mpi_simple`` and ``serial``.
423
+ :return: the results, in order. For ``mpi_simple``, on rank 0 only; the
424
+ other ranks get ``[]``.
425
+ :raises ValueError: if ``mode`` or ``chunksize`` is invalid.
426
+ :raises ImportError: if the backend of ``mode`` is not available: dask or
427
+ mpi4py is not installed, or ``multiinterpreter`` on Python < 3.14.
428
+
429
+ Any exception that ``f`` raises is re-raised, with the remote traceback as
430
+ its cause where the backend provides it.
431
+ """
432
+ backend = _validate(mode, chunksize)
433
+ if processes is not None and processes <= 1:
434
+ return _collect(map(f, *args), return_results)
435
+ # like map, stop at the shortest iterable
436
+ iterable = zip(*args, strict=False)
437
+ return backend(f, iterable, processes=processes, return_results=return_results, chunksize=chunksize)
438
+
439
+
440
+ def starmap_parallel(
441
+ f: Callable[..., T],
442
+ args: Iterable[Iterable[Any]],
443
+ processes: int | None = None,
444
+ mode: Mode = "multiprocessing",
445
+ return_results: bool = True,
446
+ chunksize: int = 1,
447
+ ) -> list[T]:
448
+ """Equivalent to ``list(itertools.starmap(f, args))``, but in parallel.
449
+
450
+ :param args: an iterable of iterables, each the arguments of one call of
451
+ ``f``.
452
+
453
+ See :func:`map_parallel` for the other parameters.
454
+ """
455
+ backend = _validate(mode, chunksize)
456
+ if processes is not None and processes <= 1:
457
+ return _collect(starmap(f, args), return_results)
458
+ return backend(f, map(tuple, args), processes=processes, return_results=return_results, chunksize=chunksize)
map_parallel/py.typed ADDED
File without changes
@@ -0,0 +1,131 @@
1
+ Metadata-Version: 2.4
2
+ Name: map-parallel
3
+ Version: 0.3.0
4
+ Summary: A drop-in replacement of map/starmap but in parallel with different backends.
5
+ Keywords: map,starmap,parallel,concurrency,multiprocessing,multithreading,subinterpreters,free-threading,dask,mpi,mpi4py
6
+ Author: Kolen Cheung
7
+ Author-email: Kolen Cheung <christian.kolen@gmail.com>
8
+ License-Expression: BSD-3-Clause
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Programming Language :: Python :: 3.15
22
+ Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
23
+ Classifier: Programming Language :: Python :: Implementation :: CPython
24
+ Classifier: Topic :: System :: Distributed Computing
25
+ Classifier: Topic :: Utilities
26
+ Classifier: Typing :: Typed
27
+ Requires-Dist: distributed>=2023.1 ; extra == 'dask'
28
+ Requires-Dist: mpi4py>=3.1 ; extra == 'mpi'
29
+ Requires-Python: >=3.11
30
+ Project-URL: Homepage, https://github.com/ickc/python-map_parallel
31
+ Project-URL: Documentation, https://ickc.github.io/python-map_parallel
32
+ Project-URL: Repository, https://github.com/ickc/python-map_parallel
33
+ Project-URL: Issues, https://github.com/ickc/python-map_parallel/issues
34
+ Project-URL: Changelog, https://github.com/ickc/python-map_parallel/blob/main/CHANGELOG.md
35
+ Provides-Extra: dask
36
+ Provides-Extra: mpi
37
+ Description-Content-Type: text/markdown
38
+
39
+ # map_parallel
40
+
41
+ [![Test](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml/badge.svg)](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
42
+ [![Documentation](https://github.com/ickc/python-map_parallel/actions/workflows/docs.yml/badge.svg)](https://ickc.github.io/python-map_parallel)
43
+ [![PyPI](https://img.shields.io/pypi/v/map-parallel.svg)](https://pypi.org/project/map-parallel)
44
+ [![Python versions](https://img.shields.io/pypi/pyversions/map-parallel.svg)](https://pypi.org/project/map-parallel)
45
+ [![conda-forge](https://img.shields.io/conda/vn/conda-forge/map_parallel.svg)](https://anaconda.org/conda-forge/map_parallel)
46
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4641295.svg)](https://doi.org/10.5281/zenodo.4641295)
47
+ [![License](https://img.shields.io/pypi/l/map-parallel.svg)](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
48
+
49
+ A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
50
+ backends to choose from: processes, threads, subinterpreters, dask and MPI.
51
+
52
+ ```python
53
+ from map_parallel import map_parallel, starmap_parallel
54
+
55
+
56
+ def f(x, y):
57
+ return x**y
58
+
59
+
60
+ if __name__ == "__main__":
61
+ # list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
62
+ map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
63
+ # list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
64
+ starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
65
+ ```
66
+
67
+ The same code runs on any backend: change `mode`, and nothing else.
68
+
69
+ ## Installation
70
+
71
+ ```sh
72
+ pip install map-parallel
73
+ # with the optional backends
74
+ pip install "map-parallel[dask]"
75
+ pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
76
+ # or
77
+ conda install -c conda-forge map_parallel
78
+ ```
79
+
80
+ map_parallel itself is pure Python with no dependencies, supports Python 3.11
81
+ and later, and is tested on free-threaded builds of Python (3.13t on).
82
+
83
+ ## Backends
84
+
85
+ | `mode` | runs on | suits |
86
+ | --- | --- | --- |
87
+ | `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
88
+ | `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
89
+ | `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
90
+ | `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
91
+ | `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
92
+ | `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
93
+ | `serial` | `map` | debugging |
94
+
95
+ [Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
96
+ explains which to use when, and why.
97
+
98
+ Both functions take the same keyword arguments:
99
+
100
+ - `processes`: the number of workers, or `None` for the backend's default; `1`
101
+ runs serially.
102
+ - `chunksize`: the number of calls sent to a worker at a time. Raise it when
103
+ each call is quick, to cut the overhead per call.
104
+ - `return_results`: `False` to discard the results, e.g. when `f` writes its
105
+ output to files.
106
+
107
+ An exception raised by `f` propagates to the caller, whatever the backend, and
108
+ whether or not the results are returned. For `mpi_simple`, it is raised on every
109
+ rank, so that no rank waits forever for the one that failed.
110
+
111
+ Except for `multithreading` and `serial`, `f`, its arguments and its results go
112
+ to other processes or interpreters, so they must be picklable: define `f` at
113
+ the top level of a module, and guard the script's entry point with
114
+ `if __name__ == "__main__":`.
115
+
116
+ The MPI modes run under `mpiexec`:
117
+
118
+ ```sh
119
+ mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
120
+ mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
121
+ ```
122
+
123
+ ## Documentation
124
+
125
+ <https://ickc.github.io/python-map_parallel>
126
+
127
+ ## Citation
128
+
129
+ If map_parallel contributes to your research, please cite it through its
130
+ [Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
131
+ the latest version.
@@ -0,0 +1,6 @@
1
+ map_parallel/__init__.py,sha256=xc4ksJlD5W2J-6tycroTPNXJZi4U7359QbSO4simLL4,15589
2
+ map_parallel/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ map_parallel-0.3.0.dist-info/licenses/LICENSE,sha256=pi20xUSVtptZkx-fV-iprBSzsNtyzxWbK0Y3153oOBM,1516
4
+ map_parallel-0.3.0.dist-info/WHEEL,sha256=G_-rrEibZr7OEM7zdH1dSO6s-VdjfnS5PA4CLAdI9Vw,80
5
+ map_parallel-0.3.0.dist-info/METADATA,sha256=Aeoi11BDjP48J4rShK1KhWhMc4IAY-TU7nB1Frbu9sI,6253
6
+ map_parallel-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.13.9
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,11 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2020, Kolen Cheung. All rights reserved.
4
+
5
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6
+
7
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
9
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
10
+
11
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.