map-parallel 0.1.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. map_parallel-0.3.0/CHANGELOG.md +57 -0
  2. map_parallel-0.3.0/PKG-INFO +131 -0
  3. map_parallel-0.3.0/README.md +93 -0
  4. map_parallel-0.3.0/docs/backends.md +121 -0
  5. map_parallel-0.3.0/docs/changelog.md +2 -0
  6. map_parallel-0.3.0/docs/conf.py +37 -0
  7. map_parallel-0.3.0/docs/contributing.md +2 -0
  8. map_parallel-0.3.0/docs/index.md +12 -0
  9. map_parallel-0.3.0/pyproject.toml +290 -0
  10. map_parallel-0.3.0/pyproject.toml.orig +210 -0
  11. map_parallel-0.3.0/src/map_parallel/__init__.py +458 -0
  12. map_parallel-0.3.0/src/map_parallel/py.typed +0 -0
  13. map_parallel-0.3.0/tests/conftest.py +2 -0
  14. map_parallel-0.3.0/tests/mpi/check.py +76 -0
  15. map_parallel-0.3.0/tests/test_core.py +245 -0
  16. map_parallel-0.3.0/tests/test_dask.py +97 -0
  17. map_parallel-0.3.0/tests/test_mpi.py +92 -0
  18. map_parallel-0.3.0/tests/workloads.py +67 -0
  19. map-parallel-0.1.1/.bumpversion.cfg +0 -20
  20. map-parallel-0.1.1/.cookiecutterrc +0 -70
  21. map-parallel-0.1.1/.coveragerc +0 -17
  22. map-parallel-0.1.1/.editorconfig +0 -20
  23. map-parallel-0.1.1/.readthedocs.yml +0 -10
  24. map-parallel-0.1.1/AUTHORS.rst +0 -5
  25. map-parallel-0.1.1/CHANGELOG.rst +0 -8
  26. map-parallel-0.1.1/CONTRIBUTING.rst +0 -87
  27. map-parallel-0.1.1/MANIFEST.in +0 -19
  28. map-parallel-0.1.1/PKG-INFO +0 -90
  29. map-parallel-0.1.1/README.rst +0 -116
  30. map-parallel-0.1.1/ci/bootstrap.py +0 -89
  31. map-parallel-0.1.1/ci/requirements.txt +0 -4
  32. map-parallel-0.1.1/docs/authors.rst +0 -1
  33. map-parallel-0.1.1/docs/changelog.rst +0 -1
  34. map-parallel-0.1.1/docs/conf.py +0 -47
  35. map-parallel-0.1.1/docs/contributing.rst +0 -1
  36. map-parallel-0.1.1/docs/index.rst +0 -22
  37. map-parallel-0.1.1/docs/installation.rst +0 -7
  38. map-parallel-0.1.1/docs/readme.rst +0 -1
  39. map-parallel-0.1.1/docs/reference/index.rst +0 -7
  40. map-parallel-0.1.1/docs/reference/map_parallel.rst +0 -9
  41. map-parallel-0.1.1/docs/requirements.txt +0 -2
  42. map-parallel-0.1.1/docs/spelling_wordlist.txt +0 -11
  43. map-parallel-0.1.1/docs/usage.rst +0 -7
  44. map-parallel-0.1.1/setup.cfg +0 -35
  45. map-parallel-0.1.1/setup.py +0 -82
  46. map-parallel-0.1.1/src/map_parallel/__init__.py +0 -155
  47. map-parallel-0.1.1/src/map_parallel.egg-info/PKG-INFO +0 -90
  48. map-parallel-0.1.1/src/map_parallel.egg-info/SOURCES.txt +0 -37
  49. map-parallel-0.1.1/src/map_parallel.egg-info/dependency_links.txt +0 -1
  50. map-parallel-0.1.1/src/map_parallel.egg-info/not-zip-safe +0 -1
  51. map-parallel-0.1.1/src/map_parallel.egg-info/top_level.txt +0 -1
  52. map-parallel-0.1.1/tests/map_parallel_mpi_test.py +0 -20
  53. map-parallel-0.1.1/tests/starmap_parallel_mpi_test.py +0 -20
  54. map-parallel-0.1.1/tests/test_map_parallel.py +0 -39
  55. map-parallel-0.1.1/tox.ini +0 -103
  56. {map-parallel-0.1.1 → map_parallel-0.3.0}/LICENSE +0 -0
@@ -0,0 +1,57 @@
1
+ # Changelog
2
+
3
+ ## v0.3.0 (2026-11-20)
4
+
5
+ Fixes:
6
+
7
+ - An exception raised by the mapped function now always propagates. With
8
+ `return_results=False`, it was silently discarded
9
+ ([#2](https://github.com/ickc/python-map_parallel/issues/2)). For
10
+ `mpi_simple`, it is now raised on every rank, where the other ranks used to
11
+ wait for the failed one forever.
12
+ - An unknown `mode` raises `ValueError`, where it used to run serially, so that
13
+ a typo such as `mode="multiprocesing"` went unnoticed. **This may break code
14
+ that relied on it**; `mode="serial"` is the way to run serially.
15
+ - `starmap_parallel` on the `multiprocessing`, `multithreading` and `dask`
16
+ backends calls `f(*args)` for each tuple of arguments, as
17
+ `itertools.starmap` does, where tuples of unequal lengths were mixed up.
18
+ - `dask`: equal arguments are no longer taken for one call whose result is
19
+ reused (`pure=False`), and the `LocalCluster` and `Client` are closed after
20
+ the call, where they were left running.
21
+ - `return_results=False` returns `[]` whatever the mode or the number of
22
+ processes; serially it returned the results.
23
+ - `mpi_simple` joins the ranks' results in one pass, where `sum(..., [])` copied
24
+ them once per rank.
25
+
26
+ New:
27
+
28
+ - `mode="multiinterpreter"`, on `concurrent.futures.InterpreterPoolExecutor`
29
+ (Python 3.14 on): parallel pure Python in one process.
30
+ - `chunksize`, for every backend but `mpi_simple` and `serial`: the number of
31
+ calls sent to a worker at a time, to cut the overhead when each call is quick.
32
+ - `dask` runs on the active `distributed.Client` if there is one, so on any
33
+ cluster it is connected to; and with `processes`, starts that many
34
+ single-threaded workers.
35
+ - `mpi` passes `processes` to `MPIPoolExecutor` as `max_workers`, which matters
36
+ when it spawns its workers (without `python -m mpi4py.futures`).
37
+ - `map_parallel.MODES`, the valid values of `mode`, and type hints (`py.typed`).
38
+ - Tested on Python 3.11 to 3.15 and on the free-threaded builds 3.13t to 3.15t,
39
+ on Linux (x86-64 and ARM64), macOS (x86-64 and ARM64) and Windows (x86-64).
40
+ - The docs, at <https://ickc.github.io/python-map_parallel>, have a page on
41
+ choosing a backend.
42
+
43
+ Packaging:
44
+
45
+ - Require Python 3.11 or later.
46
+ - The optional dependencies are extras: `map-parallel[dask]`,
47
+ `map-parallel[mpi]`.
48
+ - Built with uv_build and released from GitHub Actions through trusted
49
+ publishing; developed with pixi.
50
+
51
+ ## v0.2.0 (2021-06-09)
52
+
53
+ - Add the `mpi_simple` mode.
54
+
55
+ ## v0.1.1 (2020-11-20)
56
+
57
+ - First release on PyPI.
@@ -0,0 +1,131 @@
1
+ Metadata-Version: 2.4
2
+ Name: map-parallel
3
+ Version: 0.3.0
4
+ Summary: A drop-in replacement of map/starmap but in parallel with different backends.
5
+ Keywords: map,starmap,parallel,concurrency,multiprocessing,multithreading,subinterpreters,free-threading,dask,mpi,mpi4py
6
+ Author: Kolen Cheung
7
+ Author-email: Kolen Cheung <christian.kolen@gmail.com>
8
+ License-Expression: BSD-3-Clause
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Programming Language :: Python :: 3.15
22
+ Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
23
+ Classifier: Programming Language :: Python :: Implementation :: CPython
24
+ Classifier: Topic :: System :: Distributed Computing
25
+ Classifier: Topic :: Utilities
26
+ Classifier: Typing :: Typed
27
+ Requires-Dist: distributed>=2023.1 ; extra == 'dask'
28
+ Requires-Dist: mpi4py>=3.1 ; extra == 'mpi'
29
+ Requires-Python: >=3.11
30
+ Project-URL: Homepage, https://github.com/ickc/python-map_parallel
31
+ Project-URL: Documentation, https://ickc.github.io/python-map_parallel
32
+ Project-URL: Repository, https://github.com/ickc/python-map_parallel
33
+ Project-URL: Issues, https://github.com/ickc/python-map_parallel/issues
34
+ Project-URL: Changelog, https://github.com/ickc/python-map_parallel/blob/main/CHANGELOG.md
35
+ Provides-Extra: dask
36
+ Provides-Extra: mpi
37
+ Description-Content-Type: text/markdown
38
+
39
+ # map_parallel
40
+
41
+ [![Test](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml/badge.svg)](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
42
+ [![Documentation](https://github.com/ickc/python-map_parallel/actions/workflows/docs.yml/badge.svg)](https://ickc.github.io/python-map_parallel)
43
+ [![PyPI](https://img.shields.io/pypi/v/map-parallel.svg)](https://pypi.org/project/map-parallel)
44
+ [![Python versions](https://img.shields.io/pypi/pyversions/map-parallel.svg)](https://pypi.org/project/map-parallel)
45
+ [![conda-forge](https://img.shields.io/conda/vn/conda-forge/map_parallel.svg)](https://anaconda.org/conda-forge/map_parallel)
46
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4641295.svg)](https://doi.org/10.5281/zenodo.4641295)
47
+ [![License](https://img.shields.io/pypi/l/map-parallel.svg)](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
48
+
49
+ A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
50
+ backends to choose from: processes, threads, subinterpreters, dask and MPI.
51
+
52
+ ```python
53
+ from map_parallel import map_parallel, starmap_parallel
54
+
55
+
56
+ def f(x, y):
57
+ return x**y
58
+
59
+
60
+ if __name__ == "__main__":
61
+ # list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
62
+ map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
63
+ # list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
64
+ starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
65
+ ```
66
+
67
+ The same code runs on any backend: change `mode`, and nothing else.
68
+
69
+ ## Installation
70
+
71
+ ```sh
72
+ pip install map-parallel
73
+ # with the optional backends
74
+ pip install "map-parallel[dask]"
75
+ pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
76
+ # or
77
+ conda install -c conda-forge map_parallel
78
+ ```
79
+
80
+ map_parallel itself is pure Python with no dependencies, supports Python 3.11
81
+ and later, and is tested on free-threaded builds of Python (3.13t on).
82
+
83
+ ## Backends
84
+
85
+ | `mode` | runs on | suits |
86
+ | --- | --- | --- |
87
+ | `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
88
+ | `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
89
+ | `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
90
+ | `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
91
+ | `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
92
+ | `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
93
+ | `serial` | `map` | debugging |
94
+
95
+ [Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
96
+ explains which to use when, and why.
97
+
98
+ Both functions take the same keyword arguments:
99
+
100
+ - `processes`: the number of workers, or `None` for the backend's default; `1`
101
+ runs serially.
102
+ - `chunksize`: the number of calls sent to a worker at a time. Raise it when
103
+ each call is quick, to cut the overhead per call.
104
+ - `return_results`: `False` to discard the results, e.g. when `f` writes its
105
+ output to files.
106
+
107
+ An exception raised by `f` propagates to the caller, whatever the backend, and
108
+ whether or not the results are returned. For `mpi_simple`, it is raised on every
109
+ rank, so that no rank waits forever for the one that failed.
110
+
111
+ Except for `multithreading` and `serial`, `f`, its arguments and its results go
112
+ to other processes or interpreters, so they must be picklable: define `f` at
113
+ the top level of a module, and guard the script's entry point with
114
+ `if __name__ == "__main__":`.
115
+
116
+ The MPI modes run under `mpiexec`:
117
+
118
+ ```sh
119
+ mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
120
+ mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
121
+ ```
122
+
123
+ ## Documentation
124
+
125
+ <https://ickc.github.io/python-map_parallel>
126
+
127
+ ## Citation
128
+
129
+ If map_parallel contributes to your research, please cite it through its
130
+ [Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
131
+ the latest version.
@@ -0,0 +1,93 @@
1
+ # map_parallel
2
+
3
+ [![Test](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml/badge.svg)](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
4
+ [![Documentation](https://github.com/ickc/python-map_parallel/actions/workflows/docs.yml/badge.svg)](https://ickc.github.io/python-map_parallel)
5
+ [![PyPI](https://img.shields.io/pypi/v/map-parallel.svg)](https://pypi.org/project/map-parallel)
6
+ [![Python versions](https://img.shields.io/pypi/pyversions/map-parallel.svg)](https://pypi.org/project/map-parallel)
7
+ [![conda-forge](https://img.shields.io/conda/vn/conda-forge/map_parallel.svg)](https://anaconda.org/conda-forge/map_parallel)
8
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4641295.svg)](https://doi.org/10.5281/zenodo.4641295)
9
+ [![License](https://img.shields.io/pypi/l/map-parallel.svg)](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
10
+
11
+ A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
12
+ backends to choose from: processes, threads, subinterpreters, dask and MPI.
13
+
14
+ ```python
15
+ from map_parallel import map_parallel, starmap_parallel
16
+
17
+
18
+ def f(x, y):
19
+ return x**y
20
+
21
+
22
+ if __name__ == "__main__":
23
+ # list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
24
+ map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
25
+ # list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
26
+ starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
27
+ ```
28
+
29
+ The same code runs on any backend: change `mode`, and nothing else.
30
+
31
+ ## Installation
32
+
33
+ ```sh
34
+ pip install map-parallel
35
+ # with the optional backends
36
+ pip install "map-parallel[dask]"
37
+ pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
38
+ # or
39
+ conda install -c conda-forge map_parallel
40
+ ```
41
+
42
+ map_parallel itself is pure Python with no dependencies, supports Python 3.11
43
+ and later, and is tested on free-threaded builds of Python (3.13t on).
44
+
45
+ ## Backends
46
+
47
+ | `mode` | runs on | suits |
48
+ | --- | --- | --- |
49
+ | `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
50
+ | `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
51
+ | `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
52
+ | `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
53
+ | `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
54
+ | `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
55
+ | `serial` | `map` | debugging |
56
+
57
+ [Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
58
+ explains which to use when, and why.
59
+
60
+ Both functions take the same keyword arguments:
61
+
62
+ - `processes`: the number of workers, or `None` for the backend's default; `1`
63
+ runs serially.
64
+ - `chunksize`: the number of calls sent to a worker at a time. Raise it when
65
+ each call is quick, to cut the overhead per call.
66
+ - `return_results`: `False` to discard the results, e.g. when `f` writes its
67
+ output to files.
68
+
69
+ An exception raised by `f` propagates to the caller, whatever the backend, and
70
+ whether or not the results are returned. For `mpi_simple`, it is raised on every
71
+ rank, so that no rank waits forever for the one that failed.
72
+
73
+ Except for `multithreading` and `serial`, `f`, its arguments and its results go
74
+ to other processes or interpreters, so they must be picklable: define `f` at
75
+ the top level of a module, and guard the script's entry point with
76
+ `if __name__ == "__main__":`.
77
+
78
+ The MPI modes run under `mpiexec`:
79
+
80
+ ```sh
81
+ mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
82
+ mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
83
+ ```
84
+
85
+ ## Documentation
86
+
87
+ <https://ickc.github.io/python-map_parallel>
88
+
89
+ ## Citation
90
+
91
+ If map_parallel contributes to your research, please cite it through its
92
+ [Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
93
+ the latest version.
@@ -0,0 +1,121 @@
1
+ # Choosing a backend
2
+
3
+ `map_parallel` runs the same map on any backend, so trying another is a change
4
+ of `mode`. Which is fastest depends on what `f` spends its time on, and on
5
+ what crosses between the workers and the caller.
6
+
7
+ ## What `f` spends its time on
8
+
9
+ **CPU-bound pure Python.** With a GIL, only one thread runs Python bytecode at a
10
+ time, so `multithreading` is no faster than `serial`. `multiprocessing` and
11
+ `multiinterpreter` give each worker a GIL of its own. On a free-threaded build
12
+ (3.14t on), there is no GIL, and `multithreading` scales too, without starting
13
+ processes or interpreters.
14
+
15
+ **I/O-bound** (waiting on files, the network, subprocesses). Every backend
16
+ overlaps the waiting; `multithreading` costs the least. For code written with
17
+ `async`/`await`, `asyncio.gather` is the native way, and not a job for
18
+ `map_parallel`.
19
+
20
+ **Work that releases the GIL**: most of NumPy, SciPy, compression, hashing, and
21
+ many C extensions. `multithreading`, which shares memory and copies nothing.
22
+ How far it scales, though, depends on more than the GIL: work bound by memory
23
+ bandwidth (sorting or copying arrays larger than the caches) stops scaling once
24
+ the cores together saturate the memory bus, whatever the backend.
25
+
26
+ ## What crosses between the workers and the caller
27
+
28
+ Except for `multithreading` and `serial`, every argument and every result is
29
+ pickled, sent, and unpickled. For `multiprocessing` they go through a pipe, one
30
+ at a time through the caller. If `f` takes or returns large objects, such as
31
+ arrays of hundreds of megabytes, the transfer can cost more than `f` itself, and
32
+ the pool can be slower than `serial`. Then:
33
+
34
+ - send something small, from which the worker can make the large object itself:
35
+ a path, a seed, an index into a file;
36
+ - return something small, or write the output to a file in the worker and pass
37
+ `return_results=False`;
38
+ - or, if `f` releases the GIL, use `multithreading`, which copies nothing.
39
+
40
+ On Linux before Python 3.14, `multiprocessing` starts the workers by `fork`,
41
+ and a worker inherits the caller's memory as it was when the pool started. That shares
42
+ only what the worker inherits, not what it is sent: the arguments are pickled
43
+ whatever the start method.
44
+
45
+ ## Chunks
46
+
47
+ Each call has a fixed overhead: a future, a lock, and for the backends other
48
+ than `multithreading`, pickling. When `f` is quick, the overhead dominates;
49
+ `chunksize` sends that many calls to a worker at once. A larger `chunksize`
50
+ divides the overhead, at the cost of balancing the load less well when the
51
+ duration of `f` varies.
52
+
53
+ ## Measured
54
+
55
+ `tools/benchmark.py` on a Ryzen 9 9950X (16 cores, 32 threads), Python 3.15.2,
56
+ wall time:
57
+
58
+ - `cpu ×8`: 8 calls of CPU-bound pure Python, ~0.27 s each, on 8 workers;
59
+ - `io ×8`: 8 calls that each sleep 0.25 s, on 8 workers;
60
+ - `overhead`: 100,000 calls that do nearly nothing, on 4 workers, at
61
+ `chunksize` 1 and 100.
62
+
63
+ With the GIL:
64
+
65
+ | `mode` | cpu ×8 | io ×8 | overhead, chunksize 1 | chunksize 100 |
66
+ | --- | --: | --: | --: | --: |
67
+ | `serial` | 2.13 s | 2.00 s | 0.00 s | 0.00 s |
68
+ | `multiprocessing` | 0.32 s | 0.26 s | 2.72 s | 0.04 s |
69
+ | `multithreading` | 2.14 s | 0.25 s | 0.55 s | 0.02 s |
70
+ | `multiinterpreter` | 0.33 s | 0.29 s | 2.09 s | 0.08 s |
71
+
72
+ Free-threaded:
73
+
74
+ | `mode` | cpu ×8 | io ×8 | overhead, chunksize 1 | chunksize 100 |
75
+ | --- | --: | --: | --: | --: |
76
+ | `serial` | 2.20 s | 2.00 s | 0.00 s | 0.00 s |
77
+ | `multiprocessing` | 0.34 s | 0.26 s | 1.67 s | 0.05 s |
78
+ | `multithreading` | 0.31 s | 0.25 s | 0.38 s | 0.01 s |
79
+ | `multiinterpreter` | 0.52 s | 0.52 s | 1.16 s | 0.15 s |
80
+
81
+ - With the GIL, CPU-bound work in threads takes as long as `serial`; without
82
+ it, threads do as well as processes.
83
+ - Every backend overlaps waiting.
84
+ - One call at a time, the overhead of a process or an interpreter is some
85
+ 20–30 µs a call, four to five times a thread's; in chunks of 100, it is
86
+ nearly gone. The
87
+ cost of starting the workers is in every number here, and is the most of
88
+ what `multiinterpreter` takes beyond the others: interpreters start more
89
+ slowly on the free-threaded build.
90
+
91
+ ## Beyond one machine
92
+
93
+ `dask` runs on the active `distributed.Client`, if there is one, so the same
94
+ call runs on whatever cluster the client is connected to: SLURM or PBS through
95
+ dask-jobqueue, Kubernetes, and so on. Without one, it starts a
96
+ `distributed.LocalCluster` of `processes` workers, which is useful mostly for
97
+ trying the code out before a real cluster.
98
+
99
+ `mpi` and `mpi_simple` run under `mpiexec`, across the nodes of an HPC job.
100
+ `mpi` hands out calls dynamically to workers as they become free, and suits
101
+ calls whose duration varies; `mpi_simple` splits the calls into one contiguous
102
+ block per rank up front, with no scheduler, and works with any MPI
103
+ implementation, including those that `mpi4py.futures` does not.
104
+
105
+ ## Gotchas
106
+
107
+ - **Picklability.** `f` must be importable by name in the workers: defined at
108
+ the top level of a module, not a lambda, a closure, or a method of a local
109
+ class. `functools.partial` of such a function works, which is how to pass
110
+ keyword arguments. `multithreading` and `serial` have no such limit.
111
+ `dask` pickles with cloudpickle, and `multiinterpreter` can share functions
112
+ that use no closure, so both accept lambdas too.
113
+ - **`if __name__ == "__main__":`.** Where workers start by `spawn` (macOS,
114
+ Windows) or `forkserver` (Linux from Python 3.14), each one imports the main
115
+ module afresh, which then must not start the map again at import.
116
+ - **Oversubscription.** `processes=None` is each backend's default: one
117
+ worker per core for `multiprocessing`, but `min(32, cores + 4)` for
118
+ `multithreading` and `multiinterpreter`, since I/O-bound work wants more
119
+ threads than cores; for CPU-bound work in interpreters, pass `processes`. If `f` is itself multi-threaded, as NumPy's
120
+ linear algebra is, many workers times many threads each oversubscribe the
121
+ cores; limit one or the other, e.g. with `OMP_NUM_THREADS=1`.
@@ -0,0 +1,2 @@
1
+ ```{include} ../CHANGELOG.md
2
+ ```
@@ -0,0 +1,37 @@
1
+ # https://www.sphinx-doc.org/en/master/usage/configuration.html
2
+ from importlib.metadata import version as get_version
3
+
4
+ project = "map_parallel"
5
+ author = "Kolen Cheung"
6
+ copyright = f"2020, {author}"
7
+ version = release = get_version("map-parallel")
8
+
9
+ extensions = [
10
+ "myst_parser",
11
+ "sphinx.ext.apidoc",
12
+ "sphinx.ext.autodoc",
13
+ "sphinx.ext.intersphinx",
14
+ "sphinx.ext.viewcode",
15
+ ]
16
+ source_suffix = {".md": "markdown", ".rst": "restructuredtext"}
17
+ exclude_patterns = ["_build"]
18
+
19
+ html_theme = "furo"
20
+ html_title = f"{project} {version}"
21
+
22
+ autodoc_member_order = "bysource"
23
+ apidoc_modules = [
24
+ {
25
+ "path": "../src/map_parallel",
26
+ "destination": "api",
27
+ "separate_modules": True,
28
+ "module_first": True,
29
+ },
30
+ ]
31
+ intersphinx_mapping = {
32
+ "python": ("https://docs.python.org/3", None),
33
+ "distributed": ("https://distributed.dask.org/en/stable", None),
34
+ "mpi4py": ("https://mpi4py.readthedocs.io/en/stable", None),
35
+ }
36
+
37
+ myst_heading_anchors = 3
@@ -0,0 +1,2 @@
1
+ ```{include} ../CONTRIBUTING.md
2
+ ```
@@ -0,0 +1,12 @@
1
+ ```{include} ../README.md
2
+ ```
3
+
4
+ ```{toctree}
5
+ :hidden:
6
+ :maxdepth: 2
7
+
8
+ backends
9
+ api/modules
10
+ changelog
11
+ contributing
12
+ ```