map-parallel 0.1.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- map_parallel-0.3.0/CHANGELOG.md +57 -0
- map_parallel-0.3.0/PKG-INFO +131 -0
- map_parallel-0.3.0/README.md +93 -0
- map_parallel-0.3.0/docs/backends.md +121 -0
- map_parallel-0.3.0/docs/changelog.md +2 -0
- map_parallel-0.3.0/docs/conf.py +37 -0
- map_parallel-0.3.0/docs/contributing.md +2 -0
- map_parallel-0.3.0/docs/index.md +12 -0
- map_parallel-0.3.0/pyproject.toml +290 -0
- map_parallel-0.3.0/pyproject.toml.orig +210 -0
- map_parallel-0.3.0/src/map_parallel/__init__.py +458 -0
- map_parallel-0.3.0/src/map_parallel/py.typed +0 -0
- map_parallel-0.3.0/tests/conftest.py +2 -0
- map_parallel-0.3.0/tests/mpi/check.py +76 -0
- map_parallel-0.3.0/tests/test_core.py +245 -0
- map_parallel-0.3.0/tests/test_dask.py +97 -0
- map_parallel-0.3.0/tests/test_mpi.py +92 -0
- map_parallel-0.3.0/tests/workloads.py +67 -0
- map-parallel-0.1.1/.bumpversion.cfg +0 -20
- map-parallel-0.1.1/.cookiecutterrc +0 -70
- map-parallel-0.1.1/.coveragerc +0 -17
- map-parallel-0.1.1/.editorconfig +0 -20
- map-parallel-0.1.1/.readthedocs.yml +0 -10
- map-parallel-0.1.1/AUTHORS.rst +0 -5
- map-parallel-0.1.1/CHANGELOG.rst +0 -8
- map-parallel-0.1.1/CONTRIBUTING.rst +0 -87
- map-parallel-0.1.1/MANIFEST.in +0 -19
- map-parallel-0.1.1/PKG-INFO +0 -90
- map-parallel-0.1.1/README.rst +0 -116
- map-parallel-0.1.1/ci/bootstrap.py +0 -89
- map-parallel-0.1.1/ci/requirements.txt +0 -4
- map-parallel-0.1.1/docs/authors.rst +0 -1
- map-parallel-0.1.1/docs/changelog.rst +0 -1
- map-parallel-0.1.1/docs/conf.py +0 -47
- map-parallel-0.1.1/docs/contributing.rst +0 -1
- map-parallel-0.1.1/docs/index.rst +0 -22
- map-parallel-0.1.1/docs/installation.rst +0 -7
- map-parallel-0.1.1/docs/readme.rst +0 -1
- map-parallel-0.1.1/docs/reference/index.rst +0 -7
- map-parallel-0.1.1/docs/reference/map_parallel.rst +0 -9
- map-parallel-0.1.1/docs/requirements.txt +0 -2
- map-parallel-0.1.1/docs/spelling_wordlist.txt +0 -11
- map-parallel-0.1.1/docs/usage.rst +0 -7
- map-parallel-0.1.1/setup.cfg +0 -35
- map-parallel-0.1.1/setup.py +0 -82
- map-parallel-0.1.1/src/map_parallel/__init__.py +0 -155
- map-parallel-0.1.1/src/map_parallel.egg-info/PKG-INFO +0 -90
- map-parallel-0.1.1/src/map_parallel.egg-info/SOURCES.txt +0 -37
- map-parallel-0.1.1/src/map_parallel.egg-info/dependency_links.txt +0 -1
- map-parallel-0.1.1/src/map_parallel.egg-info/not-zip-safe +0 -1
- map-parallel-0.1.1/src/map_parallel.egg-info/top_level.txt +0 -1
- map-parallel-0.1.1/tests/map_parallel_mpi_test.py +0 -20
- map-parallel-0.1.1/tests/starmap_parallel_mpi_test.py +0 -20
- map-parallel-0.1.1/tests/test_map_parallel.py +0 -39
- map-parallel-0.1.1/tox.ini +0 -103
- {map-parallel-0.1.1 → map_parallel-0.3.0}/LICENSE +0 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## v0.3.0 (2026-11-20)
|
|
4
|
+
|
|
5
|
+
Fixes:
|
|
6
|
+
|
|
7
|
+
- An exception raised by the mapped function now always propagates. With
|
|
8
|
+
`return_results=False`, it was silently discarded
|
|
9
|
+
([#2](https://github.com/ickc/python-map_parallel/issues/2)). For
|
|
10
|
+
`mpi_simple`, it is now raised on every rank, where the other ranks used to
|
|
11
|
+
wait for the failed one forever.
|
|
12
|
+
- An unknown `mode` raises `ValueError`, where it used to run serially, so that
|
|
13
|
+
a typo such as `mode="multiprocesing"` went unnoticed. **This may break code
|
|
14
|
+
that relied on it**; `mode="serial"` is the way to run serially.
|
|
15
|
+
- `starmap_parallel` on the `multiprocessing`, `multithreading` and `dask`
|
|
16
|
+
backends calls `f(*args)` for each tuple of arguments, as
|
|
17
|
+
`itertools.starmap` does, where tuples of unequal lengths were mixed up.
|
|
18
|
+
- `dask`: equal arguments are no longer taken for one call whose result is
|
|
19
|
+
reused (`pure=False`), and the `LocalCluster` and `Client` are closed after
|
|
20
|
+
the call, where they were left running.
|
|
21
|
+
- `return_results=False` returns `[]` whatever the mode or the number of
|
|
22
|
+
processes; serially it returned the results.
|
|
23
|
+
- `mpi_simple` joins the ranks' results in one pass, where `sum(..., [])` copied
|
|
24
|
+
them once per rank.
|
|
25
|
+
|
|
26
|
+
New:
|
|
27
|
+
|
|
28
|
+
- `mode="multiinterpreter"`, on `concurrent.futures.InterpreterPoolExecutor`
|
|
29
|
+
(Python 3.14 on): parallel pure Python in one process.
|
|
30
|
+
- `chunksize`, for every backend but `mpi_simple` and `serial`: the number of
|
|
31
|
+
calls sent to a worker at a time, to cut the overhead when each call is quick.
|
|
32
|
+
- `dask` runs on the active `distributed.Client` if there is one, so on any
|
|
33
|
+
cluster it is connected to; and with `processes`, starts that many
|
|
34
|
+
single-threaded workers.
|
|
35
|
+
- `mpi` passes `processes` to `MPIPoolExecutor` as `max_workers`, which matters
|
|
36
|
+
when it spawns its workers (without `python -m mpi4py.futures`).
|
|
37
|
+
- `map_parallel.MODES`, the valid values of `mode`, and type hints (`py.typed`).
|
|
38
|
+
- Tested on Python 3.11 to 3.15 and on the free-threaded builds 3.13t to 3.15t,
|
|
39
|
+
on Linux (x86-64 and ARM64), macOS (x86-64 and ARM64) and Windows (x86-64).
|
|
40
|
+
- The docs, at <https://ickc.github.io/python-map_parallel>, have a page on
|
|
41
|
+
choosing a backend.
|
|
42
|
+
|
|
43
|
+
Packaging:
|
|
44
|
+
|
|
45
|
+
- Require Python 3.11 or later.
|
|
46
|
+
- The optional dependencies are extras: `map-parallel[dask]`,
|
|
47
|
+
`map-parallel[mpi]`.
|
|
48
|
+
- Built with uv_build and released from GitHub Actions through trusted
|
|
49
|
+
publishing; developed with pixi.
|
|
50
|
+
|
|
51
|
+
## v0.2.0 (2021-06-09)
|
|
52
|
+
|
|
53
|
+
- Add the `mpi_simple` mode.
|
|
54
|
+
|
|
55
|
+
## v0.1.1 (2020-11-20)
|
|
56
|
+
|
|
57
|
+
- First release on PyPI.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: map-parallel
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A drop-in replacement of map/starmap but in parallel with different backends.
|
|
5
|
+
Keywords: map,starmap,parallel,concurrency,multiprocessing,multithreading,subinterpreters,free-threading,dask,mpi,mpi4py
|
|
6
|
+
Author: Kolen Cheung
|
|
7
|
+
Author-email: Kolen Cheung <christian.kolen@gmail.com>
|
|
8
|
+
License-Expression: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
|
+
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
23
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
24
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
25
|
+
Classifier: Topic :: Utilities
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Dist: distributed>=2023.1 ; extra == 'dask'
|
|
28
|
+
Requires-Dist: mpi4py>=3.1 ; extra == 'mpi'
|
|
29
|
+
Requires-Python: >=3.11
|
|
30
|
+
Project-URL: Homepage, https://github.com/ickc/python-map_parallel
|
|
31
|
+
Project-URL: Documentation, https://ickc.github.io/python-map_parallel
|
|
32
|
+
Project-URL: Repository, https://github.com/ickc/python-map_parallel
|
|
33
|
+
Project-URL: Issues, https://github.com/ickc/python-map_parallel/issues
|
|
34
|
+
Project-URL: Changelog, https://github.com/ickc/python-map_parallel/blob/main/CHANGELOG.md
|
|
35
|
+
Provides-Extra: dask
|
|
36
|
+
Provides-Extra: mpi
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# map_parallel
|
|
40
|
+
|
|
41
|
+
[](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
|
|
42
|
+
[](https://ickc.github.io/python-map_parallel)
|
|
43
|
+
[](https://pypi.org/project/map-parallel)
|
|
44
|
+
[](https://pypi.org/project/map-parallel)
|
|
45
|
+
[](https://anaconda.org/conda-forge/map_parallel)
|
|
46
|
+
[](https://doi.org/10.5281/zenodo.4641295)
|
|
47
|
+
[](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
|
|
48
|
+
|
|
49
|
+
A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
|
|
50
|
+
backends to choose from: processes, threads, subinterpreters, dask and MPI.
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from map_parallel import map_parallel, starmap_parallel
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def f(x, y):
|
|
57
|
+
return x**y
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == "__main__":
|
|
61
|
+
# list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
|
|
62
|
+
map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
|
|
63
|
+
# list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
|
|
64
|
+
starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The same code runs on any backend: change `mode`, and nothing else.
|
|
68
|
+
|
|
69
|
+
## Installation
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
pip install map-parallel
|
|
73
|
+
# with the optional backends
|
|
74
|
+
pip install "map-parallel[dask]"
|
|
75
|
+
pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
|
|
76
|
+
# or
|
|
77
|
+
conda install -c conda-forge map_parallel
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
map_parallel itself is pure Python with no dependencies, supports Python 3.11
|
|
81
|
+
and later, and is tested on free-threaded builds of Python (3.13t on).
|
|
82
|
+
|
|
83
|
+
## Backends
|
|
84
|
+
|
|
85
|
+
| `mode` | runs on | suits |
|
|
86
|
+
| --- | --- | --- |
|
|
87
|
+
| `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
|
|
88
|
+
| `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
|
|
89
|
+
| `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
|
|
90
|
+
| `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
|
|
91
|
+
| `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
|
|
92
|
+
| `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
|
|
93
|
+
| `serial` | `map` | debugging |
|
|
94
|
+
|
|
95
|
+
[Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
|
|
96
|
+
explains which to use when, and why.
|
|
97
|
+
|
|
98
|
+
Both functions take the same keyword arguments:
|
|
99
|
+
|
|
100
|
+
- `processes`: the number of workers, or `None` for the backend's default; `1`
|
|
101
|
+
runs serially.
|
|
102
|
+
- `chunksize`: the number of calls sent to a worker at a time. Raise it when
|
|
103
|
+
each call is quick, to cut the overhead per call.
|
|
104
|
+
- `return_results`: `False` to discard the results, e.g. when `f` writes its
|
|
105
|
+
output to files.
|
|
106
|
+
|
|
107
|
+
An exception raised by `f` propagates to the caller, whatever the backend, and
|
|
108
|
+
whether or not the results are returned. For `mpi_simple`, it is raised on every
|
|
109
|
+
rank, so that no rank waits forever for the one that failed.
|
|
110
|
+
|
|
111
|
+
Except for `multithreading` and `serial`, `f`, its arguments and its results go
|
|
112
|
+
to other processes or interpreters, so they must be picklable: define `f` at
|
|
113
|
+
the top level of a module, and guard the script's entry point with
|
|
114
|
+
`if __name__ == "__main__":`.
|
|
115
|
+
|
|
116
|
+
The MPI modes run under `mpiexec`:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
|
|
120
|
+
mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Documentation
|
|
124
|
+
|
|
125
|
+
<https://ickc.github.io/python-map_parallel>
|
|
126
|
+
|
|
127
|
+
## Citation
|
|
128
|
+
|
|
129
|
+
If map_parallel contributes to your research, please cite it through its
|
|
130
|
+
[Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
|
|
131
|
+
the latest version.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# map_parallel
|
|
2
|
+
|
|
3
|
+
[](https://github.com/ickc/python-map_parallel/actions/workflows/test.yml)
|
|
4
|
+
[](https://ickc.github.io/python-map_parallel)
|
|
5
|
+
[](https://pypi.org/project/map-parallel)
|
|
6
|
+
[](https://pypi.org/project/map-parallel)
|
|
7
|
+
[](https://anaconda.org/conda-forge/map_parallel)
|
|
8
|
+
[](https://doi.org/10.5281/zenodo.4641295)
|
|
9
|
+
[](https://github.com/ickc/python-map_parallel/blob/main/LICENSE)
|
|
10
|
+
|
|
11
|
+
A drop-in replacement of `map` and `itertools.starmap`, but in parallel, with
|
|
12
|
+
backends to choose from: processes, threads, subinterpreters, dask and MPI.
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from map_parallel import map_parallel, starmap_parallel
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def f(x, y):
|
|
19
|
+
return x**y
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
if __name__ == "__main__":
|
|
23
|
+
# list(map(f, [1, 2, 3], [2, 2, 2])), on a pool of processes
|
|
24
|
+
map_parallel(f, [1, 2, 3], [2, 2, 2]) # [1, 4, 9]
|
|
25
|
+
# list(itertools.starmap(f, [(1, 2), (2, 2), (3, 2)])), on 4 threads
|
|
26
|
+
starmap_parallel(f, [(1, 2), (2, 2), (3, 2)], mode="multithreading", processes=4) # [1, 4, 9]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The same code runs on any backend: change `mode`, and nothing else.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
pip install map-parallel
|
|
35
|
+
# with the optional backends
|
|
36
|
+
pip install "map-parallel[dask]"
|
|
37
|
+
pip install "map-parallel[mpi]" # needs an MPI implementation, see mpi4py's documentation
|
|
38
|
+
# or
|
|
39
|
+
conda install -c conda-forge map_parallel
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
map_parallel itself is pure Python with no dependencies, supports Python 3.11
|
|
43
|
+
and later, and is tested on free-threaded builds of Python (3.13t on).
|
|
44
|
+
|
|
45
|
+
## Backends
|
|
46
|
+
|
|
47
|
+
| `mode` | runs on | suits |
|
|
48
|
+
| --- | --- | --- |
|
|
49
|
+
| `multiprocessing` (default) | [`ProcessPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) | CPU-bound pure Python, with a GIL |
|
|
50
|
+
| `multithreading` | [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) | I/O-bound work; work that releases the GIL; anything, on free-threaded Python |
|
|
51
|
+
| `multiinterpreter` | [`InterpreterPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#interpreterpoolexecutor), Python 3.14+ | CPU-bound pure Python, in one process |
|
|
52
|
+
| `dask` | [dask.distributed](https://distributed.dask.org): the active `Client`, or a `LocalCluster` | a cluster you already have |
|
|
53
|
+
| `mpi` | [`mpi4py.futures.MPIPoolExecutor`](https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html) | HPC, with dynamic load balancing |
|
|
54
|
+
| `mpi_simple` | [mpi4py](https://mpi4py.readthedocs.io), one equal block of work per rank | HPC, where `mpi` does not work |
|
|
55
|
+
| `serial` | `map` | debugging |
|
|
56
|
+
|
|
57
|
+
[Choosing a backend](https://ickc.github.io/python-map_parallel/backends/)
|
|
58
|
+
explains which to use when, and why.
|
|
59
|
+
|
|
60
|
+
Both functions take the same keyword arguments:
|
|
61
|
+
|
|
62
|
+
- `processes`: the number of workers, or `None` for the backend's default; `1`
|
|
63
|
+
runs serially.
|
|
64
|
+
- `chunksize`: the number of calls sent to a worker at a time. Raise it when
|
|
65
|
+
each call is quick, to cut the overhead per call.
|
|
66
|
+
- `return_results`: `False` to discard the results, e.g. when `f` writes its
|
|
67
|
+
output to files.
|
|
68
|
+
|
|
69
|
+
An exception raised by `f` propagates to the caller, whatever the backend, and
|
|
70
|
+
whether or not the results are returned. For `mpi_simple`, it is raised on every
|
|
71
|
+
rank, so that no rank waits forever for the one that failed.
|
|
72
|
+
|
|
73
|
+
Except for `multithreading` and `serial`, `f`, its arguments and its results go
|
|
74
|
+
to other processes or interpreters, so they must be picklable: define `f` at
|
|
75
|
+
the top level of a module, and guard the script's entry point with
|
|
76
|
+
`if __name__ == "__main__":`.
|
|
77
|
+
|
|
78
|
+
The MPI modes run under `mpiexec`:
|
|
79
|
+
|
|
80
|
+
```sh
|
|
81
|
+
mpiexec -n 4 python -m mpi4py.futures script.py # mode="mpi"
|
|
82
|
+
mpiexec -n 4 python script.py # mode="mpi_simple": every rank runs the script
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Documentation
|
|
86
|
+
|
|
87
|
+
<https://ickc.github.io/python-map_parallel>
|
|
88
|
+
|
|
89
|
+
## Citation
|
|
90
|
+
|
|
91
|
+
If map_parallel contributes to your research, please cite it through its
|
|
92
|
+
[Zenodo record](https://doi.org/10.5281/zenodo.4641295), whose DOI resolves to
|
|
93
|
+
the latest version.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# Choosing a backend
|
|
2
|
+
|
|
3
|
+
`map_parallel` runs the same map on any backend, so trying another is a change
|
|
4
|
+
of `mode`. Which is fastest depends on what `f` spends its time on, and on
|
|
5
|
+
what crosses between the workers and the caller.
|
|
6
|
+
|
|
7
|
+
## What `f` spends its time on
|
|
8
|
+
|
|
9
|
+
**CPU-bound pure Python.** With a GIL, only one thread runs Python bytecode at a
|
|
10
|
+
time, so `multithreading` is no faster than `serial`. `multiprocessing` and
|
|
11
|
+
`multiinterpreter` give each worker a GIL of its own. On a free-threaded build
|
|
12
|
+
(3.14t on), there is no GIL, and `multithreading` scales too, without starting
|
|
13
|
+
processes or interpreters.
|
|
14
|
+
|
|
15
|
+
**I/O-bound** (waiting on files, the network, subprocesses). Every backend
|
|
16
|
+
overlaps the waiting; `multithreading` costs the least. For code written with
|
|
17
|
+
`async`/`await`, `asyncio.gather` is the native way, and not a job for
|
|
18
|
+
`map_parallel`.
|
|
19
|
+
|
|
20
|
+
**Work that releases the GIL**: most of NumPy, SciPy, compression, hashing, and
|
|
21
|
+
many C extensions. `multithreading`, which shares memory and copies nothing.
|
|
22
|
+
How far it scales, though, depends on more than the GIL: work bound by memory
|
|
23
|
+
bandwidth (sorting or copying arrays larger than the caches) stops scaling once
|
|
24
|
+
the cores together saturate the memory bus, whatever the backend.
|
|
25
|
+
|
|
26
|
+
## What crosses between the workers and the caller
|
|
27
|
+
|
|
28
|
+
Except for `multithreading` and `serial`, every argument and every result is
|
|
29
|
+
pickled, sent, and unpickled. For `multiprocessing` they go through a pipe, one
|
|
30
|
+
at a time through the caller. If `f` takes or returns large objects, such as
|
|
31
|
+
arrays of hundreds of megabytes, the transfer can cost more than `f` itself, and
|
|
32
|
+
the pool can be slower than `serial`. Then:
|
|
33
|
+
|
|
34
|
+
- send something small, from which the worker can make the large object itself:
|
|
35
|
+
a path, a seed, an index into a file;
|
|
36
|
+
- return something small, or write the output to a file in the worker and pass
|
|
37
|
+
`return_results=False`;
|
|
38
|
+
- or, if `f` releases the GIL, use `multithreading`, which copies nothing.
|
|
39
|
+
|
|
40
|
+
On Linux before Python 3.14, `multiprocessing` starts the workers by `fork`,
|
|
41
|
+
and a worker inherits the caller's memory as it was when the pool started. That shares
|
|
42
|
+
only what the worker inherits, not what it is sent: the arguments are pickled
|
|
43
|
+
whatever the start method.
|
|
44
|
+
|
|
45
|
+
## Chunks
|
|
46
|
+
|
|
47
|
+
Each call has a fixed overhead: a future, a lock, and for the backends other
|
|
48
|
+
than `multithreading`, pickling. When `f` is quick, the overhead dominates;
|
|
49
|
+
`chunksize` sends that many calls to a worker at once. A larger `chunksize`
|
|
50
|
+
divides the overhead, at the cost of balancing the load less well when the
|
|
51
|
+
duration of `f` varies.
|
|
52
|
+
|
|
53
|
+
## Measured
|
|
54
|
+
|
|
55
|
+
`tools/benchmark.py` on a Ryzen 9 9950X (16 cores, 32 threads), Python 3.15.2,
|
|
56
|
+
wall time:
|
|
57
|
+
|
|
58
|
+
- `cpu ×8`: 8 calls of CPU-bound pure Python, ~0.27 s each, on 8 workers;
|
|
59
|
+
- `io ×8`: 8 calls that each sleep 0.25 s, on 8 workers;
|
|
60
|
+
- `overhead`: 100,000 calls that do nearly nothing, on 4 workers, at
|
|
61
|
+
`chunksize` 1 and 100.
|
|
62
|
+
|
|
63
|
+
With the GIL:
|
|
64
|
+
|
|
65
|
+
| `mode` | cpu ×8 | io ×8 | overhead, chunksize 1 | chunksize 100 |
|
|
66
|
+
| --- | --: | --: | --: | --: |
|
|
67
|
+
| `serial` | 2.13 s | 2.00 s | 0.00 s | 0.00 s |
|
|
68
|
+
| `multiprocessing` | 0.32 s | 0.26 s | 2.72 s | 0.04 s |
|
|
69
|
+
| `multithreading` | 2.14 s | 0.25 s | 0.55 s | 0.02 s |
|
|
70
|
+
| `multiinterpreter` | 0.33 s | 0.29 s | 2.09 s | 0.08 s |
|
|
71
|
+
|
|
72
|
+
Free-threaded:
|
|
73
|
+
|
|
74
|
+
| `mode` | cpu ×8 | io ×8 | overhead, chunksize 1 | chunksize 100 |
|
|
75
|
+
| --- | --: | --: | --: | --: |
|
|
76
|
+
| `serial` | 2.20 s | 2.00 s | 0.00 s | 0.00 s |
|
|
77
|
+
| `multiprocessing` | 0.34 s | 0.26 s | 1.67 s | 0.05 s |
|
|
78
|
+
| `multithreading` | 0.31 s | 0.25 s | 0.38 s | 0.01 s |
|
|
79
|
+
| `multiinterpreter` | 0.52 s | 0.52 s | 1.16 s | 0.15 s |
|
|
80
|
+
|
|
81
|
+
- With the GIL, CPU-bound work in threads takes as long as `serial`; without
|
|
82
|
+
it, threads do as well as processes.
|
|
83
|
+
- Every backend overlaps waiting.
|
|
84
|
+
- One call at a time, the overhead of a process or an interpreter is some
|
|
85
|
+
20–30 µs a call, four to five times a thread's; in chunks of 100, it is
|
|
86
|
+
nearly gone. The
|
|
87
|
+
cost of starting the workers is in every number here, and is the most of
|
|
88
|
+
what `multiinterpreter` takes beyond the others: interpreters start more
|
|
89
|
+
slowly on the free-threaded build.
|
|
90
|
+
|
|
91
|
+
## Beyond one machine
|
|
92
|
+
|
|
93
|
+
`dask` runs on the active `distributed.Client`, if there is one, so the same
|
|
94
|
+
call runs on whatever cluster the client is connected to: SLURM or PBS through
|
|
95
|
+
dask-jobqueue, Kubernetes, and so on. Without one, it starts a
|
|
96
|
+
`distributed.LocalCluster` of `processes` workers, which is useful mostly for
|
|
97
|
+
trying the code out before a real cluster.
|
|
98
|
+
|
|
99
|
+
`mpi` and `mpi_simple` run under `mpiexec`, across the nodes of an HPC job.
|
|
100
|
+
`mpi` hands out calls dynamically to workers as they become free, and suits
|
|
101
|
+
calls whose duration varies; `mpi_simple` splits the calls into one contiguous
|
|
102
|
+
block per rank up front, with no scheduler, and works with any MPI
|
|
103
|
+
implementation, including those that `mpi4py.futures` does not.
|
|
104
|
+
|
|
105
|
+
## Gotchas
|
|
106
|
+
|
|
107
|
+
- **Picklability.** `f` must be importable by name in the workers: defined at
|
|
108
|
+
the top level of a module, not a lambda, a closure, or a method of a local
|
|
109
|
+
class. `functools.partial` of such a function works, which is how to pass
|
|
110
|
+
keyword arguments. `multithreading` and `serial` have no such limit.
|
|
111
|
+
`dask` pickles with cloudpickle, and `multiinterpreter` can share functions
|
|
112
|
+
that use no closure, so both accept lambdas too.
|
|
113
|
+
- **`if __name__ == "__main__":`.** Where workers start by `spawn` (macOS,
|
|
114
|
+
Windows) or `forkserver` (Linux from Python 3.14), each one imports the main
|
|
115
|
+
module afresh, which then must not start the map again at import.
|
|
116
|
+
- **Oversubscription.** `processes=None` is each backend's default: one
|
|
117
|
+
worker per core for `multiprocessing`, but `min(32, cores + 4)` for
|
|
118
|
+
`multithreading` and `multiinterpreter`, since I/O-bound work wants more
|
|
119
|
+
threads than cores; for CPU-bound work in interpreters, pass `processes`. If `f` is itself multi-threaded, as NumPy's
|
|
120
|
+
linear algebra is, many workers times many threads each oversubscribe the
|
|
121
|
+
cores; limit one or the other, e.g. with `OMP_NUM_THREADS=1`.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
|
2
|
+
from importlib.metadata import version as get_version
|
|
3
|
+
|
|
4
|
+
project = "map_parallel"
|
|
5
|
+
author = "Kolen Cheung"
|
|
6
|
+
copyright = f"2020, {author}"
|
|
7
|
+
version = release = get_version("map-parallel")
|
|
8
|
+
|
|
9
|
+
extensions = [
|
|
10
|
+
"myst_parser",
|
|
11
|
+
"sphinx.ext.apidoc",
|
|
12
|
+
"sphinx.ext.autodoc",
|
|
13
|
+
"sphinx.ext.intersphinx",
|
|
14
|
+
"sphinx.ext.viewcode",
|
|
15
|
+
]
|
|
16
|
+
source_suffix = {".md": "markdown", ".rst": "restructuredtext"}
|
|
17
|
+
exclude_patterns = ["_build"]
|
|
18
|
+
|
|
19
|
+
html_theme = "furo"
|
|
20
|
+
html_title = f"{project} {version}"
|
|
21
|
+
|
|
22
|
+
autodoc_member_order = "bysource"
|
|
23
|
+
apidoc_modules = [
|
|
24
|
+
{
|
|
25
|
+
"path": "../src/map_parallel",
|
|
26
|
+
"destination": "api",
|
|
27
|
+
"separate_modules": True,
|
|
28
|
+
"module_first": True,
|
|
29
|
+
},
|
|
30
|
+
]
|
|
31
|
+
intersphinx_mapping = {
|
|
32
|
+
"python": ("https://docs.python.org/3", None),
|
|
33
|
+
"distributed": ("https://distributed.dask.org/en/stable", None),
|
|
34
|
+
"mpi4py": ("https://mpi4py.readthedocs.io/en/stable", None),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
myst_heading_anchors = 3
|