numba-mpi 1.1.2__tar.gz → 1.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.github/workflows/readme_snippets.yml +2 -2
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.github/workflows/tests+pypi.yml +40 -20
- {numba_mpi-1.1.2/numba_mpi.egg-info → numba_mpi-1.1.4}/PKG-INFO +41 -22
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/README.md +28 -20
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/requests.py +2 -2
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/common.py +7 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4/numba_mpi.egg-info}/PKG-INFO +41 -22
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/setup.py +2 -2
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_bcast.py +4 -4
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/test_paper_listings.py +1 -1
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.github/numba_mpi_logo.png +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.github/numba_mpi_logo.svg +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.github/workflows/stale.yml +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.gitignore +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.pre-commit-config.yaml +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.vscode/settings.json +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/.zenodo.json +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/CITATION.cff +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/CODE_OF_CONDUCT.md +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/LICENSE +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/__init__.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/__init__.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/allreduce.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/barrier.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/bcast.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/initialized.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/irecv.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/isend.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/operator.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/rank.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/recv.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/reduce.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/scatter_gather.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/send.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/size.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/api/wtime.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi/utils.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi.egg-info/SOURCES.txt +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi.egg-info/dependency_links.txt +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi.egg-info/requires.txt +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/numba_mpi.egg-info/top_level.txt +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/setup.cfg +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/__init__.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_allreduce.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_barrier.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_init.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_isend_irecv.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_rank.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_reduce.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_scatter_gather.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_send_recv.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_size.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/api/test_wtime.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/common.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/exchange.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/hello.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/mpi4py_with_error.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/numba_mpi.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/py-pde.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/test.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/paper_listings/timing.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/test_version.py +0 -0
- {numba_mpi-1.1.2 → numba_mpi-1.1.4}/tests/utils.py +0 -0
@@ -18,10 +18,10 @@ jobs:
|
|
18
18
|
pylint:
|
19
19
|
runs-on: ubuntu-latest
|
20
20
|
steps:
|
21
|
-
- uses: actions/checkout@
|
21
|
+
- uses: actions/checkout@v4
|
22
22
|
- uses: mpi4py/setup-mpi@v1
|
23
23
|
- name: Set up Python
|
24
|
-
uses: actions/setup-python@
|
24
|
+
uses: actions/setup-python@v5
|
25
25
|
with:
|
26
26
|
python-version: "3.10"
|
27
27
|
- name: Install dependencies
|
@@ -38,11 +38,11 @@ jobs:
|
|
38
38
|
precommit:
|
39
39
|
runs-on: ubuntu-latest
|
40
40
|
steps:
|
41
|
-
- uses: actions/checkout@
|
41
|
+
- uses: actions/checkout@v4
|
42
42
|
- name: Set up Python
|
43
|
-
uses: actions/setup-python@
|
43
|
+
uses: actions/setup-python@v5
|
44
44
|
with:
|
45
|
-
python-version: "3.10"
|
45
|
+
python-version: "3.10"
|
46
46
|
- name: Linting
|
47
47
|
run: |
|
48
48
|
pip install pre-commit
|
@@ -55,10 +55,10 @@ jobs:
|
|
55
55
|
permissions:
|
56
56
|
contents: write
|
57
57
|
steps:
|
58
|
-
- uses: actions/checkout@
|
58
|
+
- uses: actions/checkout@v4
|
59
59
|
with:
|
60
60
|
persist-credentials: false
|
61
|
-
- uses: actions/setup-python@
|
61
|
+
- uses: actions/setup-python@v5
|
62
62
|
with:
|
63
63
|
python-version: "3.10"
|
64
64
|
- uses: mpi4py/setup-mpi@v1
|
@@ -78,9 +78,9 @@ jobs:
|
|
78
78
|
zenodo_json:
|
79
79
|
runs-on: ubuntu-latest
|
80
80
|
steps:
|
81
|
-
- uses: actions/checkout@
|
81
|
+
- uses: actions/checkout@v4
|
82
82
|
- uses: notiz-dev/github-action-json-property@release
|
83
|
-
with:
|
83
|
+
with:
|
84
84
|
path: '.zenodo.json'
|
85
85
|
prop_path: 'creators'
|
86
86
|
|
@@ -89,42 +89,53 @@ jobs:
|
|
89
89
|
strategy:
|
90
90
|
matrix:
|
91
91
|
mpi-np: [2, 3]
|
92
|
-
platform:
|
93
|
-
|
92
|
+
platform:
|
93
|
+
- ubuntu-latest
|
94
|
+
- windows-latest
|
95
|
+
- macos-13 # arch: Intel
|
96
|
+
- macos-15 # arch: arm64
|
97
|
+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
94
98
|
mpi: [ 'mpich', 'openmpi', 'msmpi', 'intelmpi']
|
95
99
|
exclude:
|
96
100
|
- platform: macos-13
|
97
101
|
mpi: msmpi
|
98
102
|
- platform: macos-13
|
99
103
|
mpi: intelmpi
|
100
|
-
- platform: macos-13
|
104
|
+
- platform: macos-13 # TODO: # 163
|
105
|
+
mpi: openmpi
|
106
|
+
- platform: macos-15
|
107
|
+
mpi: msmpi
|
108
|
+
- platform: macos-15
|
109
|
+
mpi: intelmpi
|
110
|
+
- platform: macos-15
|
101
111
|
mpi: openmpi
|
102
112
|
- platform: ubuntu-latest
|
103
113
|
mpi: msmpi
|
114
|
+
- platform: ubuntu-latest # TODO: #162
|
115
|
+
mpi: mpich
|
104
116
|
- platform: windows-latest
|
105
117
|
mpi: mpich
|
106
118
|
- platform: windows-latest
|
107
119
|
mpi: openmpi
|
108
120
|
- platform: windows-latest
|
109
121
|
mpi: intelmpi
|
110
|
-
|
122
|
+
|
111
123
|
# https://github.com/numba-mpi/numba-mpi/issues/69
|
112
|
-
- platform: ubuntu-latest
|
113
|
-
python-version: 3.8
|
114
|
-
mpi: mpich
|
115
124
|
- platform: ubuntu-latest
|
116
125
|
python-version: 3.9
|
117
126
|
mpi: mpich
|
118
127
|
fail-fast: false
|
119
128
|
runs-on: ${{ matrix.platform }}
|
120
129
|
steps:
|
121
|
-
- uses: actions/checkout@
|
122
|
-
- uses: actions/setup-python@
|
130
|
+
- uses: actions/checkout@v4
|
131
|
+
- uses: actions/setup-python@v5
|
123
132
|
with:
|
124
133
|
python-version: ${{ matrix.python-version }}
|
125
134
|
- uses: mpi4py/setup-mpi@v1
|
126
135
|
with:
|
127
136
|
mpi: ${{ matrix.mpi }}
|
137
|
+
- if: ${{ matrix.platform == 'macos-15' }}
|
138
|
+
run: echo DYLD_FALLBACK_LIBRARY_PATH="/opt/homebrew/lib:/usr/lib:$DYLD_FALLBACK_LIBRARY_PATH" >> $GITHUB_ENV
|
128
139
|
- if: matrix.mpi == 'intelmpi'
|
129
140
|
run: |
|
130
141
|
# checking if multiple MPI libs are OK
|
@@ -165,13 +176,22 @@ jobs:
|
|
165
176
|
cd /tmp # make sure local files are not picked up
|
166
177
|
python -We -c "import numba_mpi"
|
167
178
|
|
168
|
-
- if:
|
179
|
+
- if: |-
|
180
|
+
${{
|
181
|
+
github.repository == 'numba-mpi/numba-mpi' &&
|
182
|
+
github.event_name == 'push' &&
|
183
|
+
github.ref == 'refs/heads/main'
|
184
|
+
}}
|
169
185
|
uses: pypa/gh-action-pypi-publish@release/v1.12
|
170
186
|
with:
|
171
187
|
attestations: false
|
172
188
|
repository_url: https://test.pypi.org/legacy/
|
173
189
|
|
174
|
-
- if:
|
190
|
+
- if: |-
|
191
|
+
${{
|
192
|
+
github.repository == 'numba-mpi/numba-mpi' &&
|
193
|
+
startsWith(github.ref, 'refs/tags')
|
194
|
+
}}
|
175
195
|
uses: pypa/gh-action-pypi-publish@release/v1.12
|
176
196
|
with:
|
177
197
|
attestations: false
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: numba-mpi
|
3
|
-
Version: 1.1.
|
3
|
+
Version: 1.1.4
|
4
4
|
Summary: Numba @jittable MPI wrappers tested on Linux, macOS and Windows
|
5
5
|
Home-page: https://github.com/numba-mpi/numba-mpi
|
6
6
|
Author: https://github.com/numba-mpi/numba-mpi/graphs/contributors
|
@@ -18,6 +18,17 @@ Requires-Dist: psutil
|
|
18
18
|
Provides-Extra: tests
|
19
19
|
Requires-Dist: pytest<8.0.0; extra == "tests"
|
20
20
|
Requires-Dist: py-pde; extra == "tests"
|
21
|
+
Dynamic: author
|
22
|
+
Dynamic: description
|
23
|
+
Dynamic: description-content-type
|
24
|
+
Dynamic: home-page
|
25
|
+
Dynamic: license
|
26
|
+
Dynamic: license-file
|
27
|
+
Dynamic: project-url
|
28
|
+
Dynamic: provides-extra
|
29
|
+
Dynamic: requires-dist
|
30
|
+
Dynamic: requires-python
|
31
|
+
Dynamic: summary
|
21
32
|
|
22
33
|
# <img src="https://raw.githubusercontent.com/numba-mpi/numba-mpi/main/.github/numba_mpi_logo.png" width=128 height=142 alt="numba-mpi logo"> numba-mpi
|
23
34
|
|
@@ -35,31 +46,32 @@ Requires-Dist: py-pde; extra == "tests"
|
|
35
46
|
[](https://zenodo.org/badge/latestdoi/316911228)
|
36
47
|
|
37
48
|
### Overview
|
38
|
-
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi
|
49
|
+
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi paper in SoftwareX (open access)](https://www.sciencedirect.com/science/article/pii/S235271102400267X) (please cite if numba-mpi is used in your research).
|
39
50
|
|
40
51
|
Support is provided for a subset of MPI routines covering: `size`/`rank`, `send`/`recv`, `allreduce`, `reduce`, `bcast`, `scatter`/`gather` & `allgather`, `barrier`, `wtime`
|
41
52
|
and basic asynchronous communication with `isend`/`irecv` (only for contiguous arrays); for request handling including `wait`/`waitall`/`waitany` and `test`/`testall`/`testany`.
|
42
53
|
|
43
|
-
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
54
|
+
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
44
55
|
Auto-generated docstring-based API docs are published on the web: https://numba-mpi.github.io/numba-mpi
|
45
56
|
|
46
|
-
Packages can be obtained from
|
47
|
-
[PyPI](https://pypi.org/project/numba-mpi),
|
48
|
-
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
57
|
+
Packages can be obtained from
|
58
|
+
[PyPI](https://pypi.org/project/numba-mpi),
|
59
|
+
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
49
60
|
[Arch Linux](https://aur.archlinux.org/packages/python-numba-mpi)
|
50
61
|
or by invoking `pip install git+https://github.com/numba-mpi/numba-mpi.git`.
|
51
62
|
|
52
63
|
numba-mpi is a pure-Python package.
|
53
64
|
The codebase includes a test suite used through the GitHub Actions workflows ([thanks to mpi4py's setup-mpi](https://github.com/mpi4py/setup-mpi)!)
|
54
|
-
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
55
|
-
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
56
|
-
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
57
|
-
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)).
|
65
|
+
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
66
|
+
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
67
|
+
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
68
|
+
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)). Note, that some of those
|
69
|
+
combinations may not be fully supported yet - see [Known Issues](#known-issues) for more information.
|
58
70
|
|
59
71
|
Features that are not implemented yet include (help welcome!):
|
60
72
|
- support for non-default communicators
|
61
73
|
- support for `MPI_IN_PLACE` in `[all]gather`/`scatter` and `allreduce`
|
62
|
-
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
74
|
+
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
63
75
|
- ...
|
64
76
|
|
65
77
|
### Hello world send/recv example:
|
@@ -82,21 +94,21 @@ hello()
|
|
82
94
|
### Example comparing numba-mpi vs. mpi4py performance:
|
83
95
|
|
84
96
|
The example below compares `Numba`+`mpi4py` vs. `Numba`+`numba-mpi` performance.
|
85
|
-
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
86
|
-
dividing the workload into `n_intervals` handled by separate MPI processes
|
97
|
+
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
98
|
+
dividing the workload into `n_intervals` handled by separate MPI processes
|
87
99
|
and then obtaining a sum using `allreduce` (see, e.g., analogous [Matlab docs example](https://www.mathworks.com/help/parallel-computing/numerical-estimation-of-pi-using-message-passing.html)).
|
88
100
|
The computation is carried out in a JIT-compiled function `get_pi_part()` and is repeated
|
89
|
-
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
101
|
+
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
90
102
|
inside of the JIT-compiled block for `mpi4py` and `numba-mpi`, respectively.
|
91
103
|
Timing is repeated `N_REPEAT` times and the minimum time is reported.
|
92
104
|
The generated plot shown below depicts the speedup obtained by replacing `mpi4py`
|
93
|
-
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
105
|
+
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
94
106
|
interval. The speedup, which stems from avoiding roundtrips between JIT-compiled
|
95
107
|
and Python code is significant (150%-300%) in all cases. The more often communication
|
96
|
-
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
108
|
+
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
97
109
|
in the actual number crunching (within the `get_pi_part()` function) or in the employed communication logic
|
98
110
|
(handled by the same MPI library) differs between the `mpi4py` or `numba-mpi` solutions.
|
99
|
-
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
111
|
+
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
100
112
|
repeatedly entering and leaving the JIT-compiled block if using `mpi4py`, which can be
|
101
113
|
eliminated by using `numba-mpi`, and which the measured differences in execution time
|
102
114
|
stem from.
|
@@ -156,6 +168,14 @@ if numba_mpi.rank() == 0:
|
|
156
168
|
|
157
169
|

|
158
170
|
|
171
|
+
### Known Issues
|
172
|
+
|
173
|
+
**NOTE**: Issues listed below only relate to combinations of platforms and MPI distributions that we target to support, but due to various reason are currently not working and are temporarily excluded from automated testing:
|
174
|
+
|
175
|
+
- tests on Ubuntu 2024.4 that use MPICH are not run due to failures caused by newer version of MPICH (`4.2.0`); note, that previous tests ran
|
176
|
+
using version `4.0.2` of MPICH (that is installed by default on Ubuntu 2022.4 using `apt`) were passing (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/162) - TODO #162),
|
177
|
+
- tests on Intel MacOS (v13) that use OpenMPI are currently not run due to failures being under investigation (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/163) - TODO #163),
|
178
|
+
- `numba-mpi` currently does not support ARM-based MacOS, due to required code improvement (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/164) - TODO #164).
|
159
179
|
|
160
180
|
### MPI resources on the web:
|
161
181
|
|
@@ -177,11 +197,10 @@ if numba_mpi.rank() == 0:
|
|
177
197
|
|
178
198
|
### Acknowledgements:
|
179
199
|
|
180
|
-
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
200
|
+
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
181
201
|
through [GitHub issues](https://github.com/numba-mpi/numba-mpi/issues).
|
182
202
|
|
183
203
|
Development of numba-mpi has been supported by the [Polish National Science Centre](https://ncn.gov.pl/en) (grant no. 2020/39/D/ST10/01220),
|
184
|
-
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
185
|
-
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
204
|
+
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
205
|
+
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
186
206
|
for providing computer facilities and support within computational grant no. PLG/2023/016369.
|
187
|
-
|
@@ -14,31 +14,32 @@
|
|
14
14
|
[](https://zenodo.org/badge/latestdoi/316911228)
|
15
15
|
|
16
16
|
### Overview
|
17
|
-
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi
|
17
|
+
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi paper in SoftwareX (open access)](https://www.sciencedirect.com/science/article/pii/S235271102400267X) (please cite if numba-mpi is used in your research).
|
18
18
|
|
19
19
|
Support is provided for a subset of MPI routines covering: `size`/`rank`, `send`/`recv`, `allreduce`, `reduce`, `bcast`, `scatter`/`gather` & `allgather`, `barrier`, `wtime`
|
20
20
|
and basic asynchronous communication with `isend`/`irecv` (only for contiguous arrays); for request handling including `wait`/`waitall`/`waitany` and `test`/`testall`/`testany`.
|
21
21
|
|
22
|
-
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
22
|
+
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
23
23
|
Auto-generated docstring-based API docs are published on the web: https://numba-mpi.github.io/numba-mpi
|
24
24
|
|
25
|
-
Packages can be obtained from
|
26
|
-
[PyPI](https://pypi.org/project/numba-mpi),
|
27
|
-
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
25
|
+
Packages can be obtained from
|
26
|
+
[PyPI](https://pypi.org/project/numba-mpi),
|
27
|
+
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
28
28
|
[Arch Linux](https://aur.archlinux.org/packages/python-numba-mpi)
|
29
29
|
or by invoking `pip install git+https://github.com/numba-mpi/numba-mpi.git`.
|
30
30
|
|
31
31
|
numba-mpi is a pure-Python package.
|
32
32
|
The codebase includes a test suite used through the GitHub Actions workflows ([thanks to mpi4py's setup-mpi](https://github.com/mpi4py/setup-mpi)!)
|
33
|
-
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
34
|
-
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
35
|
-
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
36
|
-
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)).
|
33
|
+
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
34
|
+
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
35
|
+
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
36
|
+
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)). Note, that some of those
|
37
|
+
combinations may not be fully supported yet - see [Known Issues](#known-issues) for more information.
|
37
38
|
|
38
39
|
Features that are not implemented yet include (help welcome!):
|
39
40
|
- support for non-default communicators
|
40
41
|
- support for `MPI_IN_PLACE` in `[all]gather`/`scatter` and `allreduce`
|
41
|
-
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
42
|
+
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
42
43
|
- ...
|
43
44
|
|
44
45
|
### Hello world send/recv example:
|
@@ -61,21 +62,21 @@ hello()
|
|
61
62
|
### Example comparing numba-mpi vs. mpi4py performance:
|
62
63
|
|
63
64
|
The example below compares `Numba`+`mpi4py` vs. `Numba`+`numba-mpi` performance.
|
64
|
-
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
65
|
-
dividing the workload into `n_intervals` handled by separate MPI processes
|
65
|
+
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
66
|
+
dividing the workload into `n_intervals` handled by separate MPI processes
|
66
67
|
and then obtaining a sum using `allreduce` (see, e.g., analogous [Matlab docs example](https://www.mathworks.com/help/parallel-computing/numerical-estimation-of-pi-using-message-passing.html)).
|
67
68
|
The computation is carried out in a JIT-compiled function `get_pi_part()` and is repeated
|
68
|
-
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
69
|
+
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
69
70
|
inside of the JIT-compiled block for `mpi4py` and `numba-mpi`, respectively.
|
70
71
|
Timing is repeated `N_REPEAT` times and the minimum time is reported.
|
71
72
|
The generated plot shown below depicts the speedup obtained by replacing `mpi4py`
|
72
|
-
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
73
|
+
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
73
74
|
interval. The speedup, which stems from avoiding roundtrips between JIT-compiled
|
74
75
|
and Python code is significant (150%-300%) in all cases. The more often communication
|
75
|
-
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
76
|
+
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
76
77
|
in the actual number crunching (within the `get_pi_part()` function) or in the employed communication logic
|
77
78
|
(handled by the same MPI library) differs between the `mpi4py` or `numba-mpi` solutions.
|
78
|
-
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
79
|
+
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
79
80
|
repeatedly entering and leaving the JIT-compiled block if using `mpi4py`, which can be
|
80
81
|
eliminated by using `numba-mpi`, and which the measured differences in execution time
|
81
82
|
stem from.
|
@@ -135,6 +136,14 @@ if numba_mpi.rank() == 0:
|
|
135
136
|
|
136
137
|

|
137
138
|
|
139
|
+
### Known Issues
|
140
|
+
|
141
|
+
**NOTE**: Issues listed below only relate to combinations of platforms and MPI distributions that we target to support, but due to various reason are currently not working and are temporarily excluded from automated testing:
|
142
|
+
|
143
|
+
- tests on Ubuntu 2024.4 that use MPICH are not run due to failures caused by newer version of MPICH (`4.2.0`); note, that previous tests ran
|
144
|
+
using version `4.0.2` of MPICH (that is installed by default on Ubuntu 2022.4 using `apt`) were passing (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/162) - TODO #162),
|
145
|
+
- tests on Intel MacOS (v13) that use OpenMPI are currently not run due to failures being under investigation (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/163) - TODO #163),
|
146
|
+
- `numba-mpi` currently does not support ARM-based MacOS, due to required code improvement (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/164) - TODO #164).
|
138
147
|
|
139
148
|
### MPI resources on the web:
|
140
149
|
|
@@ -156,11 +165,10 @@ if numba_mpi.rank() == 0:
|
|
156
165
|
|
157
166
|
### Acknowledgements:
|
158
167
|
|
159
|
-
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
168
|
+
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
160
169
|
through [GitHub issues](https://github.com/numba-mpi/numba-mpi/issues).
|
161
170
|
|
162
171
|
Development of numba-mpi has been supported by the [Polish National Science Centre](https://ncn.gov.pl/en) (grant no. 2020/39/D/ST10/01220),
|
163
|
-
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
164
|
-
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
172
|
+
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
173
|
+
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
165
174
|
for providing computer facilities and support within computational grant no. PLG/2023/016369.
|
166
|
-
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
import ctypes
|
4
4
|
import os
|
5
|
+
import sys
|
5
6
|
from ctypes.util import find_library
|
6
7
|
from pathlib import Path
|
7
8
|
|
@@ -72,6 +73,12 @@ else:
|
|
72
73
|
break
|
73
74
|
|
74
75
|
if LIB is None:
|
76
|
+
if sys.platform == "darwin":
|
77
|
+
raise RuntimeError(
|
78
|
+
"""MPI library not found, if MPI was installed with Homebrew, export the following:
|
79
|
+
DYLD_FALLBACK_LIBRARY_PATH="/opt/homebrew/lib:/usr/lib:$DYLD_FALLBACK_LIBRARY_PATH
|
80
|
+
"""
|
81
|
+
)
|
75
82
|
raise RuntimeError("no MPI library found")
|
76
83
|
|
77
84
|
libmpi = ctypes.CDLL(LIB)
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: numba-mpi
|
3
|
-
Version: 1.1.
|
3
|
+
Version: 1.1.4
|
4
4
|
Summary: Numba @jittable MPI wrappers tested on Linux, macOS and Windows
|
5
5
|
Home-page: https://github.com/numba-mpi/numba-mpi
|
6
6
|
Author: https://github.com/numba-mpi/numba-mpi/graphs/contributors
|
@@ -18,6 +18,17 @@ Requires-Dist: psutil
|
|
18
18
|
Provides-Extra: tests
|
19
19
|
Requires-Dist: pytest<8.0.0; extra == "tests"
|
20
20
|
Requires-Dist: py-pde; extra == "tests"
|
21
|
+
Dynamic: author
|
22
|
+
Dynamic: description
|
23
|
+
Dynamic: description-content-type
|
24
|
+
Dynamic: home-page
|
25
|
+
Dynamic: license
|
26
|
+
Dynamic: license-file
|
27
|
+
Dynamic: project-url
|
28
|
+
Dynamic: provides-extra
|
29
|
+
Dynamic: requires-dist
|
30
|
+
Dynamic: requires-python
|
31
|
+
Dynamic: summary
|
21
32
|
|
22
33
|
# <img src="https://raw.githubusercontent.com/numba-mpi/numba-mpi/main/.github/numba_mpi_logo.png" width=128 height=142 alt="numba-mpi logo"> numba-mpi
|
23
34
|
|
@@ -35,31 +46,32 @@ Requires-Dist: py-pde; extra == "tests"
|
|
35
46
|
[](https://zenodo.org/badge/latestdoi/316911228)
|
36
47
|
|
37
48
|
### Overview
|
38
|
-
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi
|
49
|
+
numba-mpi provides Python wrappers to the C MPI API callable from within [Numba JIT-compiled code](https://numba.readthedocs.io/en/stable/user/jit.html) (@jit mode). For an outline of the project, rationale, architecture, and features, refer to: [numba-mpi paper in SoftwareX (open access)](https://www.sciencedirect.com/science/article/pii/S235271102400267X) (please cite if numba-mpi is used in your research).
|
39
50
|
|
40
51
|
Support is provided for a subset of MPI routines covering: `size`/`rank`, `send`/`recv`, `allreduce`, `reduce`, `bcast`, `scatter`/`gather` & `allgather`, `barrier`, `wtime`
|
41
52
|
and basic asynchronous communication with `isend`/`irecv` (only for contiguous arrays); for request handling including `wait`/`waitall`/`waitany` and `test`/`testall`/`testany`.
|
42
53
|
|
43
|
-
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
54
|
+
The API uses NumPy and supports both numeric and character datatypes (e.g., `broadcast`).
|
44
55
|
Auto-generated docstring-based API docs are published on the web: https://numba-mpi.github.io/numba-mpi
|
45
56
|
|
46
|
-
Packages can be obtained from
|
47
|
-
[PyPI](https://pypi.org/project/numba-mpi),
|
48
|
-
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
57
|
+
Packages can be obtained from
|
58
|
+
[PyPI](https://pypi.org/project/numba-mpi),
|
59
|
+
[Conda Forge](https://anaconda.org/conda-forge/numba-mpi),
|
49
60
|
[Arch Linux](https://aur.archlinux.org/packages/python-numba-mpi)
|
50
61
|
or by invoking `pip install git+https://github.com/numba-mpi/numba-mpi.git`.
|
51
62
|
|
52
63
|
numba-mpi is a pure-Python package.
|
53
64
|
The codebase includes a test suite used through the GitHub Actions workflows ([thanks to mpi4py's setup-mpi](https://github.com/mpi4py/setup-mpi)!)
|
54
|
-
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
55
|
-
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
56
|
-
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
57
|
-
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)).
|
65
|
+
for automated testing on: Linux ([MPICH](https://www.mpich.org/), [OpenMPI](https://www.open-mpi.org/doc/)
|
66
|
+
& [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)),
|
67
|
+
macOS ([MPICH](https://www.mpich.org/) & [OpenMPI](https://www.open-mpi.org/doc/)) and
|
68
|
+
Windows ([MS MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)). Note, that some of those
|
69
|
+
combinations may not be fully supported yet - see [Known Issues](#known-issues) for more information.
|
58
70
|
|
59
71
|
Features that are not implemented yet include (help welcome!):
|
60
72
|
- support for non-default communicators
|
61
73
|
- support for `MPI_IN_PLACE` in `[all]gather`/`scatter` and `allreduce`
|
62
|
-
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
74
|
+
- support for `MPI_Type_create_struct` (Numpy structured arrays)
|
63
75
|
- ...
|
64
76
|
|
65
77
|
### Hello world send/recv example:
|
@@ -82,21 +94,21 @@ hello()
|
|
82
94
|
### Example comparing numba-mpi vs. mpi4py performance:
|
83
95
|
|
84
96
|
The example below compares `Numba`+`mpi4py` vs. `Numba`+`numba-mpi` performance.
|
85
|
-
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
86
|
-
dividing the workload into `n_intervals` handled by separate MPI processes
|
97
|
+
The sample code estimates $\pi$ by numerical integration of $\int_0^1 (4/(1+x^2))dx=\pi$
|
98
|
+
dividing the workload into `n_intervals` handled by separate MPI processes
|
87
99
|
and then obtaining a sum using `allreduce` (see, e.g., analogous [Matlab docs example](https://www.mathworks.com/help/parallel-computing/numerical-estimation-of-pi-using-message-passing.html)).
|
88
100
|
The computation is carried out in a JIT-compiled function `get_pi_part()` and is repeated
|
89
|
-
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
101
|
+
`N_TIMES`. The repetitions and the MPI-handled reduction are done outside or
|
90
102
|
inside of the JIT-compiled block for `mpi4py` and `numba-mpi`, respectively.
|
91
103
|
Timing is repeated `N_REPEAT` times and the minimum time is reported.
|
92
104
|
The generated plot shown below depicts the speedup obtained by replacing `mpi4py`
|
93
|
-
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
105
|
+
with `numba_mpi`, plotted as a function of `N_TIMES / n_intervals` - the number of MPI calls per
|
94
106
|
interval. The speedup, which stems from avoiding roundtrips between JIT-compiled
|
95
107
|
and Python code is significant (150%-300%) in all cases. The more often communication
|
96
|
-
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
108
|
+
is needed (smaller `n_intervals`), the larger the measured speedup. Note that nothing
|
97
109
|
in the actual number crunching (within the `get_pi_part()` function) or in the employed communication logic
|
98
110
|
(handled by the same MPI library) differs between the `mpi4py` or `numba-mpi` solutions.
|
99
|
-
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
111
|
+
These are the overhead of `mpi4py` higher-level abstractions and the overhead of
|
100
112
|
repeatedly entering and leaving the JIT-compiled block if using `mpi4py`, which can be
|
101
113
|
eliminated by using `numba-mpi`, and which the measured differences in execution time
|
102
114
|
stem from.
|
@@ -156,6 +168,14 @@ if numba_mpi.rank() == 0:
|
|
156
168
|
|
157
169
|

|
158
170
|
|
171
|
+
### Known Issues
|
172
|
+
|
173
|
+
**NOTE**: Issues listed below only relate to combinations of platforms and MPI distributions that we target to support, but due to various reason are currently not working and are temporarily excluded from automated testing:
|
174
|
+
|
175
|
+
- tests on Ubuntu 2024.4 that use MPICH are not run due to failures caused by newer version of MPICH (`4.2.0`); note, that previous tests ran
|
176
|
+
using version `4.0.2` of MPICH (that is installed by default on Ubuntu 2022.4 using `apt`) were passing (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/162) - TODO #162),
|
177
|
+
- tests on Intel MacOS (v13) that use OpenMPI are currently not run due to failures being under investigation (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/163) - TODO #163),
|
178
|
+
- `numba-mpi` currently does not support ARM-based MacOS, due to required code improvement (see [related issue](https://github.com/numba-mpi/numba-mpi/issues/164) - TODO #164).
|
159
179
|
|
160
180
|
### MPI resources on the web:
|
161
181
|
|
@@ -177,11 +197,10 @@ if numba_mpi.rank() == 0:
|
|
177
197
|
|
178
198
|
### Acknowledgements:
|
179
199
|
|
180
|
-
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
200
|
+
We thank [all contributors](https://github.com/numba-mpi/numba-mpi/graphs/contributors) and users who reported feedback to the project
|
181
201
|
through [GitHub issues](https://github.com/numba-mpi/numba-mpi/issues).
|
182
202
|
|
183
203
|
Development of numba-mpi has been supported by the [Polish National Science Centre](https://ncn.gov.pl/en) (grant no. 2020/39/D/ST10/01220),
|
184
|
-
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
185
|
-
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
204
|
+
the [Max Planck Society](https://www.mpg.de/en) and the [European Union](https://erc.europa.eu/) (ERC, EmulSim, 101044662).
|
205
|
+
We further acknowledge Poland’s high-performance computing infrastructure [PLGrid](https://plgrid.pl) (HPC Centers: [ACK Cyfronet AGH](https://www.cyfronet.pl/en))
|
186
206
|
for providing computer facilities and support within computational grant no. PLG/2023/016369.
|
187
|
-
|
@@ -1,4 +1,4 @@
|
|
1
|
-
"""
|
1
|
+
"""the magick behind ``pip install ...``"""
|
2
2
|
|
3
3
|
from setuptools import find_packages, setup
|
4
4
|
|
@@ -18,7 +18,7 @@ setup(
|
|
18
18
|
url="https://github.com/numba-mpi/numba-mpi",
|
19
19
|
author="https://github.com/numba-mpi/numba-mpi/graphs/contributors",
|
20
20
|
use_scm_version={
|
21
|
-
"local_scheme":
|
21
|
+
"local_scheme": "no-local-version",
|
22
22
|
"version_scheme": "post-release",
|
23
23
|
},
|
24
24
|
python_requires=">=3.8",
|
@@ -15,8 +15,8 @@ def jit_bcast(data, root):
|
|
15
15
|
|
16
16
|
@pytest.mark.parametrize("bcast", (jit_bcast.py_func, jit_bcast))
|
17
17
|
@pytest.mark.parametrize("data_type", data_types)
|
18
|
-
|
19
|
-
|
18
|
+
@pytest.mark.parametrize("root", range(mpi.size()))
|
19
|
+
def test_bcast_np_array(data_type, bcast, root):
|
20
20
|
data = np.empty(5, data_type).astype(dtype=data_type)
|
21
21
|
datatobcast = get_random_array(5, data_type).astype(dtype=data_type)
|
22
22
|
|
@@ -34,8 +34,8 @@ def test_bcast_np_array(data_type, bcast):
|
|
34
34
|
"stringtobcast",
|
35
35
|
("test bcast", pytest.param("żółć", marks=pytest.mark.xfail(strict=True))),
|
36
36
|
)
|
37
|
-
|
38
|
-
|
37
|
+
@pytest.mark.parametrize("root", range(mpi.size()))
|
38
|
+
def test_bcast_string(stringtobcast, root):
|
39
39
|
datatobcast = np.array(stringtobcast, "c")
|
40
40
|
data = np.empty_like(datatobcast)
|
41
41
|
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|