parameterrun 0.3.0__tar.gz → 0.4.0__tar.gz

parameterrun-0.4.0/.github/workflows/ci.yml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: non-mpi / ${{ matrix.os }} / py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        run: python -m pip install uv
+
+      - name: Sync environment
+        run: uv sync
+
+      - name: Tests
+        run: uv run pytest -m "not mpi"
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Type check
+        run: uv run mypy
+
+  test-mpi-linux:
+    name: mpi / ubuntu-latest / py3.12 / 4 ranks
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Setup MPI
+        uses: mpi4py/setup-mpi@v1
+        with:
+          mpi: 'openmpi'
+
+      - name: Install uv
+        run: python -m pip install uv
+
+      - name: Sync environment with MPI extra
+        run: uv sync --extra mpi
+
+      - name: MPI tests (4 ranks)
+        run: mpirun -n 4 uv run pytest -m mpi -o addopts="-q -m mpi"

{parameterrun-0.3.0 → parameterrun-0.4.0}/.github/workflows/publish.yml

@@ -19,7 +19,7 @@ jobs:
         with:
           ref: ${{ github.event.workflow_run.head_sha }}
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.9"
 

{parameterrun-0.3.0 → parameterrun-0.4.0}/CHANGELOG.md

@@ -4,22 +4,21 @@ All notable changes to this project are documented in this file.
 
 The format is based on Keep a Changelog and the project follows Semantic Versioning.
 
-## [0.3.1] - 2026-04-01
-
-### Added
-- (Your changes here)
+## [Unreleased]
 
-## [0.3.0] - 2026-04-01
+## [0.4.0] - 2026-04-21
 
 ### Added
-- (Your changes here)
 
-## [0.3.0] - 2026-04-01
+- Added `tests/test_mpi_integration.py` with MPI numerical-equivalence coverage for `parameterrun(..., backend="mpi")`.
+- Added a dedicated Linux PR CI job that runs MPI tests with 4 ranks.
 
-### Added
-- (Your changes here)
+### Changed
 
-## [Unreleased]
+- Updated default test commands in docs and CI to run non-MPI tests with `-m "not mpi"`.
+- Registered a pytest `mpi` marker in `pyproject.toml` for explicit MPI test selection.
+- Update how the indices to compute are shared among workers. Previously, all the indices where computed by all workers,
+  which caused a significan overhead in the memory usage. Now, the indices are computed on the fly.
 
 ## [0.3.0] - 2026-04-01
 

{parameterrun-0.3.0 → parameterrun-0.4.0}/CONTRIBUTING.md

@@ -13,13 +13,22 @@ If you do not use `uv`, create a virtual environment and install equivalent dev
 ## Run quality checks
 
 ```bash
-uv run pytest
+uv run pytest -m "not mpi"
 uv run ruff check .
 uv run mypy
 uv run python -m build
 uv run python -m twine check dist/*
 ```
 
+## Run MPI checks (required on PR CI)
+
+The PR workflow includes a dedicated Linux MPI job that runs with 4 ranks.
+
+```bash
+uv sync --extra mpi
+mpirun -n 4 uv run pytest -m mpi -o addopts="-q -m mpi"
+```
+
 ## Pull request guidelines
 
 - Keep PRs focused and small when possible.

{parameterrun-0.3.0 → parameterrun-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parameterrun
-Version: 0.3.0
+Version: 0.4.0
 Summary: Parallel parameter sweeps with joblib/MPI and tqdm progress bars.
 Project-URL: Homepage, https://github.com/Davtax/ParameterRun
 Project-URL: Repository, https://github.com/Davtax/ParameterRun
@@ -210,12 +210,6 @@ By default, progress bars are enabled:
 result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_bool=True, )
 ```
 
-You can also pass extra keyword arguments to `tqdm` through `pbar_kwargs`:
-
-```python
-result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_kwargs={"leave": False}, )
-```
-
 Verbose logging can be enabled with:
 
 ```python
@@ -279,9 +273,17 @@ param_values = [[[1, 2], [10, 20]], [[100, 200, 300]], ]
 Before opening a PR or publishing a release, run:
 
 ```bash
-pytest
+pytest -m "not mpi"
 ruff check .
-mypy src/parameterrun
+mypy
 python -m build
 python -m twine check dist/*
 ```
+
+For MPI numerical-equivalence checks (Linux/OpenMPI, 4 ranks):
+
+```bash
+uv sync --extra mpi
+mpirun -n 4 uv run pytest -m mpi -o addopts="-q -m mpi"
+```
+

{parameterrun-0.3.0 → parameterrun-0.4.0}/README.md

@@ -178,12 +178,6 @@ By default, progress bars are enabled:
 result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_bool=True, )
 ```
 
-You can also pass extra keyword arguments to `tqdm` through `pbar_kwargs`:
-
-```python
-result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_kwargs={"leave": False}, )
-```
-
 Verbose logging can be enabled with:
 
 ```python
@@ -247,9 +241,17 @@ param_values = [[[1, 2], [10, 20]], [[100, 200, 300]], ]
 Before opening a PR or publishing a release, run:
 
 ```bash
-pytest
+pytest -m "not mpi"
 ruff check .
-mypy src/parameterrun
+mypy
 python -m build
 python -m twine check dist/*
 ```
+
+For MPI numerical-equivalence checks (Linux/OpenMPI, 4 ranks):
+
+```bash
+uv sync --extra mpi
+mpirun -n 4 uv run pytest -m mpi -o addopts="-q -m mpi"
+```
+

parameterrun-0.4.0/examples/minimal_example_mpi.py

@@ -0,0 +1,21 @@
+import numpy as np
+from mpi4py import MPI
+
+from parameterrun import parameterrun
+
+
+def f(x, y, scale=1):
+    return scale * (x + y)
+
+
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+size = comm.Get_size()
+
+xs = np.linspace(0, 10, 100)
+ys = np.linspace(0, 20, 50)
+
+result = parameterrun(f, param_names=[["x"], ["y"]], param_values=[[xs], [ys]], scale=2, backend='mpi')
+
+if rank == 0:
+    print(np.all(result == 2 * (xs[:, None] + ys[None, :])))

{parameterrun-0.3.0 → parameterrun-0.4.0}/pyproject.toml

@@ -64,6 +64,9 @@ packages = ["src/parameterrun"]
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 addopts = "-q --cov=parameterrun --cov-report=term-missing --cov-fail-under=70"
+markers = [
+    "mpi: tests that require mpi4py and an MPI launcher",
+]
 
 [tool.ruff]
 target-version = "py39"

{parameterrun-0.3.0 → parameterrun-0.4.0}/src/parameterrun/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 from .parallel_utils import parameterrun
 

{parameterrun-0.3.0 → parameterrun-0.4.0}/src/parameterrun/parallel_utils.py

@@ -4,6 +4,7 @@ import sys
 from collections.abc import Iterable
 from datetime import datetime
 from itertools import product
+from math import prod
 from time import time
 from typing import Any, Callable, List, Optional, Tuple, Union, cast
 
@@ -58,12 +59,8 @@ def tqdm_joblib(tqdm_object: tqdm):
 
 
 def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]], param_values: List[List[List]],
-                         indices_iterate: list, desc, n_workers: Optional[int] = -1, pbar_bool: bool = True,
+                         indices_iterate: Iterable, desc, n_workers: Optional[int] = -1, pbar_bool: bool = True,
                          leave: Optional[bool] = True, **fun_kwargs) -> Tuple[List[List], int]:
-    n_total = len(indices_iterate)
-    if n_total == 0:
-        raise ValueError('No parameter combinations generated. Please provide at least one value per group.')
-
     result = []
     if n_workers == 1:  # If only one worker, do not use joblib
         pbar = tqdm(indices_iterate, desc=desc, leave=leave, disable=not pbar_bool)
@@ -75,6 +72,8 @@ def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]],
                 delayed(fun)(**{**_get_iteration(index, param_names, param_values), **fun_kwargs}) for index in
                 indices_iterate)
 
+    n_total = len(result)
+
     n_output = len(result[0]) if isinstance(result[0], tuple) else 1
     if n_output == 1:
         result = [(result_i,) for result_i in result]
@@ -86,9 +85,19 @@ def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]],
     return result_temp, n_output
 
 
+def _product_element_from_index(lists: List[Any], k: int) -> List:
+    lengths = [len(L) for L in lists]
+
+    indices = [0] * len(lists)
+    for i in range(len(lists) - 1, -1, -1):
+        k, indices[i] = divmod(k, lengths[i])
+
+    return list(L[j] for L, j in zip(lists, indices))
+
+
 def _parameterrun_mpi(fun: Callable[..., Any], param_names: List[List[str]], param_values: List[List[List]],
-                      groups_iterate: list, desc: Optional[str], pbar_bool: bool = True,
-                      verbose: Optional[bool] = False, leave: Optional[bool] = True, **fun_kwargs) -> Tuple[
+                      desc: Optional[str], pbar_bool: bool = True, verbose: Optional[bool] = False,
+                      leave: Optional[bool] = True, seed=None, **fun_kwargs) -> Tuple[
     List[List], int]:  # pragma: no cover
     from mpi4py import MPI
 
@@ -96,19 +105,25 @@ def _parameterrun_mpi(fun: Callable[..., Any], param_names: List[List[str]], par
     rank = comm.Get_rank()
     size = comm.Get_size()
 
-    n_total = len(groups_iterate)
+    n_values = [len(group[0]) for group in param_values]
+    indices = [list(range(n_value)) for n_value in n_values]
+    n_total = prod(n_values)  # Compute the total number of iterations
 
     # Shuffle the indices to avoid bias in the computation
     if rank == 0:
+        rng = np.random.default_rng(seed=seed)
         total_indices = np.arange(n_total)
-        np.random.shuffle(total_indices)
+        rng.shuffle(total_indices)
     else:
         total_indices = None
 
     total_indices = comm.bcast(total_indices, root=0)
     indices_compute = np.array_split(total_indices, size)[rank]
 
-    groups_compute = np.array(groups_iterate)[indices_compute]
+    # groups_compute = np.array(groups_iterate)[indices_compute]
+    groups_compute = []
+    for index in indices_compute:
+        groups_compute.append(_product_element_from_index(indices, index))
 
     pbar = tqdm(groups_compute, desc=desc, leave=leave, disable=not (pbar_bool and rank == 0), file=sys.stdout)
 
@@ -158,7 +173,7 @@ def _parameterrun_mpi(fun: Callable[..., Any], param_names: List[List[str]], par
 
         pbar.close()
     else:
-        comm.send(1, dest=0, tag=0)  # Notify root that the work is done
+        comm.send(True, dest=0, tag=0)  # Notify root that the work is done
 
     n_outputs = comm.bcast(n_outputs, root=0)
 
@@ -327,7 +342,7 @@ def _convert_result_to_lists(result: Any) -> Any:
 def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], List[List[str]]],
                  param_values: Union[Iterable[Any], Iterable[Iterable[Any]], Iterable[Iterable[Iterable[Any]]]],
                  n_workers: Optional[int] = -1, pbar_bool: bool = True, verbose: Optional[bool] = False,
-                 reshape: Optional[bool] = True, result_as_array: Optional[bool] = True, backend: Optional[str] = None,
+                 reshape: bool = True, result_as_array: bool = True, backend: Optional[str] = None,
                  desc: Optional[str] = None, **kwargs) -> Union[list, np.ndarray, None]:  # noqa: E501
     """
     Run a function with multiple parameters in parallel. To indentify the parameters of interest, user must provide its
@@ -393,9 +408,6 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
     if any(n_value == 0 for n_value in n_values):
         raise ValueError('Parameter groups must contain at least one value.')
 
-    indices = [list(range(n_value)) for n_value in n_values]
-    indices_iterate = product(*indices)
-
     # Choose the backed
     if backend is not None:
         available_backends = ['joblib', 'mpi']
@@ -437,16 +449,17 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
     if backend == 'joblib':
         _log('Running under joblib', verbose)
 
-        result, n_outputs = _parameterrun_joblib(fun, formatted_param_names, formatted_param_values,
-                                                 list(indices_iterate), desc, n_workers=n_workers, pbar_bool=pbar_bool,
-                                                 **kwargs)
+        indices = [list(range(n_value)) for n_value in n_values]
+        indices_iterate = product(*indices)
+        result, n_outputs = _parameterrun_joblib(fun, formatted_param_names, formatted_param_values, indices_iterate,
+                                                 desc, n_workers=n_workers, pbar_bool=pbar_bool, **kwargs)
 
     elif backend == 'mpi':
         _log(f'Running under mpi with {size} workers', verbose and rank == 0,
              hostname=mpi_module.Get_processor_name() if mpi_module is not None else None, )
 
-        result, n_outputs = _parameterrun_mpi(fun, formatted_param_names, formatted_param_values, list(indices_iterate),
-                                              desc, pbar_bool=pbar_bool, verbose=verbose, **kwargs)
+        result, n_outputs = _parameterrun_mpi(fun, formatted_param_names, formatted_param_values, desc,
+                                              pbar_bool=pbar_bool, verbose=verbose, **kwargs)
 
     else:
         raise ValueError('Unknown backend')

parameterrun-0.4.0/tests/test_mpi_integration.py

@@ -0,0 +1,30 @@
+import numpy as np
+import pytest
+
+import parameterrun.parallel_utils as pu
+
+
+@pytest.mark.mpi
+def test_parameterrun_mpi_numerical_equivalence_and_root_return():
+    MPI = pytest.importorskip("mpi4py.MPI")
+    comm = MPI.COMM_WORLD
+
+    if comm.Get_size() != 4:
+        pytest.skip("Run this test with mpirun -n 4")
+
+    xs = np.array([0.0, 1.0, 2.0])
+    ys = np.array([10.0, 20.0])
+
+    def f(x, y, scale=1.0):
+        return scale * (x + y)
+
+    result = pu.parameterrun(f, param_names=[["x"], ["y"]], param_values=[[xs], [ys]], scale=2.0, backend="mpi",
+                             pbar_bool=False, )
+
+    expected = 2.0 * (xs[:, None] + ys[None, :])
+
+    if comm.Get_rank() == 0:
+        assert isinstance(result, np.ndarray)
+        np.testing.assert_allclose(result, expected)
+    else:
+        assert result is None

parameterrun-0.3.0/.github/workflows/ci.yml

@@ -1,40 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [ main ]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    name: ${{ matrix.os }} / py${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
-
-    steps:
-      - uses: actions/checkout@v6
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install uv
-        run: python -m pip install uv
-
-      - name: Sync environment
-        run: uv sync
-
-      - name: Tests
-        run: uv run pytest
-
-      - name: Lint
-        run: uv run ruff check .
-
-      - name: Type check
-        run: uv run mypy