parameterrun 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parameterrun-0.2.1 → parameterrun-0.3.0}/.github/workflows/ci.yml +5 -13
- {parameterrun-0.2.1 → parameterrun-0.3.0}/CHANGELOG.md +25 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/CONTRIBUTING.md +1 -1
- {parameterrun-0.2.1 → parameterrun-0.3.0}/PKG-INFO +15 -5
- {parameterrun-0.2.1 → parameterrun-0.3.0}/README.md +14 -4
- {parameterrun-0.2.1 → parameterrun-0.3.0}/examples/minimal_example.py +0 -1
- {parameterrun-0.2.1 → parameterrun-0.3.0}/src/parameterrun/__init__.py +1 -1
- {parameterrun-0.2.1 → parameterrun-0.3.0}/src/parameterrun/parallel_utils.py +80 -49
- parameterrun-0.3.0/tests/test_dephs.py +36 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/tests/test_format_input.py +22 -3
- parameterrun-0.3.0/tests/test_joblib.py +14 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/tests/test_parameter_run.py +22 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/.github/workflows/publish.yml +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/.github/workflows/release.yml +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/.gitignore +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/LICENSE +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/pyproject.toml +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/scripts/bump_version.py +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/tests/test_get_iteration.py +0 -0
- {parameterrun-0.2.1 → parameterrun-0.3.0}/tests/test_normalized_time.py +0 -0
|
@@ -3,11 +3,6 @@ name: CI
|
|
|
3
3
|
on:
|
|
4
4
|
pull_request:
|
|
5
5
|
branches: [ main ]
|
|
6
|
-
# push:
|
|
7
|
-
# branches: [ main ]
|
|
8
|
-
# Optional only if you use GitHub merge queue:
|
|
9
|
-
# merge_group:
|
|
10
|
-
# types: [checks_requested]
|
|
11
6
|
|
|
12
7
|
permissions:
|
|
13
8
|
contents: read
|
|
@@ -35,14 +30,11 @@ jobs:
|
|
|
35
30
|
- name: Sync environment
|
|
36
31
|
run: uv sync
|
|
37
32
|
|
|
33
|
+
- name: Tests
|
|
34
|
+
run: uv run pytest
|
|
35
|
+
|
|
38
36
|
- name: Lint
|
|
39
37
|
run: uv run ruff check .
|
|
40
38
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
# - name: Type check
|
|
45
|
-
# run: uv run mypy
|
|
46
|
-
|
|
47
|
-
- name: Tests
|
|
48
|
-
run: uv run pytest
|
|
39
|
+
- name: Type check
|
|
40
|
+
run: uv run mypy
|
|
@@ -4,8 +4,33 @@ All notable changes to this project are documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on Keep a Changelog and the project follows Semantic Versioning.
|
|
6
6
|
|
|
7
|
+
## [0.3.1] - 2026-04-01
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- (Your changes here)
|
|
11
|
+
|
|
12
|
+
## [0.3.0] - 2026-04-01
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
- (Your changes here)
|
|
16
|
+
|
|
17
|
+
## [0.3.0] - 2026-04-01
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
- (Your changes here)
|
|
21
|
+
|
|
7
22
|
## [Unreleased]
|
|
8
23
|
|
|
24
|
+
## [0.3.0] - 2026-04-01
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- Added `result_as_array` to `parameterrun(...)`: outputs are numpy arrays by default and can be converted to Python
|
|
29
|
+
lists with `result_as_array=False`.
|
|
30
|
+
- Removed the `pbar_kwargs` optional argument from `parameterrun(...)`, as it was unused.
|
|
31
|
+
- Added more tests for the depth of the parameter grid.
|
|
32
|
+
- Added the possibility to add parameter values as numpy arrays, which are converted to lists internally.
|
|
33
|
+
|
|
9
34
|
## [0.2.1] - 2026-04-01
|
|
10
35
|
|
|
11
36
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parameterrun
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Parallel parameter sweeps with joblib/MPI and tqdm progress bars.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Davtax/ParameterRun
|
|
6
6
|
Project-URL: Repository, https://github.com/Davtax/ParameterRun
|
|
@@ -146,9 +146,16 @@ print(mean_like)
|
|
|
146
146
|
# array([2, 3, 4])
|
|
147
147
|
|
|
148
148
|
print(square_like) # array([1, 4, 9])
|
|
149
|
+
|
|
150
|
+
# Ask for plain Python lists instead of numpy arrays
|
|
151
|
+
mean_like, square_like = parameterrun(stats, param_names="x", param_values=[1, 2, 3], n_workers=1,
|
|
152
|
+
result_as_array=False, )
|
|
153
|
+
|
|
154
|
+
print(mean_like) # [2, 3, 4]
|
|
149
155
|
```
|
|
150
156
|
|
|
151
157
|
If the function returns a single object, the result is returned as a single array when reshaping is possible.
|
|
158
|
+
Set `result_as_array=False` to convert outputs to Python lists.
|
|
152
159
|
|
|
153
160
|
For multi-dimensional sweeps, reshaping is automatic when `reshape=True` (default): the first dimensions always match
|
|
154
161
|
the parameter grid shape.
|
|
@@ -223,18 +230,18 @@ The progress-bar description is generated automatically from the function name a
|
|
|
223
230
|
`parameterrun`
|
|
224
231
|
|
|
225
232
|
```python
|
|
226
|
-
parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbose=False,
|
|
227
|
-
reshape=True, backend=None, desc=None, **kwargs, )
|
|
233
|
+
parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbose=False,
|
|
234
|
+
reshape=True, result_as_array=True, backend=None, desc=None, **kwargs, )
|
|
228
235
|
```
|
|
229
236
|
|
|
230
237
|
* `fun`: function to evaluate
|
|
231
238
|
* `param_names`: parameter name or grouped parameter names
|
|
232
|
-
* `param_values`: values corresponding to `param_names`
|
|
239
|
+
* `param_values`: values corresponding to `param_names` (any iterable, e.g. list, tuple, range, numpy array)
|
|
233
240
|
* `n_workers`: number of workers for `joblib`
|
|
234
241
|
* `pbar_bool`: enable or disable progress bars
|
|
235
242
|
* `verbose`: print backend and timing information
|
|
236
|
-
* `pbar_kwargs`: additional keyword arguments passed to tqdm
|
|
237
243
|
* `reshape`: reshape results into arrays when possible
|
|
244
|
+
* `result_as_array`: return numpy arrays by default; if `False`, convert outputs to Python lists
|
|
238
245
|
* `backend`: `"joblib"`, `"mpi"`, or `None`
|
|
239
246
|
* `desc`: custom progress-bar label
|
|
240
247
|
* `**kwargs`: extra fixed keyword arguments forwarded to `fun`
|
|
@@ -243,6 +250,9 @@ parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbo
|
|
|
243
250
|
|
|
244
251
|
`parameterrun` accepts three input styles.
|
|
245
252
|
|
|
253
|
+
At every style, value containers can be any iterable (for example lists, tuples, ranges, generators, or numpy
|
|
254
|
+
arrays).
|
|
255
|
+
|
|
246
256
|
### A. Single parameter
|
|
247
257
|
|
|
248
258
|
```python
|
|
@@ -114,9 +114,16 @@ print(mean_like)
|
|
|
114
114
|
# array([2, 3, 4])
|
|
115
115
|
|
|
116
116
|
print(square_like) # array([1, 4, 9])
|
|
117
|
+
|
|
118
|
+
# Ask for plain Python lists instead of numpy arrays
|
|
119
|
+
mean_like, square_like = parameterrun(stats, param_names="x", param_values=[1, 2, 3], n_workers=1,
|
|
120
|
+
result_as_array=False, )
|
|
121
|
+
|
|
122
|
+
print(mean_like) # [2, 3, 4]
|
|
117
123
|
```
|
|
118
124
|
|
|
119
125
|
If the function returns a single object, the result is returned as a single array when reshaping is possible.
|
|
126
|
+
Set `result_as_array=False` to convert outputs to Python lists.
|
|
120
127
|
|
|
121
128
|
For multi-dimensional sweeps, reshaping is automatic when `reshape=True` (default): the first dimensions always match
|
|
122
129
|
the parameter grid shape.
|
|
@@ -191,18 +198,18 @@ The progress-bar description is generated automatically from the function name a
|
|
|
191
198
|
`parameterrun`
|
|
192
199
|
|
|
193
200
|
```python
|
|
194
|
-
parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbose=False,
|
|
195
|
-
reshape=True, backend=None, desc=None, **kwargs, )
|
|
201
|
+
parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbose=False,
|
|
202
|
+
reshape=True, result_as_array=True, backend=None, desc=None, **kwargs, )
|
|
196
203
|
```
|
|
197
204
|
|
|
198
205
|
* `fun`: function to evaluate
|
|
199
206
|
* `param_names`: parameter name or grouped parameter names
|
|
200
|
-
* `param_values`: values corresponding to `param_names`
|
|
207
|
+
* `param_values`: values corresponding to `param_names` (any iterable, e.g. list, tuple, range, numpy array)
|
|
201
208
|
* `n_workers`: number of workers for `joblib`
|
|
202
209
|
* `pbar_bool`: enable or disable progress bars
|
|
203
210
|
* `verbose`: print backend and timing information
|
|
204
|
-
* `pbar_kwargs`: additional keyword arguments passed to tqdm
|
|
205
211
|
* `reshape`: reshape results into arrays when possible
|
|
212
|
+
* `result_as_array`: return numpy arrays by default; if `False`, convert outputs to Python lists
|
|
206
213
|
* `backend`: `"joblib"`, `"mpi"`, or `None`
|
|
207
214
|
* `desc`: custom progress-bar label
|
|
208
215
|
* `**kwargs`: extra fixed keyword arguments forwarded to `fun`
|
|
@@ -211,6 +218,9 @@ parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbo
|
|
|
211
218
|
|
|
212
219
|
`parameterrun` accepts three input styles.
|
|
213
220
|
|
|
221
|
+
At every style, value containers can be any iterable (for example lists, tuples, ranges, generators, or numpy
|
|
222
|
+
arrays).
|
|
223
|
+
|
|
214
224
|
### A. Single parameter
|
|
215
225
|
|
|
216
226
|
```python
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import contextlib
|
|
2
2
|
import inspect
|
|
3
3
|
import sys
|
|
4
|
+
from collections.abc import Iterable
|
|
4
5
|
from datetime import datetime
|
|
5
6
|
from itertools import product
|
|
6
7
|
from time import time
|
|
7
|
-
from typing import Any, Callable, List, Optional, Tuple, Union
|
|
8
|
+
from typing import Any, Callable, List, Optional, Tuple, Union, cast
|
|
8
9
|
|
|
9
10
|
import joblib
|
|
10
11
|
import numpy as np
|
|
@@ -63,8 +64,8 @@ def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]],
|
|
|
63
64
|
if n_total == 0:
|
|
64
65
|
raise ValueError('No parameter combinations generated. Please provide at least one value per group.')
|
|
65
66
|
|
|
67
|
+
result = []
|
|
66
68
|
if n_workers == 1: # If only one worker, do not use joblib
|
|
67
|
-
result = []
|
|
68
69
|
pbar = tqdm(indices_iterate, desc=desc, leave=leave, disable=not pbar_bool)
|
|
69
70
|
for index in pbar:
|
|
70
71
|
result.append(fun(**{**_get_iteration(index, param_names, param_values), **fun_kwargs}))
|
|
@@ -86,8 +87,9 @@ def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]],
|
|
|
86
87
|
|
|
87
88
|
|
|
88
89
|
def _parameterrun_mpi(fun: Callable[..., Any], param_names: List[List[str]], param_values: List[List[List]],
|
|
89
|
-
groups_iterate: list, desc: str, pbar_bool: bool = True,
|
|
90
|
-
leave: Optional[bool] = True, **fun_kwargs) -> Tuple[
|
|
90
|
+
groups_iterate: list, desc: Optional[str], pbar_bool: bool = True,
|
|
91
|
+
verbose: Optional[bool] = False, leave: Optional[bool] = True, **fun_kwargs) -> Tuple[
|
|
92
|
+
List[List], int]: # pragma: no cover
|
|
91
93
|
from mpi4py import MPI
|
|
92
94
|
|
|
93
95
|
comm = MPI.COMM_WORLD
|
|
@@ -202,6 +204,12 @@ def _flatten_param_names(param_names: List[List[str]]) -> List[str]:
|
|
|
202
204
|
return [name for group in param_names for name in group]
|
|
203
205
|
|
|
204
206
|
|
|
207
|
+
def _to_list_iterable(values: Any, error_message: str) -> List[Any]:
|
|
208
|
+
if isinstance(values, (str, bytes)) or not isinstance(values, Iterable):
|
|
209
|
+
raise ValueError(error_message)
|
|
210
|
+
return list(values)
|
|
211
|
+
|
|
212
|
+
|
|
205
213
|
def _validate_function_arguments(fun: Callable[..., Any], param_names: List[List[str]], fun_kwargs: dict) -> None:
|
|
206
214
|
signature = inspect.signature(fun)
|
|
207
215
|
parameters = signature.parameters
|
|
@@ -231,21 +239,18 @@ def _validate_function_arguments(fun: Callable[..., Any], param_names: List[List
|
|
|
231
239
|
raise ValueError(f'Unknown keyword arguments for function {fun.__name__}: {unknown_kwargs}')
|
|
232
240
|
|
|
233
241
|
|
|
234
|
-
def _format_input(param_names: Union[str,
|
|
242
|
+
def _format_input(param_names: Union[str, List[str], List[List[str]]], param_values: Any) -> Tuple[
|
|
243
|
+
List[List[str]], List[List[List[Any]]]]:
|
|
235
244
|
# Check the depth of the parameters
|
|
236
245
|
|
|
237
246
|
if isinstance(param_names, str):
|
|
238
247
|
if not param_names:
|
|
239
248
|
raise ValueError('Parameter names must be non-empty strings')
|
|
240
|
-
if not isinstance(param_values, list):
|
|
241
|
-
raise ValueError('The parameters values must be a list of values')
|
|
242
249
|
depth = 1
|
|
243
250
|
elif isinstance(param_names, list) and len(param_names) > 0 and all(isinstance(name, str) for name in param_names):
|
|
244
251
|
if any(not name for name in param_names):
|
|
245
252
|
raise ValueError('Parameter names must be non-empty strings')
|
|
246
253
|
depth = 2
|
|
247
|
-
if not isinstance(param_values, list):
|
|
248
|
-
raise ValueError('The parameters values must be a list of values')
|
|
249
254
|
elif isinstance(param_names, list) and len(param_names) > 0 and all(
|
|
250
255
|
isinstance(group, list) for group in param_names):
|
|
251
256
|
if any(len(group) == 0 for group in param_names):
|
|
@@ -253,44 +258,53 @@ def _format_input(param_names: Union[str, list], param_values: list) -> Tuple[Li
|
|
|
253
258
|
if any(any(not isinstance(name, str) or not name for name in group) for group in param_names):
|
|
254
259
|
raise ValueError('Parameter names must be non-empty strings')
|
|
255
260
|
depth = 3
|
|
256
|
-
if not isinstance(param_values, list) or len(param_values) == 0 or not isinstance(param_values[0], list):
|
|
257
|
-
raise ValueError('The parameters values must be a list of values')
|
|
258
261
|
else:
|
|
259
262
|
raise ValueError('Unknown input format for the parameters names')
|
|
260
263
|
|
|
264
|
+
normalized_param_names: List[List[str]]
|
|
265
|
+
normalized_param_values: List[List[List[Any]]]
|
|
266
|
+
|
|
261
267
|
# Correct the input if the depth is not equal to 3
|
|
262
268
|
if depth == 1:
|
|
263
269
|
# Only one parameter is provided
|
|
264
|
-
|
|
265
|
-
|
|
270
|
+
values = _to_list_iterable(param_values, 'The parameters values must be an iterable of values')
|
|
271
|
+
normalized_param_names = [[cast(str, param_names)]]
|
|
272
|
+
normalized_param_values = [[values]]
|
|
266
273
|
elif depth == 2:
|
|
267
274
|
# Groups with a single parameters are provided
|
|
268
|
-
|
|
269
|
-
|
|
275
|
+
values = _to_list_iterable(param_values, 'The parameters values must be an iterable of values')
|
|
276
|
+
normalized_param_names = [[param_name_i] for param_name_i in cast(List[str], param_names)]
|
|
277
|
+
normalized_param_values = [
|
|
278
|
+
[_to_list_iterable(param_values_i, 'Each parameter values entry must be an iterable')] for param_values_i in
|
|
279
|
+
values]
|
|
280
|
+
else:
|
|
281
|
+
groups_values = _to_list_iterable(param_values, 'The parameters values must be an iterable of values')
|
|
282
|
+
groups_values = [_to_list_iterable(values_group, 'Each parameter group values must be provided as an iterable')
|
|
283
|
+
for values_group in groups_values]
|
|
284
|
+
normalized_param_names = cast(List[List[str]], param_names)
|
|
285
|
+
normalized_param_values = [
|
|
286
|
+
[_to_list_iterable(parameter_values, 'Each parameter values entry must be an iterable') for parameter_values
|
|
287
|
+
in values_group] for values_group in groups_values]
|
|
270
288
|
|
|
271
289
|
# Check that the number of groups of parameters is the same as the number of groups of values
|
|
272
|
-
if len(
|
|
290
|
+
if len(normalized_param_names) != len(normalized_param_values):
|
|
273
291
|
raise ValueError('The number of groups of parameters must be the same as the number of groups of values')
|
|
274
292
|
|
|
275
|
-
for names_group, values_group in zip(
|
|
276
|
-
if not isinstance(values_group, list):
|
|
277
|
-
raise ValueError('Each parameter group values must be provided as a list')
|
|
293
|
+
for names_group, values_group in zip(normalized_param_names, normalized_param_values):
|
|
278
294
|
if len(names_group) != len(values_group):
|
|
279
295
|
raise ValueError('Each parameter group must include one values list per parameter')
|
|
280
|
-
if any(not isinstance(parameter_values, list) for parameter_values in values_group):
|
|
281
|
-
raise ValueError('Each parameter values entry must be a list')
|
|
282
296
|
|
|
283
297
|
# Check that all parameters in the same group have the same length
|
|
284
|
-
for group in
|
|
298
|
+
for group in normalized_param_values:
|
|
285
299
|
if len(set([len(param) for param in group])) != 1:
|
|
286
300
|
raise ValueError('All parameters in the same group must have the same length')
|
|
287
301
|
|
|
288
|
-
all_names = _flatten_param_names(
|
|
302
|
+
all_names = _flatten_param_names(normalized_param_names)
|
|
289
303
|
duplicated = sorted({name for name in all_names if all_names.count(name) > 1})
|
|
290
304
|
if duplicated:
|
|
291
305
|
raise ValueError(f'Duplicate parameter names are not allowed: {duplicated}')
|
|
292
306
|
|
|
293
|
-
return
|
|
307
|
+
return normalized_param_names, normalized_param_values
|
|
294
308
|
|
|
295
309
|
|
|
296
310
|
def _reshape_channel_result(channel_values: List[Any], n_values: List[int]) -> np.ndarray:
|
|
@@ -302,11 +316,19 @@ def _reshape_channel_result(channel_values: List[Any], n_values: List[int]) -> n
|
|
|
302
316
|
return array_result.reshape(target_shape)
|
|
303
317
|
|
|
304
318
|
|
|
319
|
+
def _convert_result_to_lists(result: Any) -> Any:
|
|
320
|
+
if isinstance(result, np.ndarray):
|
|
321
|
+
return result.tolist()
|
|
322
|
+
if isinstance(result, list):
|
|
323
|
+
return [_convert_result_to_lists(item) for item in result]
|
|
324
|
+
return result
|
|
325
|
+
|
|
326
|
+
|
|
305
327
|
def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], List[List[str]]],
|
|
306
|
-
param_values: Union[
|
|
307
|
-
|
|
308
|
-
reshape: Optional[bool] = True,
|
|
309
|
-
|
|
328
|
+
param_values: Union[Iterable[Any], Iterable[Iterable[Any]], Iterable[Iterable[Iterable[Any]]]],
|
|
329
|
+
n_workers: Optional[int] = -1, pbar_bool: bool = True, verbose: Optional[bool] = False,
|
|
330
|
+
reshape: Optional[bool] = True, result_as_array: Optional[bool] = True, backend: Optional[str] = None,
|
|
331
|
+
desc: Optional[str] = None, **kwargs) -> Union[list, np.ndarray, None]: # noqa: E501
|
|
310
332
|
"""
|
|
311
333
|
Run a function with multiple parameters in parallel. To indentify the parameters of interest, user must provide its
|
|
312
334
|
name as is written in the function definition. If more parameters should be pass to the function, they can be
|
|
@@ -332,8 +354,9 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
|
|
|
332
354
|
Function to run in parallel.
|
|
333
355
|
param_names : str or list
|
|
334
356
|
Name of the parameters to run in parallel.
|
|
335
|
-
param_values :
|
|
336
|
-
Values of the parameters to run in parallel.
|
|
357
|
+
param_values : iterable
|
|
358
|
+
Values of the parameters to run in parallel. Lists, tuples, ranges, generators, and numpy arrays are
|
|
359
|
+
accepted as long as they follow one of the supported input shapes.
|
|
337
360
|
n_workers : int, optional (default=-1)
|
|
338
361
|
Number of workers to use in parallel. If -1, the number of workers is the maximum number of cores available.
|
|
339
362
|
pbar_bool : bool, optional (default=True)
|
|
@@ -342,11 +365,12 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
|
|
|
342
365
|
Backend to use for the parallelization. If None, the backend is chosen automatically between joblib and mpi.
|
|
343
366
|
verbose : bool, optional (default=False)
|
|
344
367
|
If True, print information about the parallelization.
|
|
345
|
-
pbar_kwargs : dict, optional (default=None)
|
|
346
|
-
Dictionary with the parameters to pass to the tqdm such as desc and leave.
|
|
347
368
|
reshape : bool, optional (default=True)
|
|
348
369
|
If True, reshape the result as a (hyper)matrix. Sometimes, due to the nature of the output, the reshape is not
|
|
349
370
|
possible, in this case, the result is not reshaped.
|
|
371
|
+
result_as_array : bool, optional (default=True)
|
|
372
|
+
If True, return numpy arrays when possible (default behavior). If False, convert numpy-array outputs to Python
|
|
373
|
+
lists.
|
|
350
374
|
desc: str, optional (default=None)
|
|
351
375
|
Description of the progress bar. If None, the description denotes the function and parameters names.
|
|
352
376
|
kwargs :
|
|
@@ -359,16 +383,13 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
|
|
|
359
383
|
n_valuesN), where n_valuesI is the number of values in the I-th group.
|
|
360
384
|
"""
|
|
361
385
|
|
|
362
|
-
if pbar_kwargs is None:
|
|
363
|
-
pbar_kwargs = {}
|
|
364
|
-
|
|
365
386
|
time_start = time()
|
|
366
387
|
|
|
367
|
-
|
|
368
|
-
_validate_function_arguments(fun,
|
|
388
|
+
formatted_param_names, formatted_param_values = _format_input(param_names, param_values)
|
|
389
|
+
_validate_function_arguments(fun, formatted_param_names, kwargs)
|
|
369
390
|
|
|
370
391
|
# Create the list of dictionaries with the parameters in a nested loop
|
|
371
|
-
n_values = [len(group[0]) for group in
|
|
392
|
+
n_values = [len(group[0]) for group in formatted_param_values]
|
|
372
393
|
if any(n_value == 0 for n_value in n_values):
|
|
373
394
|
raise ValueError('Parameter groups must contain at least one value.')
|
|
374
395
|
|
|
@@ -402,43 +423,53 @@ def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], Lis
|
|
|
402
423
|
backend = 'mpi'
|
|
403
424
|
|
|
404
425
|
if desc is None:
|
|
405
|
-
param_names_pbar = [parameters_names_i for parameters_names_i in
|
|
406
|
-
|
|
426
|
+
param_names_pbar = [parameters_names_i for parameters_names_i in formatted_param_names]
|
|
427
|
+
auto_desc = f'{fun.__name__}: {param_names_pbar}'
|
|
428
|
+
|
|
429
|
+
if len(auto_desc) > 40:
|
|
430
|
+
auto_desc = auto_desc[:40] + '(...)]'
|
|
407
431
|
|
|
408
|
-
|
|
409
|
-
desc = desc[:40] + '(...)'
|
|
432
|
+
desc = auto_desc
|
|
410
433
|
|
|
411
434
|
# Execute the parallel run
|
|
435
|
+
result: List[List[Any]]
|
|
436
|
+
|
|
412
437
|
if backend == 'joblib':
|
|
413
438
|
_log('Running under joblib', verbose)
|
|
414
439
|
|
|
415
|
-
result, n_outputs = _parameterrun_joblib(fun,
|
|
416
|
-
n_workers=n_workers, pbar_bool=pbar_bool,
|
|
440
|
+
result, n_outputs = _parameterrun_joblib(fun, formatted_param_names, formatted_param_values,
|
|
441
|
+
list(indices_iterate), desc, n_workers=n_workers, pbar_bool=pbar_bool,
|
|
442
|
+
**kwargs)
|
|
417
443
|
|
|
418
444
|
elif backend == 'mpi':
|
|
419
445
|
_log(f'Running under mpi with {size} workers', verbose and rank == 0,
|
|
420
446
|
hostname=mpi_module.Get_processor_name() if mpi_module is not None else None, )
|
|
421
447
|
|
|
422
|
-
result, n_outputs = _parameterrun_mpi(fun,
|
|
423
|
-
pbar_bool=pbar_bool, verbose=verbose, **
|
|
448
|
+
result, n_outputs = _parameterrun_mpi(fun, formatted_param_names, formatted_param_values, list(indices_iterate),
|
|
449
|
+
desc, pbar_bool=pbar_bool, verbose=verbose, **kwargs)
|
|
424
450
|
|
|
425
451
|
else:
|
|
426
452
|
raise ValueError('Unknown backend')
|
|
427
453
|
|
|
428
454
|
if rank == 0:
|
|
455
|
+
final_result: Union[List[List[Any]], np.ndarray, List[np.ndarray]] = result
|
|
456
|
+
|
|
429
457
|
# Reshape the result to get the (hyper)matrix
|
|
430
458
|
if reshape:
|
|
431
459
|
try:
|
|
432
460
|
if n_outputs == 1:
|
|
433
|
-
|
|
461
|
+
final_result = _reshape_channel_result(result[0], n_values)
|
|
434
462
|
else:
|
|
435
|
-
|
|
463
|
+
final_result = [_reshape_channel_result(result[i], n_values) for i in range(n_outputs)]
|
|
436
464
|
except ValueError:
|
|
437
465
|
print('Could not reshape the result')
|
|
438
466
|
|
|
439
467
|
total_time = time() - time_start
|
|
440
468
|
_log(f'Total time: {_normalized_time(total_time)}', verbose)
|
|
441
469
|
|
|
442
|
-
|
|
470
|
+
if not result_as_array:
|
|
471
|
+
final_result = _convert_result_to_lists(final_result)
|
|
472
|
+
|
|
473
|
+
return final_result
|
|
443
474
|
else:
|
|
444
475
|
return None
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from parameterrun import parameterrun
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_depth_1():
|
|
7
|
+
def fun(x):
|
|
8
|
+
return x ** 2
|
|
9
|
+
|
|
10
|
+
xs = np.linspace(0, 10, 5)
|
|
11
|
+
result = parameterrun(fun, 'x', xs, n_workers=1, pbar_bool=False)
|
|
12
|
+
|
|
13
|
+
assert isinstance(result, np.ndarray)
|
|
14
|
+
np.testing.assert_array_equal(result, xs ** 2)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_depth_2():
|
|
18
|
+
def fun(x):
|
|
19
|
+
return x ** 2
|
|
20
|
+
|
|
21
|
+
xs = np.linspace(0, 10, 5)
|
|
22
|
+
result = parameterrun(fun, ['x'], [xs], n_workers=1, pbar_bool=False)
|
|
23
|
+
|
|
24
|
+
assert isinstance(result, np.ndarray)
|
|
25
|
+
np.testing.assert_array_equal(result, xs ** 2)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_depth_3():
|
|
29
|
+
def fun(x):
|
|
30
|
+
return x ** 2
|
|
31
|
+
|
|
32
|
+
xs = np.linspace(0, 10, 5)
|
|
33
|
+
result = parameterrun(fun, [['x']], [[xs]], n_workers=1, pbar_bool=False)
|
|
34
|
+
|
|
35
|
+
assert isinstance(result, np.ndarray)
|
|
36
|
+
np.testing.assert_array_equal(result, xs ** 2)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import numpy as np
|
|
1
2
|
import pytest
|
|
2
3
|
|
|
3
4
|
from parameterrun.parallel_utils import _format_input
|
|
@@ -25,9 +26,27 @@ def test_format_input_rejects_empty_parameter_name():
|
|
|
25
26
|
_format_input("", [1, 2])
|
|
26
27
|
|
|
27
28
|
|
|
28
|
-
def
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
def test_format_input_accepts_numpy_array_values():
|
|
30
|
+
names, values = _format_input("x", np.array([1, 2, 3]))
|
|
31
|
+
assert names == [["x"]]
|
|
32
|
+
assert values == [[[1, 2, 3]]]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_format_input_accepts_iterables_in_multiple_parameter_format():
|
|
36
|
+
names, values = _format_input(["x", "y"], (range(3), (10, 20, 30)))
|
|
37
|
+
assert names == [["x"], ["y"]]
|
|
38
|
+
assert values == [[[0, 1, 2]], [[10, 20, 30]]]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_format_input_accepts_tuple_parameter_values_inside_group():
|
|
42
|
+
names, values = _format_input([["x"]], [[(1, 2)]])
|
|
43
|
+
assert names == [["x"]]
|
|
44
|
+
assert values == [[[1, 2]]]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_format_input_rejects_string_values():
|
|
48
|
+
with pytest.raises(ValueError, match="iterable of values"):
|
|
49
|
+
_format_input("x", "123")
|
|
31
50
|
|
|
32
51
|
|
|
33
52
|
def test_format_input_rejects_duplicate_names():
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from parameterrun import parameterrun
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_joblib_2_workers():
|
|
7
|
+
def f(x):
|
|
8
|
+
return x ** 2
|
|
9
|
+
|
|
10
|
+
xs = np.linspace(0, 10, 5)
|
|
11
|
+
result = parameterrun(f, 'x', xs, n_workers=2, pbar_bool=False, backend="joblib")
|
|
12
|
+
|
|
13
|
+
assert isinstance(result, np.ndarray)
|
|
14
|
+
np.testing.assert_array_equal(result, xs ** 2)
|
|
@@ -15,6 +15,17 @@ def test_parameterrun_single_parameter_returns_1d_array():
|
|
|
15
15
|
np.testing.assert_array_equal(result, np.array([0, 1, 4, 9]))
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
def test_parameterrun_single_parameter_can_return_list():
|
|
19
|
+
def square(x):
|
|
20
|
+
return x ** 2
|
|
21
|
+
|
|
22
|
+
result = pu.parameterrun(square, param_names="x", param_values=[0, 1, 2, 3], n_workers=1, pbar_bool=False,
|
|
23
|
+
backend="joblib", result_as_array=False, )
|
|
24
|
+
|
|
25
|
+
assert isinstance(result, list)
|
|
26
|
+
assert result == [0, 1, 4, 9]
|
|
27
|
+
|
|
28
|
+
|
|
18
29
|
def test_parameterrun_grouped_parameters_builds_expected_grid():
|
|
19
30
|
def f(x, y, z, offset=0):
|
|
20
31
|
return x + y + z + offset
|
|
@@ -47,6 +58,17 @@ def test_parameterrun_multiple_outputs_returns_list_of_arrays():
|
|
|
47
58
|
np.testing.assert_array_equal(result[1], np.array([1, 4, 9]))
|
|
48
59
|
|
|
49
60
|
|
|
61
|
+
def test_parameterrun_multiple_outputs_can_return_list_of_lists():
|
|
62
|
+
def f(x):
|
|
63
|
+
return x + 1, x ** 2
|
|
64
|
+
|
|
65
|
+
result = pu.parameterrun(f, param_names="x", param_values=[1, 2, 3], n_workers=1, pbar_bool=False,
|
|
66
|
+
backend="joblib", result_as_array=False, )
|
|
67
|
+
|
|
68
|
+
assert isinstance(result, list)
|
|
69
|
+
assert result == [[2, 3, 4], [1, 4, 9]]
|
|
70
|
+
|
|
71
|
+
|
|
50
72
|
def test_parameterrun_multiple_outputs_keep_grid_shape():
|
|
51
73
|
def f(x, y):
|
|
52
74
|
return x + y, x * y
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|