parameterrun 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
parameterrun/__init__.py
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import sys
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from itertools import product
|
|
5
|
+
from time import time
|
|
6
|
+
from typing import Any, Callable, List, Optional, Tuple, Union
|
|
7
|
+
|
|
8
|
+
import joblib
|
|
9
|
+
import numpy as np
|
|
10
|
+
from joblib import Parallel, delayed
|
|
11
|
+
from tqdm.auto import tqdm
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _normalized_time(time_seg: float) -> str:
|
|
15
|
+
"""
|
|
16
|
+
Normalize the time to a human-readable format
|
|
17
|
+
"""
|
|
18
|
+
if time_seg < 60:
|
|
19
|
+
return f'{time_seg:.2f} s'
|
|
20
|
+
elif time_seg < 3600:
|
|
21
|
+
mins, sec = divmod(time_seg, 60)
|
|
22
|
+
return f'{int(mins)} min {sec:.2f} s'
|
|
23
|
+
else:
|
|
24
|
+
hours, time_seg = divmod(time_seg, 3600)
|
|
25
|
+
mins, sec = divmod(time_seg, 60)
|
|
26
|
+
return f'{int(hours)} h {int(mins)} min {sec:.2f} s'
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _log(message: str, verbose: Optional[bool] = False, hostname: Optional[str] = None) -> None:
|
|
30
|
+
if verbose:
|
|
31
|
+
# Compute timestamp
|
|
32
|
+
time_stamp = datetime.now().strftime("%H:%M:%S, %d/%m/%Y")
|
|
33
|
+
msg = f'{time_stamp}: {message}'
|
|
34
|
+
if hostname is not None:
|
|
35
|
+
msg = f'[{hostname}]: {msg}'
|
|
36
|
+
|
|
37
|
+
print(msg, flush=True)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@contextlib.contextmanager
|
|
41
|
+
def tqdm_joblib(tqdm_object: tqdm):
|
|
42
|
+
"""Context manager to patch joblib to report into tqdm progress bar given as argument"""
|
|
43
|
+
|
|
44
|
+
class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
|
|
45
|
+
def __call__(self, *args, **kwargs):
|
|
46
|
+
tqdm_object.update(n=self.batch_size)
|
|
47
|
+
return super().__call__(*args, **kwargs)
|
|
48
|
+
|
|
49
|
+
old_batch_callback = joblib.parallel.BatchCompletionCallBack
|
|
50
|
+
joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
|
|
51
|
+
try:
|
|
52
|
+
yield tqdm_object
|
|
53
|
+
finally:
|
|
54
|
+
joblib.parallel.BatchCompletionCallBack = old_batch_callback
|
|
55
|
+
tqdm_object.close()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _parameterrun_joblib(fun: Callable[..., Any], param_names: List[List[str]], param_values: List[List[List]],
|
|
59
|
+
indices_iterate: list, desc, n_workers: Optional[int] = -1, pbar_bool: bool = True,
|
|
60
|
+
leave: Optional[bool] = True, **fun_kwargs) -> Tuple[List[List], int]:
|
|
61
|
+
n_total = len(indices_iterate)
|
|
62
|
+
if n_total == 0:
|
|
63
|
+
raise ValueError('No parameter combinations generated. Please provide at least one value per group.')
|
|
64
|
+
|
|
65
|
+
if n_workers == 1: # If only one worker, do not use joblib
|
|
66
|
+
result = []
|
|
67
|
+
pbar = tqdm(indices_iterate, desc=desc, leave=leave, disable=not pbar_bool)
|
|
68
|
+
for index in pbar:
|
|
69
|
+
result.append(fun(**{**_get_iteration(index, param_names, param_values), **fun_kwargs}))
|
|
70
|
+
else:
|
|
71
|
+
with tqdm_joblib(tqdm(indices_iterate, desc=desc, leave=leave, disable=not pbar_bool)) as _:
|
|
72
|
+
result = Parallel(n_jobs=n_workers)(
|
|
73
|
+
delayed(fun)(**{**_get_iteration(index, param_names, param_values), **fun_kwargs}) for index in
|
|
74
|
+
indices_iterate)
|
|
75
|
+
|
|
76
|
+
n_output = len(result[0]) if isinstance(result[0], tuple) else 1
|
|
77
|
+
if n_output == 1:
|
|
78
|
+
result = [(result_i,) for result_i in result]
|
|
79
|
+
|
|
80
|
+
result_temp = []
|
|
81
|
+
for i in range(n_output):
|
|
82
|
+
result_temp.append([result[j][i] for j in range(n_total)])
|
|
83
|
+
|
|
84
|
+
return result_temp, n_output
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _parameterrun_mpi(fun: Callable[..., Any], param_names: List[List[str]], param_values: List[List[List]],
|
|
88
|
+
groups_iterate: list, desc: str, pbar_bool: bool = True, verbose: Optional[bool] = False,
|
|
89
|
+
leave: Optional[bool] = True, **fun_kwargs) -> Tuple[List[List], int]: # pragma: no cover
|
|
90
|
+
from mpi4py import MPI
|
|
91
|
+
|
|
92
|
+
comm = MPI.COMM_WORLD
|
|
93
|
+
rank = comm.Get_rank()
|
|
94
|
+
size = comm.Get_size()
|
|
95
|
+
|
|
96
|
+
n_total = len(groups_iterate)
|
|
97
|
+
|
|
98
|
+
# Shuffle the indices to avoid bias in the computation
|
|
99
|
+
if rank == 0:
|
|
100
|
+
total_indices = np.arange(n_total)
|
|
101
|
+
np.random.shuffle(total_indices)
|
|
102
|
+
else:
|
|
103
|
+
total_indices = None
|
|
104
|
+
|
|
105
|
+
total_indices = comm.bcast(total_indices, root=0)
|
|
106
|
+
indices_compute = np.array_split(total_indices, size)[rank]
|
|
107
|
+
|
|
108
|
+
groups_compute = np.array(groups_iterate)[indices_compute]
|
|
109
|
+
|
|
110
|
+
pbar = tqdm(groups_compute, desc=desc, leave=leave, disable=not (pbar_bool and rank == 0), file=sys.stdout)
|
|
111
|
+
|
|
112
|
+
n_outputs = 0
|
|
113
|
+
result: List[Any] = []
|
|
114
|
+
|
|
115
|
+
if 'Open MPI v5.0.6' in MPI.Get_library_version():
|
|
116
|
+
mpi_flush = True
|
|
117
|
+
else:
|
|
118
|
+
mpi_flush = False
|
|
119
|
+
|
|
120
|
+
for index in pbar:
|
|
121
|
+
if pbar_bool and rank == 0 and mpi_flush:
|
|
122
|
+
print('\r', flush=True) # Needed to show the progress bar in some mpi versions
|
|
123
|
+
sys.stdout.write("\033[F") # Move the cursor up one line
|
|
124
|
+
|
|
125
|
+
result_temp = fun(**{**_get_iteration(index, param_names, param_values), **fun_kwargs})
|
|
126
|
+
|
|
127
|
+
if not isinstance(result_temp, tuple):
|
|
128
|
+
result_temp = (result_temp,)
|
|
129
|
+
|
|
130
|
+
# Check how many parameters are returned
|
|
131
|
+
if len(result) == 0: # First iteration
|
|
132
|
+
n_outputs = len(result_temp)
|
|
133
|
+
for _ in range(n_outputs):
|
|
134
|
+
result.append([])
|
|
135
|
+
|
|
136
|
+
for i, result_temp_i in enumerate(result_temp):
|
|
137
|
+
result[i].append(result_temp_i)
|
|
138
|
+
|
|
139
|
+
_log(f'Rank {rank} finished', verbose, hostname=MPI.Get_processor_name())
|
|
140
|
+
|
|
141
|
+
# Notify root when each worker is done, and show a progress bar
|
|
142
|
+
if rank == 0:
|
|
143
|
+
progress = 0
|
|
144
|
+
pbar = tqdm(total=size - 1, desc='Waiting for workers', leave=leave, disable=not pbar_bool, file=sys.stdout)
|
|
145
|
+
|
|
146
|
+
while progress < size - 1:
|
|
147
|
+
if comm.Iprobe(source=MPI.ANY_SOURCE, tag=0):
|
|
148
|
+
if pbar_bool and mpi_flush:
|
|
149
|
+
print('\r', flush=True) # Needed to show the progress bar in some mpi versions
|
|
150
|
+
sys.stdout.write("\033[F")
|
|
151
|
+
|
|
152
|
+
comm.recv(source=MPI.ANY_SOURCE, tag=0) # Receive a signal from each worker
|
|
153
|
+
progress += 1
|
|
154
|
+
pbar.update(1)
|
|
155
|
+
|
|
156
|
+
pbar.close()
|
|
157
|
+
else:
|
|
158
|
+
comm.send(1, dest=0, tag=0) # Notify root that the work is done
|
|
159
|
+
|
|
160
|
+
n_outputs = comm.bcast(n_outputs, root=0)
|
|
161
|
+
|
|
162
|
+
if not result: # For workers that did not compute anything
|
|
163
|
+
result = [[] for _ in range(n_outputs)]
|
|
164
|
+
|
|
165
|
+
results_gathered = []
|
|
166
|
+
for i in range(n_outputs):
|
|
167
|
+
_log(f'Gathering results {i}', verbose and rank == 0, hostname=MPI.Get_processor_name())
|
|
168
|
+
|
|
169
|
+
result_gathered = comm.gather(result[i], root=0)
|
|
170
|
+
|
|
171
|
+
_log('Results gathered', verbose and rank == 0, hostname=MPI.Get_processor_name())
|
|
172
|
+
|
|
173
|
+
if rank == 0:
|
|
174
|
+
result_gathered = [item for sublist in result_gathered for item in sublist] # Flatten the list
|
|
175
|
+
|
|
176
|
+
results_unshuffled = [None] * n_total
|
|
177
|
+
for j, index in enumerate(total_indices):
|
|
178
|
+
results_unshuffled[index] = result_gathered[j]
|
|
179
|
+
|
|
180
|
+
results_gathered.append(results_unshuffled)
|
|
181
|
+
|
|
182
|
+
_log('Results reshaped', verbose and rank == 0, hostname=MPI.Get_processor_name())
|
|
183
|
+
|
|
184
|
+
return results_gathered, n_outputs
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _get_iteration(index, param_names, param_values):
|
|
188
|
+
n_groups = len(param_values)
|
|
189
|
+
n_parameters = [len(group) for group in param_values]
|
|
190
|
+
|
|
191
|
+
dic_temp = {}
|
|
192
|
+
for group_index in range(n_groups):
|
|
193
|
+
for parameter_index in range(n_parameters[group_index]):
|
|
194
|
+
dic_temp[param_names[group_index][parameter_index]] = (
|
|
195
|
+
param_values[group_index][parameter_index][index[group_index]])
|
|
196
|
+
|
|
197
|
+
return dic_temp
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _format_input(param_names: Union[str, list], param_values: list) -> Tuple[List[List[str]], List[List[List]]]:
|
|
201
|
+
# Check the depth of the parameters
|
|
202
|
+
|
|
203
|
+
if type(param_names) is str:
|
|
204
|
+
depth = 1
|
|
205
|
+
elif type(param_names[0]) is str:
|
|
206
|
+
depth = 2
|
|
207
|
+
if type(param_values) is not list:
|
|
208
|
+
raise ValueError('The parameters values must be a list of values')
|
|
209
|
+
elif type(param_names[0][0]) is str:
|
|
210
|
+
depth = 3
|
|
211
|
+
if type(param_values[0]) is not list:
|
|
212
|
+
raise ValueError('The parameters values must be a list of values')
|
|
213
|
+
else:
|
|
214
|
+
raise ValueError('Unknown input format for the parameters names')
|
|
215
|
+
|
|
216
|
+
# Correct the input if the depth is not equal to 3
|
|
217
|
+
if depth == 1:
|
|
218
|
+
# Only one parameter is provided
|
|
219
|
+
param_names = [[param_names]]
|
|
220
|
+
param_values = [[param_values]]
|
|
221
|
+
elif depth == 2:
|
|
222
|
+
# Groups with a single parameters are provided
|
|
223
|
+
param_names = [[param_name_i] for param_name_i in param_names]
|
|
224
|
+
param_values = [[param_values_i] for param_values_i in param_values]
|
|
225
|
+
|
|
226
|
+
# Check that the number of groups of parameters is the same as the number of groups of values
|
|
227
|
+
if len(param_names) != len(param_values):
|
|
228
|
+
raise ValueError('The number of groups of parameters must be the same as the number of groups of values')
|
|
229
|
+
|
|
230
|
+
# Check that all parameters in the same group have the same length
|
|
231
|
+
for group in param_values:
|
|
232
|
+
if len(set([len(param) for param in group])) != 1:
|
|
233
|
+
raise ValueError('All parameters in the same group must have the same length')
|
|
234
|
+
|
|
235
|
+
return param_names, param_values # type: ignore[return-value]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def parameterrun(fun: Callable[..., Any], param_names: Union[str, List[str], List[List[str]]],
|
|
239
|
+
param_values: Union[List, List[List], List[List[List]]], n_workers: Optional[int] = -1,
|
|
240
|
+
pbar_bool: bool = True, verbose: Optional[bool] = False, pbar_kwargs: Optional[dict] = None,
|
|
241
|
+
reshape: Optional[bool] = True, backend: Optional[str] = None, desc: Optional[str] = None, **kwargs) -> \
|
|
242
|
+
Union[list, np.ndarray, None]: # noqa: E501
|
|
243
|
+
"""
|
|
244
|
+
Run a function with multiple parameters in parallel. To indentify the parameters of interest, user must provide its
|
|
245
|
+
name as is written in the function definition. If more parameters should be pass to the function, they can be
|
|
246
|
+
provided as kwargs. The backend used for the parallelization is chosen automatically between joblib and mpi4py.
|
|
247
|
+
If mpi4py is used, the function must be run with mpirun -n n_workers python script.py. Finally, the result can be
|
|
248
|
+
reshaped as a (hyper)matrix if the reshape parameter is set to True, if the functions return multiple values,
|
|
249
|
+
the result is a list of arrays.
|
|
250
|
+
|
|
251
|
+
The parameters can be grouped in different ways, the possible options are:
|
|
252
|
+
- A single parameter: The function is run with a single parameter. In this case, param_names='param_name' and
|
|
253
|
+
param_values=[value1, value2, ...]
|
|
254
|
+
- Multiple parameters: The function is run with multiple parameters. In this case, param_names=['param_name1',
|
|
255
|
+
'param_name2', ...] and param_values=[[value1, value2, ...], [value1, value2, ...], ...]
|
|
256
|
+
- Multiple groups of parameters: The function is run with multiple groups of parameters, running the parameters in
|
|
257
|
+
the same group at the same time. In this case, param_names=[['param_name1', 'param_name2', ...], ['param_name1',
|
|
258
|
+
'param_name2', ...], ...] and param_values=[[[value1, value2, ...], [value1, value2, ...], ...], [[value1, value2,
|
|
259
|
+
...], [value1, value2, ...], ...], ...]. Note that in this case, the number of values for each parameter in the same
|
|
260
|
+
group must be the same.
|
|
261
|
+
|
|
262
|
+
Parameters
|
|
263
|
+
----------
|
|
264
|
+
fun : callable
|
|
265
|
+
Function to run in parallel.
|
|
266
|
+
param_names : str or list
|
|
267
|
+
Name of the parameters to run in parallel.
|
|
268
|
+
param_values : list
|
|
269
|
+
Values of the parameters to run in parallel.
|
|
270
|
+
n_workers : int, optional (default=-1)
|
|
271
|
+
Number of workers to use in parallel. If -1, the number of workers is the maximum number of cores available.
|
|
272
|
+
pbar_bool : bool, optional (default=True)
|
|
273
|
+
If True, show a progress bar.
|
|
274
|
+
backend : str, optional (default=None)
|
|
275
|
+
Backend to use for the parallelization. If None, the backend is chosen automatically between joblib and mpi.
|
|
276
|
+
verbose : bool, optional (default=False)
|
|
277
|
+
If True, print information about the parallelization.
|
|
278
|
+
pbar_kwargs : dict, optional (default=None)
|
|
279
|
+
Dictionary with the parameters to pass to the tqdm such as desc and leave.
|
|
280
|
+
reshape : bool, optional (default=True)
|
|
281
|
+
If True, reshape the result as a (hyper)matrix. Sometimes, due to the nature of the output, the reshape is not
|
|
282
|
+
possible, in this case, the result is not reshaped.
|
|
283
|
+
desc: str, optional (default=None)
|
|
284
|
+
Description of the progress bar. If None, the description denotes the function and parameters names.
|
|
285
|
+
kwargs :
|
|
286
|
+
Additional parameters to pass to the function.
|
|
287
|
+
|
|
288
|
+
Returns
|
|
289
|
+
-------
|
|
290
|
+
result : list or np.ndarray
|
|
291
|
+
Result of the function run in parallel. The shape of the (hyper)matrix is (n_values1, n_values2, ...,
|
|
292
|
+
n_valuesN), where n_valuesI is the number of values in the I-th group.
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
if pbar_kwargs is None:
|
|
296
|
+
pbar_kwargs = {}
|
|
297
|
+
|
|
298
|
+
time_start = time()
|
|
299
|
+
|
|
300
|
+
param_names, param_values = _format_input(param_names, param_values)
|
|
301
|
+
|
|
302
|
+
# Create the list of dictionaries with the parameters in a nested loop
|
|
303
|
+
n_values = [len(group[0]) for group in param_values]
|
|
304
|
+
if any(n_value == 0 for n_value in n_values):
|
|
305
|
+
raise ValueError('Parameter groups must contain at least one value.')
|
|
306
|
+
|
|
307
|
+
indices = [list(range(n_value)) for n_value in n_values]
|
|
308
|
+
indices_iterate = product(*indices)
|
|
309
|
+
|
|
310
|
+
# Choose the backed
|
|
311
|
+
if backend is not None:
|
|
312
|
+
available_backends = ['joblib', 'mpi']
|
|
313
|
+
if backend not in available_backends:
|
|
314
|
+
raise ValueError(f'Unknown backend {backend}. Available backends are {available_backends}')
|
|
315
|
+
|
|
316
|
+
size = 1
|
|
317
|
+
rank = 0
|
|
318
|
+
|
|
319
|
+
mpi_module = None
|
|
320
|
+
if backend is None or backend == 'mpi':
|
|
321
|
+
try:
|
|
322
|
+
from mpi4py import MPI
|
|
323
|
+
mpi_module = MPI
|
|
324
|
+
comm = MPI.COMM_WORLD
|
|
325
|
+
size = comm.Get_size()
|
|
326
|
+
rank = comm.Get_rank()
|
|
327
|
+
except ImportError as err:
|
|
328
|
+
if backend == 'mpi':
|
|
329
|
+
raise ImportError('backend="mpi" requires mpi4py to be installed and available.') from err
|
|
330
|
+
|
|
331
|
+
if size == 1 and backend is None:
|
|
332
|
+
backend = 'joblib'
|
|
333
|
+
else:
|
|
334
|
+
backend = 'mpi'
|
|
335
|
+
|
|
336
|
+
if desc is None:
|
|
337
|
+
param_names_pbar = [parameters_names_i for parameters_names_i in param_names]
|
|
338
|
+
desc = f'{fun.__name__}: {param_names_pbar}'
|
|
339
|
+
|
|
340
|
+
if len(desc) > 40:
|
|
341
|
+
desc = desc[:40] + '(...)'
|
|
342
|
+
|
|
343
|
+
# Execute the parallel run
|
|
344
|
+
if backend == 'joblib':
|
|
345
|
+
_log('Running under joblib', verbose)
|
|
346
|
+
|
|
347
|
+
result, n_outputs = _parameterrun_joblib(fun, param_names, param_values, list(indices_iterate), desc,
|
|
348
|
+
n_workers=n_workers, pbar_bool=pbar_bool, **pbar_kwargs, **kwargs)
|
|
349
|
+
|
|
350
|
+
elif backend == 'mpi':
|
|
351
|
+
_log(f'Running under mpi with {size} workers', verbose and rank == 0,
|
|
352
|
+
hostname=mpi_module.Get_processor_name() if mpi_module is not None else None, )
|
|
353
|
+
|
|
354
|
+
result, n_outputs = _parameterrun_mpi(fun, param_names, param_values, list(indices_iterate), desc,
|
|
355
|
+
pbar_bool=pbar_bool, verbose=verbose, **pbar_kwargs, **kwargs)
|
|
356
|
+
|
|
357
|
+
else:
|
|
358
|
+
raise ValueError('Unknown backend')
|
|
359
|
+
|
|
360
|
+
if rank == 0:
|
|
361
|
+
# Reshape the result to get the (hyper)matrix
|
|
362
|
+
if reshape:
|
|
363
|
+
try:
|
|
364
|
+
if n_outputs == 1:
|
|
365
|
+
result = np.array(result[0])
|
|
366
|
+
else:
|
|
367
|
+
result = [np.array(result[i]) for i in range(n_outputs)]
|
|
368
|
+
except ValueError:
|
|
369
|
+
print('Could not reshape the result')
|
|
370
|
+
|
|
371
|
+
total_time = time() - time_start
|
|
372
|
+
_log(f'Total time: {_normalized_time(total_time)}', verbose)
|
|
373
|
+
|
|
374
|
+
return result
|
|
375
|
+
else:
|
|
376
|
+
return None
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: parameterrun
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Parallel parameter sweeps with joblib/MPI and tqdm progress bars.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Davtax/ParameterRun
|
|
6
|
+
Project-URL: Repository, https://github.com/Davtax/ParameterRun
|
|
7
|
+
Project-URL: Issues, https://github.com/Davtax/ParameterRun/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/Davtax/ParameterRun/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: David Fernández-Fernández <david.fernandezf@hotmail.com>
|
|
10
|
+
License-Expression: BSD-3-Clause
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: joblib,mpi,parallel,parameter-sweep,python,scientific-computing
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: joblib>=1.3
|
|
27
|
+
Requires-Dist: numpy>=1.23
|
|
28
|
+
Requires-Dist: tqdm>=4.66
|
|
29
|
+
Provides-Extra: mpi
|
|
30
|
+
Requires-Dist: mpi4py>=3.1; extra == 'mpi'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# ParameterRun
|
|
34
|
+
|
|
35
|
+
`parameterrun` is a lightweight utility for running parameter sweeps in Python with a simple, function-based API.
|
|
36
|
+
|
|
37
|
+
It is designed for scientific and numerical workflows where you want to evaluate a function over one or more parameter
|
|
38
|
+
grids, optionally in parallel, while keeping progress bars and output reshaping convenient.
|
|
39
|
+
|
|
40
|
+
The package supports:
|
|
41
|
+
|
|
42
|
+
- single-parameter scans
|
|
43
|
+
- multi-parameter scans
|
|
44
|
+
- grouped parameters that must vary together
|
|
45
|
+
- parallel execution with `joblib`
|
|
46
|
+
- MPI execution with `mpi4py`
|
|
47
|
+
- automatic reshaping of outputs into arrays
|
|
48
|
+
- functions returning either one output or multiple outputs
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
### With `pip`
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install parameterrun
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### With optional MPI support
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install parameterrun[mpi]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Basic idea
|
|
65
|
+
|
|
66
|
+
The main entry point is:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from parameterrun import parameterrun
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The function `parameterrun(...)` evaluates a user-defined function over a set of parameter values. The parameters to
|
|
73
|
+
sweep are identified by their names exactly as they appear in the target function signature. Additional fixed arguments
|
|
74
|
+
can be passed as keyword arguments.
|
|
75
|
+
|
|
76
|
+
## Quick start
|
|
77
|
+
|
|
78
|
+
### 1. Single parameter
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from parameterrun import parameterrun
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def square(x):
|
|
85
|
+
return x ** 2
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
result = parameterrun(square, param_names="x", param_values=[0, 1, 2, 3], n_workers=1, )
|
|
89
|
+
|
|
90
|
+
print(result) # array([0, 1, 4, 9])
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### 2. Multiple independent parameters
|
|
94
|
+
|
|
95
|
+
This creates a Cartesian product of the parameter groups.
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from parameterrun import parameterrun
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def f(x, y, scale=1):
|
|
102
|
+
return scale * (x + y)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
result = parameterrun(f, param_names=["x", "y"], param_values=[[1, 2, 3], [10, 20], ], scale=2, )
|
|
106
|
+
|
|
107
|
+
result = result.reshape(3, 2)
|
|
108
|
+
print(result.shape) # (3, 2)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### 3. Grouped parameters
|
|
112
|
+
|
|
113
|
+
Parameters in the same group vary together and must have the same number of values.
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from parameterrun import parameterrun
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def f(x, y, z):
|
|
120
|
+
return x + y + z
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
result = parameterrun(f, param_names=[["x", "y"], ["z"]], param_values=[[[1, 2], [10, 20]], # (x, y) = (1,10), (2,20)
|
|
124
|
+
[[100, 200, 300]], # z varies independently
|
|
125
|
+
], n_workers=1, )
|
|
126
|
+
|
|
127
|
+
result = result.reshape(2, 3)
|
|
128
|
+
print(result.shape) # (2, 3)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
This grouped-input mode is useful when some parameters are logically linked and should not be combined independently.
|
|
132
|
+
|
|
133
|
+
## Multiple outputs
|
|
134
|
+
|
|
135
|
+
If the target function returns a tuple, parameterrun returns a list of arrays, one per output channel.
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from parameterrun import parameterrun
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def stats(x):
|
|
142
|
+
return x + 1, x ** 2
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
mean_like, square_like = parameterrun(stats, param_names="x", param_values=[1, 2, 3], n_workers=1, )
|
|
146
|
+
|
|
147
|
+
print(mean_like)
|
|
148
|
+
# array([2, 3, 4])
|
|
149
|
+
|
|
150
|
+
print(square_like) # array([1, 4, 9])
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
If the function returns a single object, the result is returned as a single array when reshaping is possible.
|
|
154
|
+
|
|
155
|
+
## Parallel backend
|
|
156
|
+
|
|
157
|
+
`parameterrun` supports two execution backends:
|
|
158
|
+
|
|
159
|
+
- `joblib`: a simple, local parallelization library that works well for most use cases.
|
|
160
|
+
- `mpi`: a distributed parallelization approach using MPI, suitable for running on clusters or supercomputers.
|
|
161
|
+
|
|
162
|
+
If `backend=None`, the package tries to detect whether MPI is available. If MPI is not available, it falls back to
|
|
163
|
+
`joblib`; otherwise it uses MPI.
|
|
164
|
+
|
|
165
|
+
### Joblib backend
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
result = parameterrun(my_function, param_names=["x", "y"], param_values=[[1, 2, 3], [4, 5]], backend="joblib",
|
|
169
|
+
n_workers=-1, )
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
If `n_workers=1`, the function runs serially without spawning joblib workers.
|
|
173
|
+
|
|
174
|
+
### MPI backend
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
result = parameterrun(my_function, param_names=["x", "y"], param_values=[[1, 2, 3], [4, 5]], backend="mpi", )
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Run the script with MPI, for example:
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
mpirun -n 4 python my_script.py
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Under MPI, only rank 0 returns the final result; the other ranks return `None`.
|
|
187
|
+
|
|
188
|
+
## Progress bars and verbosity
|
|
189
|
+
|
|
190
|
+
By default, progress bars are enabled:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_bool=True, )
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
You can also pass extra keyword arguments to `tqdm` through `pbar_kwargs`:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], pbar_kwargs={"leave": False}, )
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Verbose logging can be enabled with:
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
result = parameterrun(my_function, param_names="x", param_values=[1, 2, 3], verbose=True, )
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The progress-bar description is generated automatically from the function name and parameter names, unless you provide
|
|
209
|
+
`desc` explicitly.
|
|
210
|
+
|
|
211
|
+
## API reference
|
|
212
|
+
|
|
213
|
+
`parameterrun`
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
parameterrun(fun, param_names, param_values, n_workers=-1, pbar_bool=True, verbose=False, pbar_kwargs=None,
|
|
217
|
+
reshape=True, backend=None, desc=None, **kwargs, )
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
* `fun`: function to evaluate
|
|
221
|
+
* `param_names`: parameter name or grouped parameter names
|
|
222
|
+
* `param_values`: values corresponding to `param_names`
|
|
223
|
+
* `n_workers`: number of workers for `joblib`
|
|
224
|
+
* `pbar_bool`: enable or disable progress bars
|
|
225
|
+
* `verbose`: print backend and timing information
|
|
226
|
+
* `pbar_kwargs`: additional keyword arguments passed to tqdm
|
|
227
|
+
* `reshape`: reshape results into arrays when possible
|
|
228
|
+
* `backend`: `"joblib"`, `"mpi"`, or `None`
|
|
229
|
+
* `desc`: custom progress-bar label
|
|
230
|
+
* `**kwargs`: extra fixed keyword arguments forwarded to `fun`
|
|
231
|
+
|
|
232
|
+
## Input formats
|
|
233
|
+
|
|
234
|
+
`parameterrun` accepts three input styles.
|
|
235
|
+
|
|
236
|
+
### A. Single parameter
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
param_names = "x"
|
|
240
|
+
param_values = [1, 2, 3]
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### B. Multiple independent parameters
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
param_names = ["x", "y"]
|
|
247
|
+
param_values = [[1, 2, 3], [10, 20], ]
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### C. Multiple groups of parameters
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
param_names = [["x", "y"], ["z"]]
|
|
254
|
+
param_values = [[[1, 2], [10, 20]], [[100, 200, 300]], ]
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## Development checks
|
|
258
|
+
|
|
259
|
+
Before opening a PR or publishing a release, run:
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
pytest
|
|
263
|
+
ruff check .
|
|
264
|
+
mypy src/parameterrun
|
|
265
|
+
python -m build
|
|
266
|
+
python -m twine check dist/*
|
|
267
|
+
```
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
parameterrun/__init__.py,sha256=ihShiMZY0S5PMNQP3OKDd1_3SwIRYGVb6xO96uNZ7f8,69
|
|
2
|
+
parameterrun/parallel_utils.py,sha256=AztK0J5dMF-9uWM55aWWWY_U2S8ztQ-45poCf8Y0sOs,15446
|
|
3
|
+
parameterrun-0.1.0.dist-info/METADATA,sha256=AfCEQ7uCgIp4O7-6BcAqu_gkj7BciC5ULw9AuSAGNO0,7356
|
|
4
|
+
parameterrun-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
5
|
+
parameterrun-0.1.0.dist-info/licenses/LICENSE,sha256=r8q7OrtqDHayzNYdmNa6uMpzmCQCNUQ9DoNekm9vQec,1512
|
|
6
|
+
parameterrun-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, David Fernandez Fernandez
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|