holobench 1.3.5__py3-none-any.whl → 1.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bencher/__init__.py +12 -0
- bencher/bench_cfg.py +29 -33
- bencher/bench_plot_server.py +5 -1
- bencher/bench_report.py +14 -14
- bencher/bench_runner.py +3 -2
- bencher/bencher.py +80 -53
- bencher/class_enum.py +52 -0
- bencher/job.py +6 -4
- bencher/optuna_conversions.py +4 -3
- bencher/utils.py +55 -4
- bencher/video_writer.py +48 -11
- holobench-1.23.0.data/data/share/bencher/package.xml +33 -0
- holobench-1.23.0.dist-info/LICENSE +21 -0
- {holobench-1.3.5.dist-info → holobench-1.23.0.dist-info}/METADATA +41 -31
- holobench-1.23.0.dist-info/RECORD +20 -0
- {holobench-1.3.5.dist-info → holobench-1.23.0.dist-info}/WHEEL +2 -1
- holobench-1.23.0.dist-info/top_level.txt +1 -0
- bencher/example/benchmark_data.py +0 -200
- bencher/example/example_all.py +0 -45
- bencher/example/example_categorical.py +0 -99
- bencher/example/example_custom_sweep.py +0 -59
- bencher/example/example_docs.py +0 -34
- bencher/example/example_float3D.py +0 -101
- bencher/example/example_float_cat.py +0 -98
- bencher/example/example_floats.py +0 -89
- bencher/example/example_floats2D.py +0 -93
- bencher/example/example_holosweep.py +0 -104
- bencher/example/example_holosweep_objects.py +0 -111
- bencher/example/example_holosweep_tap.py +0 -144
- bencher/example/example_image.py +0 -82
- bencher/example/example_levels.py +0 -181
- bencher/example/example_pareto.py +0 -53
- bencher/example/example_sample_cache.py +0 -85
- bencher/example/example_sample_cache_context.py +0 -116
- bencher/example/example_simple.py +0 -134
- bencher/example/example_simple_bool.py +0 -34
- bencher/example/example_simple_cat.py +0 -47
- bencher/example/example_simple_float.py +0 -38
- bencher/example/example_strings.py +0 -46
- bencher/example/example_time_event.py +0 -62
- bencher/example/example_video.py +0 -124
- bencher/example/example_workflow.py +0 -189
- bencher/example/experimental/example_bokeh_plotly.py +0 -38
- bencher/example/experimental/example_hover_ex.py +0 -45
- bencher/example/experimental/example_hvplot_explorer.py +0 -39
- bencher/example/experimental/example_interactive.py +0 -75
- bencher/example/experimental/example_streamnd.py +0 -49
- bencher/example/experimental/example_streams.py +0 -36
- bencher/example/experimental/example_template.py +0 -40
- bencher/example/experimental/example_updates.py +0 -84
- bencher/example/experimental/example_vector.py +0 -84
- bencher/example/meta/example_meta.py +0 -171
- bencher/example/meta/example_meta_cat.py +0 -25
- bencher/example/meta/example_meta_float.py +0 -23
- bencher/example/meta/example_meta_levels.py +0 -26
- bencher/example/optuna/example_optuna.py +0 -78
- bencher/example/shelved/example_float2D_scatter.py +0 -109
- bencher/example/shelved/example_float3D_cone.py +0 -96
- bencher/example/shelved/example_kwargs.py +0 -63
- bencher/plotting/__init__.py +0 -0
- bencher/plotting/plot_filter.py +0 -110
- bencher/plotting/plt_cnt_cfg.py +0 -74
- bencher/results/__init__.py +0 -0
- bencher/results/bench_result.py +0 -80
- bencher/results/bench_result_base.py +0 -405
- bencher/results/float_formatter.py +0 -44
- bencher/results/holoview_result.py +0 -592
- bencher/results/optuna_result.py +0 -354
- bencher/results/panel_result.py +0 -113
- bencher/results/plotly_result.py +0 -65
- bencher/variables/inputs.py +0 -193
- bencher/variables/parametrised_sweep.py +0 -206
- bencher/variables/results.py +0 -176
- bencher/variables/sweep_base.py +0 -167
- bencher/variables/time.py +0 -74
- holobench-1.3.5.dist-info/RECORD +0 -74
- /bencher/example/__init__.py → /holobench-1.23.0.data/data/share/ament_index/resource_index/packages/bencher +0 -0
bencher/utils.py
CHANGED
@@ -8,7 +8,9 @@ from colorsys import hsv_to_rgb
|
|
8
8
|
from pathlib import Path
|
9
9
|
from uuid import uuid4
|
10
10
|
from functools import partial
|
11
|
-
from typing import Callable, Any
|
11
|
+
from typing import Callable, Any, List, Tuple
|
12
|
+
import param
|
13
|
+
import numpy as np
|
12
14
|
|
13
15
|
|
14
16
|
def hmap_canonical_input(dic: dict) -> tuple:
|
@@ -60,6 +62,9 @@ def get_nearest_coords(dataset: xr.Dataset, collapse_list=False, **kwargs) -> di
|
|
60
62
|
def get_nearest_coords1D(val: Any, coords) -> Any:
|
61
63
|
if isinstance(val, (int, float)):
|
62
64
|
return min(coords, key=lambda x_: abs(x_ - val))
|
65
|
+
for i in coords:
|
66
|
+
if val == i:
|
67
|
+
return i
|
63
68
|
return val
|
64
69
|
|
65
70
|
|
@@ -94,6 +99,23 @@ def un_camel(camel: str) -> str:
|
|
94
99
|
return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
|
95
100
|
|
96
101
|
|
102
|
+
def mult_tuple(inp: Tuple[float], val: float) -> Tuple[float]:
|
103
|
+
return tuple(np.array(inp) * val)
|
104
|
+
|
105
|
+
|
106
|
+
def tabs_in_markdown(regular_str: str, spaces: int = 2) -> str:
|
107
|
+
"""Given a string with tabs in the form \t convert the to   which is a double space in markdown
|
108
|
+
|
109
|
+
Args:
|
110
|
+
regular_str (str): A string with tabs in it
|
111
|
+
spaces (int): the number of spaces per tab
|
112
|
+
|
113
|
+
Returns:
|
114
|
+
str: A string with sets of to represent the tabs in markdown
|
115
|
+
"""
|
116
|
+
return regular_str.replace("\t", "".join([" "] * spaces))
|
117
|
+
|
118
|
+
|
97
119
|
def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
|
98
120
|
"""Uses the golden angle to generate colors programmatically with minimum overlap between colors.
|
99
121
|
https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
|
@@ -125,17 +147,25 @@ def color_tuple_to_css(color: tuple[float, float, float]) -> str:
|
|
125
147
|
return f"rgb{(color[0] * 255, color[1] * 255, color[2] * 255)}"
|
126
148
|
|
127
149
|
|
128
|
-
def
|
150
|
+
def color_tuple_to_255(color: tuple[float, float, float]) -> tuple[float, float, float]:
|
151
|
+
return (
|
152
|
+
min(int(color[0] * 255), 255),
|
153
|
+
min(int(color[1] * 255), 255),
|
154
|
+
min(int(color[2] * 255), 255),
|
155
|
+
)
|
156
|
+
|
157
|
+
|
158
|
+
def gen_path(filename, folder="generic", suffix=".dat"):
|
129
159
|
path = Path(f"cachedir/{folder}/{filename}/")
|
130
160
|
path.mkdir(parents=True, exist_ok=True)
|
131
161
|
return f"{path.absolute().as_posix()}/{filename}_{uuid4()}{suffix}"
|
132
162
|
|
133
163
|
|
134
|
-
def gen_video_path(video_name: str, extension: str = ".
|
164
|
+
def gen_video_path(video_name: str = "vid", extension: str = ".mp4") -> str:
|
135
165
|
return gen_path(video_name, "vid", extension)
|
136
166
|
|
137
167
|
|
138
|
-
def gen_image_path(image_name: str, filetype=".png") -> str:
|
168
|
+
def gen_image_path(image_name: str = "img", filetype=".png") -> str:
|
139
169
|
return gen_path(image_name, "img", filetype)
|
140
170
|
|
141
171
|
|
@@ -146,3 +176,24 @@ def callable_name(any_callable: Callable[..., Any]) -> str:
|
|
146
176
|
return any_callable.__name__
|
147
177
|
except AttributeError:
|
148
178
|
return str(any_callable)
|
179
|
+
|
180
|
+
|
181
|
+
def listify(obj) -> list:
|
182
|
+
"""Take an object and turn it into a list if its not already a list. However if the object is none, don't turn it into a list"""
|
183
|
+
if obj is None:
|
184
|
+
return None
|
185
|
+
if isinstance(obj, list):
|
186
|
+
return obj
|
187
|
+
if isinstance(obj, tuple):
|
188
|
+
return list(obj)
|
189
|
+
return [obj]
|
190
|
+
|
191
|
+
|
192
|
+
def get_name(var):
|
193
|
+
if isinstance(var, param.Parameter):
|
194
|
+
return var.name
|
195
|
+
return var
|
196
|
+
|
197
|
+
|
198
|
+
def params_to_str(param_list: List[param.Parameter]):
|
199
|
+
return [get_name(i) for i in param_list]
|
bencher/video_writer.py
CHANGED
@@ -1,30 +1,67 @@
|
|
1
|
-
from bencher import gen_video_path, gen_image_path
|
2
|
-
from PIL import Image
|
3
1
|
import numpy as np
|
2
|
+
import moviepy.video.io.ImageSequenceClip
|
3
|
+
from pathlib import Path
|
4
|
+
from .utils import gen_video_path, gen_image_path
|
5
|
+
|
6
|
+
import moviepy
|
7
|
+
from PIL import Image, ImageDraw
|
4
8
|
|
5
9
|
|
6
10
|
class VideoWriter:
|
7
11
|
def __init__(self, filename: str = "vid") -> None:
|
8
12
|
self.images = []
|
13
|
+
self.image_files = []
|
14
|
+
self.video_files = []
|
9
15
|
self.filename = gen_video_path(filename)
|
10
16
|
|
11
17
|
def append(self, img):
|
12
18
|
self.images.append(img)
|
13
19
|
|
14
|
-
def write(self
|
15
|
-
|
20
|
+
def write(self) -> str:
|
21
|
+
if len(self.images) > 0:
|
22
|
+
clip = moviepy.video.io.ImageSequenceClip.ImageSequenceClip(
|
23
|
+
self.images, fps=30, with_mask=False, load_images=True
|
24
|
+
)
|
25
|
+
self.write_video_raw(clip)
|
26
|
+
return self.filename
|
27
|
+
|
28
|
+
@staticmethod
|
29
|
+
def create_label(label, width=None, height=16, color=(255, 255, 255)):
|
30
|
+
if width is None:
|
31
|
+
width = len(label) * 10
|
32
|
+
new_img = Image.new("RGB", (width, height), color=color)
|
33
|
+
# ImageDraw.Draw(new_img).text((width/2, 0), label, (0, 0, 0),align="center",achor="ms")
|
34
|
+
ImageDraw.Draw(new_img).text(
|
35
|
+
(width / 2.0, 0), label, (0, 0, 0), anchor="mt", font_size=height
|
36
|
+
)
|
37
|
+
|
38
|
+
return new_img
|
16
39
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
40
|
+
@staticmethod
|
41
|
+
def label_image(path: Path, label, padding=20, color=(255, 255, 255)) -> Path:
|
42
|
+
image = Image.open(path)
|
43
|
+
new_img = VideoWriter.create_label(
|
44
|
+
label, image.size[0], image.size[1] + padding, color=color
|
45
|
+
)
|
46
|
+
new_img.paste(image, (0, padding))
|
47
|
+
return new_img
|
21
48
|
|
22
|
-
|
23
|
-
|
49
|
+
def write_video_raw(self, video_clip: moviepy.video.VideoClip, fps: int = 30) -> str:
|
50
|
+
video_clip.write_videofile(
|
51
|
+
self.filename,
|
52
|
+
codec="libx264",
|
53
|
+
audio=False,
|
54
|
+
bitrate="0",
|
55
|
+
fps=fps,
|
56
|
+
ffmpeg_params=["-crf", "23"],
|
57
|
+
threads=8,
|
58
|
+
)
|
59
|
+
video_clip.close()
|
24
60
|
return self.filename
|
25
61
|
|
26
62
|
|
27
|
-
def add_image(np_array: np.ndarray, name: str = "img"):
|
63
|
+
def add_image(np_array: np.ndarray, name: str = "img") -> str:
|
64
|
+
"""Creates a file on disk from a numpy array and returns the created image path"""
|
28
65
|
filename = gen_image_path(name)
|
29
66
|
Image.fromarray(np_array).save(filename)
|
30
67
|
return filename
|
@@ -0,0 +1,33 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<package format="3">
|
3
|
+
<name>bencher</name>
|
4
|
+
<version>0.1.0</version>
|
5
|
+
<description>A package for benchmarking the performance of arbitrary functions</description>
|
6
|
+
<maintainer email="austin.gregg-smith@dyson.com">Austin Gregg-Smith</maintainer>
|
7
|
+
<license>MIT</license>
|
8
|
+
|
9
|
+
<depend>python3-diskcache</depend>
|
10
|
+
|
11
|
+
<depend>python3-pandas</depend>
|
12
|
+
<depend>python3-seaborn</depend>
|
13
|
+
<depend>python3-matplotlib</depend>
|
14
|
+
<depend>python3-numpy</depend>
|
15
|
+
<depend>python3-pytest</depend>
|
16
|
+
<depend>python3-hypothesis</depend>
|
17
|
+
<depend>xarray</depend>
|
18
|
+
<depend>python3-zarr</depend>
|
19
|
+
<depend>python3-param</depend>
|
20
|
+
<depend>python3-panel</depend>
|
21
|
+
<depend>python3-hvplot</depend>
|
22
|
+
<depend>python3-optuna</depend>
|
23
|
+
<depend>python3-plotly</depend>
|
24
|
+
|
25
|
+
<!-- TO REMOVE WHEN WE UPGRADE TO PYTHON 3.11, This is in the standard library for python>=3.11 -->
|
26
|
+
<depend>strenum</depend>
|
27
|
+
|
28
|
+
<test_depend>python3-pytest-cov</test_depend>
|
29
|
+
|
30
|
+
<export>
|
31
|
+
<build_type>ament_python</build_type>
|
32
|
+
</export>
|
33
|
+
</package>
|
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2023 Dyson AI
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
@@ -1,37 +1,40 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: holobench
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.23.0
|
4
4
|
Summary: A package for benchmarking the performance of arbitrary functions
|
5
5
|
Author-email: Austin Gregg-Smith <blooop@gmail.com>
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
Requires-Dist: numpy>=1.0,<=1.26.2
|
10
|
-
Requires-Dist: param>=1.13.0,<=2.0.1
|
11
|
-
Requires-Dist: hvplot>=0.8,<=0.9.1
|
12
|
-
Requires-Dist: matplotlib>=3.6.3,<=3.8.2
|
13
|
-
Requires-Dist: panel>=1.3.6,<=1.3.6
|
14
|
-
Requires-Dist: diskcache>=5.6,<=5.6.3
|
15
|
-
Requires-Dist: optuna>=3.2,<=3.5.0
|
16
|
-
Requires-Dist: xarray>=2023.7,<=2023.12.0
|
17
|
-
Requires-Dist: plotly>=5.15,<=5.18.0
|
18
|
-
Requires-Dist: sortedcontainers>=2.4,<=2.4
|
19
|
-
Requires-Dist: pandas>=2.0,<=2.1.4
|
20
|
-
Requires-Dist: strenum>=0.4.0,<=0.4.15
|
21
|
-
Requires-Dist: scikit-learn>=1.2,<=1.3.2
|
22
|
-
Requires-Dist: str2bool>=1.1,<=1.1
|
23
|
-
Requires-Dist: scoop>=0.7.0,<=0.7.2.0
|
24
|
-
Requires-Dist: moviepy>=1.0.3,<=1.0.3
|
25
|
-
Requires-Dist: black>=23,<=23.12.1 ; extra == "test"
|
26
|
-
Requires-Dist: pylint>=2.16,<=3.0.3 ; extra == "test"
|
27
|
-
Requires-Dist: pytest-cov>=4.1,<=4.1 ; extra == "test"
|
28
|
-
Requires-Dist: pytest>=7.4,<=7.4.4 ; extra == "test"
|
29
|
-
Requires-Dist: hypothesis>=6.82,<=6.92.2 ; extra == "test"
|
30
|
-
Requires-Dist: ruff>=0.0.280,<=0.1.9 ; extra == "test"
|
31
|
-
Requires-Dist: coverage>=7.2.7,<=7.4.0 ; extra == "test"
|
6
|
+
Maintainer: austin.gregg-smith
|
7
|
+
Maintainer-email: austin.gregg-smith@dyson.com
|
8
|
+
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
32
9
|
Project-URL: Home, https://github.com/dyson-ai/bencher
|
33
|
-
Project-URL:
|
10
|
+
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
11
|
+
Description-Content-Type: text/markdown
|
12
|
+
License-File: LICENSE
|
13
|
+
Requires-Dist: holoviews <=1.18.3,>=1.15
|
14
|
+
Requires-Dist: numpy <=1.26.4,>=1.0
|
15
|
+
Requires-Dist: param <=2.1.0,>=1.13.0
|
16
|
+
Requires-Dist: hvplot <=0.10.0,>=0.8
|
17
|
+
Requires-Dist: matplotlib <=3.9.0,>=3.6.3
|
18
|
+
Requires-Dist: panel <=1.4.4,>=1.3.6
|
19
|
+
Requires-Dist: diskcache <=5.6.3,>=5.6
|
20
|
+
Requires-Dist: optuna <=3.6.1,>=3.2
|
21
|
+
Requires-Dist: xarray <=2024.5.0,>=2023.7
|
22
|
+
Requires-Dist: plotly <=5.22.0,>=5.15
|
23
|
+
Requires-Dist: sortedcontainers <=2.4,>=2.4
|
24
|
+
Requires-Dist: pandas <=2.2.2,>=2.0
|
25
|
+
Requires-Dist: strenum <=0.4.15,>=0.4.0
|
26
|
+
Requires-Dist: scikit-learn <=1.5.0,>=1.2
|
27
|
+
Requires-Dist: str2bool <=1.1,>=1.1
|
28
|
+
Requires-Dist: scoop <=0.7.2.0,>=0.7.0
|
29
|
+
Requires-Dist: moviepy <=1.0.3,>=1.0.3
|
34
30
|
Provides-Extra: test
|
31
|
+
Requires-Dist: black <=24.4.2,>=23 ; extra == 'test'
|
32
|
+
Requires-Dist: pylint <=3.2.2,>=2.17.7 ; extra == 'test'
|
33
|
+
Requires-Dist: pytest-cov <=5.0.0,>=4.1 ; extra == 'test'
|
34
|
+
Requires-Dist: pytest <=8.2.1,>=7.4 ; extra == 'test'
|
35
|
+
Requires-Dist: hypothesis <=6.103.0,>=6.82 ; extra == 'test'
|
36
|
+
Requires-Dist: ruff <=0.4.7,>=0.3 ; extra == 'test'
|
37
|
+
Requires-Dist: coverage <=7.5.3,>=7.2.7 ; extra == 'test'
|
35
38
|
|
36
39
|
# Bencher
|
37
40
|
|
@@ -45,14 +48,21 @@ Provides-Extra: test
|
|
45
48
|
[](https://pypi.org/project/holobench/)
|
46
49
|
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
47
50
|
[](https://opensource.org/license/mit/)
|
48
|
-
[](https://www.python.org/downloads/release/python-310/)
|
52
|
+
[](https://www.python.org/downloads/release/python-311/)
|
53
|
+
[](https://pixi.sh)
|
54
|
+
|
55
|
+
## Badges
|
56
|
+
|
57
|
+
### Pixi Badge
|
58
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
49
59
|
|
50
60
|
|
51
61
|
## Intro
|
52
62
|
|
53
|
-
Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics.
|
63
|
+
Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics. It calculates the [cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) of a set of variables
|
54
64
|
|
55
|
-
Parameters for bencher are defined using the param
|
65
|
+
Parameters for bencher are defined using the [param](https://param.holoviz.org/) library as a config class with extra metadata that describes the bounds of the search space you want to measure. You must define a benchmarking function that accepts an instance of the config class and return a dictionary with string metric names and float values.
|
56
66
|
|
57
67
|
Parameters are benchmarked by passing in a list N parameters, and an N-Dimensional tensor is returned. You can optionally sample each point multiple times to get back a distribution and also track its value over time. By default the data will be plotted automatically based on the types of parameters you are sampling (e.g, continous, discrete), but you can also pass in a callback to customize plotting.
|
58
68
|
|
@@ -0,0 +1,20 @@
|
|
1
|
+
bencher/__init__.py,sha256=XCrJ5Nbw3k1J6j8qVEf6kH5sMmECIXzII-t9SdB8gEI,1558
|
2
|
+
bencher/bench_cfg.py,sha256=8rvJyeQXalZmYF8Lb-NKb9RFJs0w08k9ogcZSR1rhgs,18413
|
3
|
+
bencher/bench_plot_server.py,sha256=D00_SOrHa2IT8zAjwetoNL6tEiHSHvXnbea9iElCLVk,4195
|
4
|
+
bencher/bench_report.py,sha256=jh3T_q9KByZDeMPMf0KNJojZukxRzkfaYGeuWQU8MKM,10528
|
5
|
+
bencher/bench_runner.py,sha256=-SzAKd6QbPJ05KaW3vteFIkE-UtlFS55Ob9QeE5eRXw,6202
|
6
|
+
bencher/bencher.py,sha256=SlSje8ZvWdRnmRs4ZLXGx3bDQRMHkDOgvyBIsLClUTE,33400
|
7
|
+
bencher/caching.py,sha256=AusaNrzGGlj5m6zcwcqnTn55Mam2mQdF--oqelO806M,1627
|
8
|
+
bencher/class_enum.py,sha256=kYHW9qKkKcNdwaXizZL-fTptS_DUEGv4c88yCehk3gc,1492
|
9
|
+
bencher/job.py,sha256=swa0VwrZf41v7qNjreVDIYUU6r_dfuLipPZbg_w5x7c,6089
|
10
|
+
bencher/optuna_conversions.py,sha256=an-LfPsQXyyvhIZnG8Wl1RQVYMvJj7WOi3YNqoUnuxQ,5356
|
11
|
+
bencher/utils.py,sha256=9KAThtIG8jNd0nd4Wft8miNM_yHWmZUkIBfJh19pzgI,6480
|
12
|
+
bencher/video_writer.py,sha256=B-V1tALd3oPDytaAsl8I6qUztDQlFbkp9gSYco-ah44,2175
|
13
|
+
bencher/worker_job.py,sha256=FREi0yWQACFmH86R1j-LH72tALEFkKhLDmmoGQY9Jh4,1571
|
14
|
+
holobench-1.23.0.data/data/share/ament_index/resource_index/packages/bencher,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
|
+
holobench-1.23.0.data/data/share/bencher/package.xml,sha256=HxWM9qIEiLbE60tG0aKsS7q3UaSKDyCMD4-1nYw8vOs,1045
|
16
|
+
holobench-1.23.0.dist-info/LICENSE,sha256=dSHXTdRY4Y7qGFMv63UksV700iff7iE-p7GGs6Sbnvo,1065
|
17
|
+
holobench-1.23.0.dist-info/METADATA,sha256=UrC2MLXikRWBDwgLOqPo0OFgM6079n6ciWTnkktDZX4,5691
|
18
|
+
holobench-1.23.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
19
|
+
holobench-1.23.0.dist-info/top_level.txt,sha256=rkP5-F_W08mOD-25ZPkt0HJsHxedb2EiRcRA7IP6Ceg,8
|
20
|
+
holobench-1.23.0.dist-info/RECORD,,
|
@@ -0,0 +1 @@
|
|
1
|
+
bencher
|
@@ -1,200 +0,0 @@
|
|
1
|
-
"""This file contains an example of how to define benchmarking parameters sweeps. Categorical values are defined as enums and passed to EnumSweep classes, other types of sweeps are defined by their respective classes.
|
2
|
-
|
3
|
-
You can define a subclass which contains an input configuration which can be passed to a function in a type safe way. You can combine the subclass with a higher level class which contains more configuation parameters. This is to help manage the complexity of large configuration/parameter spaces.
|
4
|
-
"""
|
5
|
-
|
6
|
-
import math
|
7
|
-
import random
|
8
|
-
from enum import auto
|
9
|
-
|
10
|
-
from strenum import StrEnum
|
11
|
-
|
12
|
-
|
13
|
-
from bencher.variables.inputs import IntSweep, FloatSweep, StringSweep, EnumSweep, BoolSweep
|
14
|
-
from bencher.variables.results import ResultVar, OptDir
|
15
|
-
|
16
|
-
from bencher.variables.parametrised_sweep import ParametrizedSweep
|
17
|
-
|
18
|
-
|
19
|
-
class PostprocessFn(StrEnum):
|
20
|
-
"""Apply a postprocessing step to the data"""
|
21
|
-
|
22
|
-
absolute = auto() # return the abs of the output data
|
23
|
-
negate = auto() # return the negative of the output data
|
24
|
-
|
25
|
-
|
26
|
-
class NoiseDistribution(StrEnum):
|
27
|
-
"""A categorical variable describing the types of random noise"""
|
28
|
-
|
29
|
-
uniform = auto() # uniform random noiase
|
30
|
-
gaussian = auto() # gaussian noise
|
31
|
-
lognorm = auto() # lognorm noise
|
32
|
-
|
33
|
-
|
34
|
-
class NoiseCfg(ParametrizedSweep):
|
35
|
-
"""A class for storing the parameters for generating various types of noise"""
|
36
|
-
|
37
|
-
noisy = BoolSweep(
|
38
|
-
default=False, doc="Optionally add random noise to the output of the function"
|
39
|
-
)
|
40
|
-
|
41
|
-
noise_distribution = EnumSweep(NoiseDistribution, doc=NoiseDistribution.__doc__)
|
42
|
-
|
43
|
-
sigma = FloatSweep(
|
44
|
-
default=1,
|
45
|
-
bounds=[0, 10],
|
46
|
-
doc="The standard deviation of the noise",
|
47
|
-
units="v",
|
48
|
-
)
|
49
|
-
|
50
|
-
|
51
|
-
def calculate_noise(config: NoiseCfg) -> float:
|
52
|
-
"""Generate a float value based on a noise distribution and scale
|
53
|
-
|
54
|
-
Args:
|
55
|
-
config (NoiseCfg): see NoiseCfg type
|
56
|
-
|
57
|
-
Returns:
|
58
|
-
float: a noisy float value
|
59
|
-
"""
|
60
|
-
|
61
|
-
noise = 0.0
|
62
|
-
if config.noisy:
|
63
|
-
match config.noise_distribution:
|
64
|
-
case NoiseDistribution.uniform:
|
65
|
-
noise = random.uniform(0, config.sigma)
|
66
|
-
case NoiseDistribution.gaussian:
|
67
|
-
noise = random.gauss(0, config.sigma)
|
68
|
-
case NoiseDistribution.lognorm:
|
69
|
-
noise = random.lognormvariate(0, config.sigma)
|
70
|
-
|
71
|
-
return noise
|
72
|
-
|
73
|
-
|
74
|
-
class ExampleBenchCfgIn(NoiseCfg):
|
75
|
-
"""A class for representing a set of configuration options for the example worker. This class inherits from NoiseCfg so it also stores all its values as well."""
|
76
|
-
|
77
|
-
theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
|
78
|
-
offset = FloatSweep(default=0, bounds=[0, 0.3], doc="dc offset", units="v", samples=30)
|
79
|
-
postprocess_fn = EnumSweep(PostprocessFn)
|
80
|
-
|
81
|
-
|
82
|
-
class ExampleBenchCfgOut(ParametrizedSweep):
|
83
|
-
out_sin = ResultVar(units="v", direction=OptDir.minimize, doc="sin of theta with some noise")
|
84
|
-
out_cos = ResultVar(units="v", direction=OptDir.minimize, doc="cos of theta with some noise")
|
85
|
-
out_bool = ResultVar(units="%", doc="sin > 0.5")
|
86
|
-
|
87
|
-
|
88
|
-
def negate_fn(fn_input: float):
|
89
|
-
"""returns the negative of the input
|
90
|
-
|
91
|
-
Args:
|
92
|
-
fn_input (float): any float value
|
93
|
-
|
94
|
-
Returns:
|
95
|
-
float: negative of the input
|
96
|
-
"""
|
97
|
-
return -fn_input
|
98
|
-
|
99
|
-
|
100
|
-
def bench_function(cfg: ExampleBenchCfgIn) -> ExampleBenchCfgOut:
|
101
|
-
"""Takes an ExampleBenchCfgIn and returns a ExampleBenchCfgOut output"""
|
102
|
-
out = ExampleBenchCfgOut()
|
103
|
-
noise = calculate_noise(cfg)
|
104
|
-
|
105
|
-
postprocess_fn = abs if cfg.postprocess_fn == PostprocessFn.absolute else negate_fn
|
106
|
-
|
107
|
-
out.out_sin = postprocess_fn(cfg.offset + math.sin(cfg.theta) + noise)
|
108
|
-
out.out_cos = postprocess_fn(cfg.offset + math.cos(cfg.theta) + noise)
|
109
|
-
|
110
|
-
out.out_bool = out.out_sin > 0.5
|
111
|
-
return out
|
112
|
-
|
113
|
-
|
114
|
-
class ExampleBenchCfg(ParametrizedSweep):
|
115
|
-
theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
|
116
|
-
offset = FloatSweep(default=0, bounds=[0, 0.3], doc="dc offset", units="v", samples=30)
|
117
|
-
postprocess_fn = EnumSweep(PostprocessFn)
|
118
|
-
|
119
|
-
noisy = BoolSweep(
|
120
|
-
default=False, doc="Optionally add random noise to the output of the function"
|
121
|
-
)
|
122
|
-
noise_distribution = EnumSweep(NoiseDistribution, doc=NoiseDistribution.__doc__)
|
123
|
-
sigma = FloatSweep(
|
124
|
-
default=1,
|
125
|
-
bounds=[0, 10],
|
126
|
-
doc="The standard deviation of the noise",
|
127
|
-
units="v",
|
128
|
-
)
|
129
|
-
|
130
|
-
out_sin = ResultVar(units="v", direction=OptDir.minimize, doc="sin of theta with some noise")
|
131
|
-
out_cos = ResultVar(units="v", direction=OptDir.minimize, doc="cos of theta with some noise")
|
132
|
-
out_bool = ResultVar(units="%", doc="sin > 0.5")
|
133
|
-
|
134
|
-
def __call__(self, **kwwargs) -> dict:
|
135
|
-
self.update_params_from_kwargs(**kwwargs)
|
136
|
-
|
137
|
-
noise = self.calculate_noise()
|
138
|
-
postprocess_fn = abs if self.postprocess_fn == PostprocessFn.absolute else negate_fn
|
139
|
-
|
140
|
-
self.out_sin = postprocess_fn(self.offset + math.sin(self.theta) + noise)
|
141
|
-
self.out_cos = postprocess_fn(self.offset + math.cos(self.theta) + noise)
|
142
|
-
self.out_bool = self.out_sin > 0.5
|
143
|
-
return self.get_results_values_as_dict()
|
144
|
-
|
145
|
-
def calculate_noise(self):
|
146
|
-
noise = 0.0
|
147
|
-
if self.noisy:
|
148
|
-
match self.noise_distribution:
|
149
|
-
case NoiseDistribution.uniform:
|
150
|
-
noise = random.uniform(0, self.sigma)
|
151
|
-
case NoiseDistribution.gaussian:
|
152
|
-
noise = random.gauss(0, self.sigma)
|
153
|
-
case NoiseDistribution.lognorm:
|
154
|
-
noise = random.lognormvariate(0, self.sigma)
|
155
|
-
|
156
|
-
return noise
|
157
|
-
|
158
|
-
|
159
|
-
def call(**kwargs) -> dict:
|
160
|
-
return ExampleBenchCfg().__call__(**kwargs)
|
161
|
-
|
162
|
-
|
163
|
-
class AllSweepVars(ParametrizedSweep):
|
164
|
-
"""A class containing all the sweep types, This class is used for unit testing how the configuration classes are serialised and hashed"""
|
165
|
-
|
166
|
-
var_float = FloatSweep(default=5, bounds=(0, 10), units="m/s")
|
167
|
-
var_int = IntSweep(default=3, bounds=[0, 4])
|
168
|
-
var_int_big = IntSweep(default=0, bounds=[0, 100], samples=3)
|
169
|
-
var_bool = BoolSweep()
|
170
|
-
var_string = StringSweep(["string1", "string2"])
|
171
|
-
var_enum = EnumSweep(PostprocessFn)
|
172
|
-
|
173
|
-
result = ResultVar()
|
174
|
-
|
175
|
-
def __call__(self, **kwargs) -> dict:
|
176
|
-
self.update_params_from_kwargs(**kwargs)
|
177
|
-
self.result = self.var_float + self.var_int
|
178
|
-
return self.get_results_values_as_dict()
|
179
|
-
|
180
|
-
|
181
|
-
class SimpleBenchClass(ParametrizedSweep):
|
182
|
-
var1 = IntSweep(default=0, bounds=[0, 2])
|
183
|
-
|
184
|
-
result = ResultVar()
|
185
|
-
|
186
|
-
def __call__(self, **kwargs) -> dict:
|
187
|
-
self.update_params_from_kwargs(**kwargs)
|
188
|
-
self.result = self.var1
|
189
|
-
return self.get_results_values_as_dict()
|
190
|
-
|
191
|
-
|
192
|
-
class SimpleBenchClassFloat(ParametrizedSweep):
|
193
|
-
var1 = FloatSweep(bounds=[0, 100])
|
194
|
-
|
195
|
-
result = ResultVar()
|
196
|
-
|
197
|
-
def __call__(self, **kwargs) -> dict:
|
198
|
-
self.update_params_from_kwargs(**kwargs)
|
199
|
-
self.result = self.var1
|
200
|
-
return self.get_results_values_as_dict()
|
bencher/example/example_all.py
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
import bencher as bch
|
2
|
-
|
3
|
-
from bencher.example.example_categorical import example_categorical
|
4
|
-
from bencher.example.example_floats import example_floats
|
5
|
-
from bencher.example.example_floats2D import example_floats2D
|
6
|
-
from bencher.example.example_pareto import example_pareto
|
7
|
-
from bencher.example.example_simple_cat import example_1D_cat
|
8
|
-
from bencher.example.example_simple_float import example_1D_float
|
9
|
-
from bencher.example.example_float_cat import example_float_cat
|
10
|
-
from bencher.example.example_time_event import example_time_event
|
11
|
-
from bencher.example.example_float3D import example_floats3D
|
12
|
-
|
13
|
-
from bencher.example.example_custom_sweep import example_custom_sweep
|
14
|
-
from bencher.example.example_holosweep import example_holosweep
|
15
|
-
from bencher.example.example_holosweep_tap import example_holosweep_tap
|
16
|
-
|
17
|
-
from bencher.example.optuna.example_optuna import optuna_rastrigin
|
18
|
-
from bencher.example.example_sample_cache import example_sample_cache
|
19
|
-
|
20
|
-
# from bencher.example.example_workflow import example_floats2D_workflow, example_floats3D_workflow
|
21
|
-
|
22
|
-
|
23
|
-
if __name__ == "__main__":
|
24
|
-
run_cfg = bch.BenchRunCfg()
|
25
|
-
run_cfg.overwrite_sample_cache = True
|
26
|
-
bench_runner = bch.BenchRunner("bencher_examples", run_cfg=run_cfg)
|
27
|
-
|
28
|
-
bench_runner.add_run(example_categorical)
|
29
|
-
bench_runner.add_run(example_floats)
|
30
|
-
bench_runner.add_run(example_floats2D)
|
31
|
-
bench_runner.add_run(example_floats3D)
|
32
|
-
bench_runner.add_run(example_1D_cat)
|
33
|
-
bench_runner.add_run(example_1D_float)
|
34
|
-
bench_runner.add_run(example_pareto)
|
35
|
-
bench_runner.add_run(example_float_cat)
|
36
|
-
bench_runner.add_run(example_time_event)
|
37
|
-
bench_runner.add_run(example_custom_sweep)
|
38
|
-
bench_runner.add_run(example_holosweep)
|
39
|
-
bench_runner.add_run(example_holosweep_tap)
|
40
|
-
bench_runner.add_run(optuna_rastrigin)
|
41
|
-
bench_runner.add_run(example_sample_cache)
|
42
|
-
|
43
|
-
# bench_runner.run(level=2, show=True, grouped=True)
|
44
|
-
|
45
|
-
bench_runner.run(level=4, show=True, grouped=True, save=False)
|