holobench 1.19.0__tar.gz → 1.23.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- holobench-1.23.0/LICENSE +21 -0
- {holobench-1.19.0 → holobench-1.23.0}/PKG-INFO +39 -29
- {holobench-1.19.0 → holobench-1.23.0}/README.md +9 -2
- {holobench-1.19.0 → holobench-1.23.0}/bencher/__init__.py +10 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_runner.py +1 -1
- {holobench-1.19.0 → holobench-1.23.0}/bencher/bencher.py +3 -1
- {holobench-1.19.0 → holobench-1.23.0}/bencher/optuna_conversions.py +3 -2
- {holobench-1.19.0 → holobench-1.23.0}/bencher/utils.py +28 -2
- holobench-1.23.0/bencher/video_writer.py +67 -0
- holobench-1.23.0/holobench.egg-info/PKG-INFO +95 -0
- holobench-1.23.0/holobench.egg-info/SOURCES.txt +47 -0
- holobench-1.23.0/holobench.egg-info/dependency_links.txt +1 -0
- holobench-1.23.0/holobench.egg-info/not-zip-safe +1 -0
- holobench-1.23.0/holobench.egg-info/requires.txt +26 -0
- holobench-1.23.0/holobench.egg-info/top_level.txt +1 -0
- holobench-1.23.0/package.xml +33 -0
- holobench-1.23.0/pyproject.toml +142 -0
- holobench-1.23.0/setup.cfg +10 -0
- holobench-1.23.0/setup.py +18 -0
- holobench-1.23.0/test/test_bench_examples.py +129 -0
- holobench-1.23.0/test/test_bench_report.py +55 -0
- holobench-1.23.0/test/test_bench_result_base.py +79 -0
- holobench-1.23.0/test/test_bench_runner.py +196 -0
- holobench-1.23.0/test/test_bencher.py +383 -0
- holobench-1.23.0/test/test_cache.py +46 -0
- holobench-1.23.0/test/test_class_enum.py +10 -0
- holobench-1.23.0/test/test_combinations.py +182 -0
- holobench-1.23.0/test/test_composable_container_base.py +77 -0
- holobench-1.23.0/test/test_composable_container_video.py +255 -0
- holobench-1.23.0/test/test_float_formatter.py +26 -0
- holobench-1.23.0/test/test_job.py +98 -0
- holobench-1.23.0/test/test_meta_tests.py +55 -0
- holobench-1.23.0/test/test_parametrized_sweep.py +74 -0
- holobench-1.23.0/test/test_plot_filter.py +91 -0
- holobench-1.23.0/test/test_plot_server.py +55 -0
- holobench-1.23.0/test/test_sample_cache.py +113 -0
- holobench-1.23.0/test/test_sweep_base.py +198 -0
- holobench-1.23.0/test/test_sweep_vars.py +176 -0
- holobench-1.23.0/test/test_utils.py +176 -0
- holobench-1.23.0/test/test_vars.py +69 -0
- holobench-1.19.0/bencher/example/benchmark_data.py +0 -200
- holobench-1.19.0/bencher/example/example_all.py +0 -45
- holobench-1.19.0/bencher/example/example_categorical.py +0 -99
- holobench-1.19.0/bencher/example/example_custom_sweep.py +0 -59
- holobench-1.19.0/bencher/example/example_docs.py +0 -34
- holobench-1.19.0/bencher/example/example_filepath.py +0 -27
- holobench-1.19.0/bencher/example/example_float3D.py +0 -101
- holobench-1.19.0/bencher/example/example_float_cat.py +0 -99
- holobench-1.19.0/bencher/example/example_floats.py +0 -89
- holobench-1.19.0/bencher/example/example_floats2D.py +0 -93
- holobench-1.19.0/bencher/example/example_holosweep.py +0 -98
- holobench-1.19.0/bencher/example/example_holosweep_objects.py +0 -111
- holobench-1.19.0/bencher/example/example_holosweep_tap.py +0 -144
- holobench-1.19.0/bencher/example/example_image.py +0 -138
- holobench-1.19.0/bencher/example/example_levels.py +0 -181
- holobench-1.19.0/bencher/example/example_pareto.py +0 -53
- holobench-1.19.0/bencher/example/example_sample_cache.py +0 -85
- holobench-1.19.0/bencher/example/example_sample_cache_context.py +0 -116
- holobench-1.19.0/bencher/example/example_simple.py +0 -134
- holobench-1.19.0/bencher/example/example_simple_bool.py +0 -35
- holobench-1.19.0/bencher/example/example_simple_cat.py +0 -48
- holobench-1.19.0/bencher/example/example_simple_float.py +0 -38
- holobench-1.19.0/bencher/example/example_strings.py +0 -46
- holobench-1.19.0/bencher/example/example_time_event.py +0 -63
- holobench-1.19.0/bencher/example/example_video.py +0 -127
- holobench-1.19.0/bencher/example/example_workflow.py +0 -189
- holobench-1.19.0/bencher/example/experimental/example_bokeh_plotly.py +0 -38
- holobench-1.19.0/bencher/example/experimental/example_hover_ex.py +0 -45
- holobench-1.19.0/bencher/example/experimental/example_hvplot_explorer.py +0 -39
- holobench-1.19.0/bencher/example/experimental/example_interactive.py +0 -75
- holobench-1.19.0/bencher/example/experimental/example_streamnd.py +0 -49
- holobench-1.19.0/bencher/example/experimental/example_streams.py +0 -36
- holobench-1.19.0/bencher/example/experimental/example_template.py +0 -40
- holobench-1.19.0/bencher/example/experimental/example_updates.py +0 -84
- holobench-1.19.0/bencher/example/experimental/example_vector.py +0 -84
- holobench-1.19.0/bencher/example/meta/example_meta.py +0 -171
- holobench-1.19.0/bencher/example/meta/example_meta_cat.py +0 -25
- holobench-1.19.0/bencher/example/meta/example_meta_float.py +0 -23
- holobench-1.19.0/bencher/example/meta/example_meta_levels.py +0 -26
- holobench-1.19.0/bencher/example/optuna/example_optuna.py +0 -78
- holobench-1.19.0/bencher/example/shelved/example_float2D_scatter.py +0 -109
- holobench-1.19.0/bencher/example/shelved/example_float3D_cone.py +0 -96
- holobench-1.19.0/bencher/example/shelved/example_kwargs.py +0 -63
- holobench-1.19.0/bencher/plotting/__init__.py +0 -0
- holobench-1.19.0/bencher/plotting/plot_filter.py +0 -110
- holobench-1.19.0/bencher/plotting/plt_cnt_cfg.py +0 -74
- holobench-1.19.0/bencher/results/__init__.py +0 -0
- holobench-1.19.0/bencher/results/bench_result.py +0 -94
- holobench-1.19.0/bencher/results/bench_result_base.py +0 -475
- holobench-1.19.0/bencher/results/composable_container/__init__.py +0 -0
- holobench-1.19.0/bencher/results/composable_container/composable_container_base.py +0 -60
- holobench-1.19.0/bencher/results/composable_container/composable_container_panel.py +0 -40
- holobench-1.19.0/bencher/results/composable_container/composable_container_video.py +0 -76
- holobench-1.19.0/bencher/results/float_formatter.py +0 -44
- holobench-1.19.0/bencher/results/holoview_result.py +0 -745
- holobench-1.19.0/bencher/results/optuna_result.py +0 -354
- holobench-1.19.0/bencher/results/panel_result.py +0 -41
- holobench-1.19.0/bencher/results/plotly_result.py +0 -65
- holobench-1.19.0/bencher/results/video_result.py +0 -38
- holobench-1.19.0/bencher/results/video_summary.py +0 -217
- holobench-1.19.0/bencher/variables/inputs.py +0 -179
- holobench-1.19.0/bencher/variables/parametrised_sweep.py +0 -208
- holobench-1.19.0/bencher/variables/results.py +0 -214
- holobench-1.19.0/bencher/variables/sweep_base.py +0 -162
- holobench-1.19.0/bencher/variables/time.py +0 -70
- holobench-1.19.0/bencher/video_writer.py +0 -121
- holobench-1.19.0/pyproject.toml +0 -106
- {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_cfg.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_plot_server.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_report.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/caching.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/class_enum.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/job.py +0 -0
- {holobench-1.19.0 → holobench-1.23.0}/bencher/worker_job.py +0 -0
- /holobench-1.19.0/bencher/example/__init__.py → /holobench-1.23.0/resource/bencher +0 -0
holobench-1.23.0/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2023 Dyson AI
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
@@ -1,37 +1,40 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: holobench
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.23.0
|
4
4
|
Summary: A package for benchmarking the performance of arbitrary functions
|
5
5
|
Author-email: Austin Gregg-Smith <blooop@gmail.com>
|
6
|
-
|
7
|
-
|
8
|
-
Requires-Dist: numpy>=1.0,<=1.26.4
|
9
|
-
Requires-Dist: param>=1.13.0,<=2.1.0
|
10
|
-
Requires-Dist: hvplot>=0.8,<=0.10.0
|
11
|
-
Requires-Dist: matplotlib>=3.6.3,<=3.8.4
|
12
|
-
Requires-Dist: panel>=1.3.6,<=1.4.2
|
13
|
-
Requires-Dist: diskcache>=5.6,<=5.6.3
|
14
|
-
Requires-Dist: optuna>=3.2,<=3.6.1
|
15
|
-
Requires-Dist: xarray>=2023.7,<=2024.5.0
|
16
|
-
Requires-Dist: plotly>=5.15,<=5.22.0
|
17
|
-
Requires-Dist: sortedcontainers>=2.4,<=2.4
|
18
|
-
Requires-Dist: pandas>=2.0,<=2.2.2
|
19
|
-
Requires-Dist: strenum>=0.4.0,<=0.4.15
|
20
|
-
Requires-Dist: scikit-learn>=1.2,<=1.4.2
|
21
|
-
Requires-Dist: str2bool>=1.1,<=1.1
|
22
|
-
Requires-Dist: scoop>=0.7.0,<=0.7.2.0
|
23
|
-
Requires-Dist: moviepy>=1.0.3,<=1.0.3
|
24
|
-
Requires-Dist: black>=23,<=24.4.2 ; extra == "test"
|
25
|
-
Requires-Dist: pylint>=2.16,<=3.1.0 ; extra == "test"
|
26
|
-
Requires-Dist: pytest-cov>=4.1,<=5.0.0 ; extra == "test"
|
27
|
-
Requires-Dist: pytest>=7.4,<=8.2.0 ; extra == "test"
|
28
|
-
Requires-Dist: hypothesis>=6.82,<=6.101.0 ; extra == "test"
|
29
|
-
Requires-Dist: ruff>=0.0.280,<=0.4.4 ; extra == "test"
|
30
|
-
Requires-Dist: coverage>=7.2.7,<=7.5.1 ; extra == "test"
|
31
|
-
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
32
|
-
Project-URL: Home, https://github.com/dyson-ai/bencher
|
6
|
+
Maintainer: austin.gregg-smith
|
7
|
+
Maintainer-email: austin.gregg-smith@dyson.com
|
33
8
|
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
9
|
+
Project-URL: Home, https://github.com/dyson-ai/bencher
|
10
|
+
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
11
|
+
Description-Content-Type: text/markdown
|
12
|
+
License-File: LICENSE
|
13
|
+
Requires-Dist: holoviews<=1.18.3,>=1.15
|
14
|
+
Requires-Dist: numpy<=1.26.4,>=1.0
|
15
|
+
Requires-Dist: param<=2.1.0,>=1.13.0
|
16
|
+
Requires-Dist: hvplot<=0.10.0,>=0.8
|
17
|
+
Requires-Dist: matplotlib<=3.9.0,>=3.6.3
|
18
|
+
Requires-Dist: panel<=1.4.4,>=1.3.6
|
19
|
+
Requires-Dist: diskcache<=5.6.3,>=5.6
|
20
|
+
Requires-Dist: optuna<=3.6.1,>=3.2
|
21
|
+
Requires-Dist: xarray<=2024.5.0,>=2023.7
|
22
|
+
Requires-Dist: plotly<=5.22.0,>=5.15
|
23
|
+
Requires-Dist: sortedcontainers<=2.4,>=2.4
|
24
|
+
Requires-Dist: pandas<=2.2.2,>=2.0
|
25
|
+
Requires-Dist: strenum<=0.4.15,>=0.4.0
|
26
|
+
Requires-Dist: scikit-learn<=1.5.0,>=1.2
|
27
|
+
Requires-Dist: str2bool<=1.1,>=1.1
|
28
|
+
Requires-Dist: scoop<=0.7.2.0,>=0.7.0
|
29
|
+
Requires-Dist: moviepy<=1.0.3,>=1.0.3
|
34
30
|
Provides-Extra: test
|
31
|
+
Requires-Dist: black<=24.4.2,>=23; extra == "test"
|
32
|
+
Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
|
33
|
+
Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
|
34
|
+
Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
|
35
|
+
Requires-Dist: hypothesis<=6.103.0,>=6.82; extra == "test"
|
36
|
+
Requires-Dist: ruff<=0.4.7,>=0.3; extra == "test"
|
37
|
+
Requires-Dist: coverage<=7.5.3,>=7.2.7; extra == "test"
|
35
38
|
|
36
39
|
# Bencher
|
37
40
|
|
@@ -45,7 +48,14 @@ Provides-Extra: test
|
|
45
48
|
[](https://pypi.org/project/holobench/)
|
46
49
|
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
47
50
|
[](https://opensource.org/license/mit/)
|
48
|
-
[](https://www.python.org/downloads/release/python-310/)
|
52
|
+
[](https://www.python.org/downloads/release/python-311/)
|
53
|
+
[](https://pixi.sh)
|
54
|
+
|
55
|
+
## Badges
|
56
|
+
|
57
|
+
### Pixi Badge
|
58
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
49
59
|
|
50
60
|
|
51
61
|
## Intro
|
@@ -10,7 +10,14 @@
|
|
10
10
|
[](https://pypi.org/project/holobench/)
|
11
11
|
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
12
12
|
[](https://opensource.org/license/mit/)
|
13
|
-
[](https://www.python.org/downloads/release/python-310/)
|
14
|
+
[](https://www.python.org/downloads/release/python-311/)
|
15
|
+
[](https://pixi.sh)
|
16
|
+
|
17
|
+
## Badges
|
18
|
+
|
19
|
+
### Pixi Badge
|
20
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
14
21
|
|
15
22
|
|
16
23
|
## Intro
|
@@ -47,4 +54,4 @@ Bencher is designed to work with stochastic pure functions with no side effects.
|
|
47
54
|
|
48
55
|
### Example Output
|
49
56
|
|
50
|
-
https://dyson-ai.github.io/bencher/
|
57
|
+
https://dyson-ai.github.io/bencher/
|
@@ -22,6 +22,15 @@ from .variables.results import (
|
|
22
22
|
curve,
|
23
23
|
)
|
24
24
|
|
25
|
+
from .results.composable_container.composable_container_base import (
|
26
|
+
ComposeType,
|
27
|
+
ComposableContainerBase,
|
28
|
+
)
|
29
|
+
from .results.composable_container.composable_container_video import (
|
30
|
+
ComposableContainerVideo,
|
31
|
+
RenderCfg,
|
32
|
+
)
|
33
|
+
|
25
34
|
from .plotting.plot_filter import VarRange, PlotFilter
|
26
35
|
from .utils import (
|
27
36
|
hmap_canonical_input,
|
@@ -31,6 +40,7 @@ from .utils import (
|
|
31
40
|
gen_image_path,
|
32
41
|
gen_video_path,
|
33
42
|
lerp,
|
43
|
+
tabs_in_markdown,
|
34
44
|
)
|
35
45
|
from .variables.parametrised_sweep import ParametrizedSweep
|
36
46
|
from .caching import CachedParams
|
@@ -121,7 +121,7 @@ class BenchRunner:
|
|
121
121
|
self.show_publish(report_level, show, publish, save, debug)
|
122
122
|
return self.results
|
123
123
|
|
124
|
-
def show_publish(self, report, show, publish, save, debug):
|
124
|
+
def show_publish(self, report: BenchReport, show: bool, publish: bool, save: bool, debug: bool):
|
125
125
|
if save:
|
126
126
|
report.save_index()
|
127
127
|
if publish and self.publisher is not None:
|
@@ -220,9 +220,11 @@ class Bench(BenchPlotServer):
|
|
220
220
|
relationship_cb=None,
|
221
221
|
plot_callbacks: List | bool = None,
|
222
222
|
) -> List[BenchResult]:
|
223
|
-
results = []
|
224
223
|
if relationship_cb is None:
|
225
224
|
relationship_cb = combinations
|
225
|
+
if input_vars is None:
|
226
|
+
input_vars = self.worker_class_instance.get_inputs_only()
|
227
|
+
results = []
|
226
228
|
for it in range(iterations):
|
227
229
|
for input_group in relationship_cb(input_vars, group_size):
|
228
230
|
title_gen = title + "Sweeping " + " vs ".join(params_to_str(input_group))
|
@@ -1,4 +1,5 @@
|
|
1
1
|
from typing import List
|
2
|
+
import logging
|
2
3
|
|
3
4
|
import optuna
|
4
5
|
import panel as pn
|
@@ -158,8 +159,8 @@ def summarise_optuna_study(study: optuna.study.Study) -> pn.pane.panel:
|
|
158
159
|
row.append(plot_param_importances(study))
|
159
160
|
try:
|
160
161
|
row.append(plot_pareto_front(study))
|
161
|
-
except Exception:
|
162
|
-
|
162
|
+
except Exception as e: # pylint: disable=broad-except
|
163
|
+
logging.exception(e)
|
163
164
|
|
164
165
|
row.append(
|
165
166
|
pn.pane.Markdown(f"```\nBest value: {study.best_value}\nParams: {study.best_params}```")
|
@@ -8,8 +8,9 @@ from colorsys import hsv_to_rgb
|
|
8
8
|
from pathlib import Path
|
9
9
|
from uuid import uuid4
|
10
10
|
from functools import partial
|
11
|
-
from typing import Callable, Any, List
|
11
|
+
from typing import Callable, Any, List, Tuple
|
12
12
|
import param
|
13
|
+
import numpy as np
|
13
14
|
|
14
15
|
|
15
16
|
def hmap_canonical_input(dic: dict) -> tuple:
|
@@ -98,6 +99,23 @@ def un_camel(camel: str) -> str:
|
|
98
99
|
return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
|
99
100
|
|
100
101
|
|
102
|
+
def mult_tuple(inp: Tuple[float], val: float) -> Tuple[float]:
|
103
|
+
return tuple(np.array(inp) * val)
|
104
|
+
|
105
|
+
|
106
|
+
def tabs_in_markdown(regular_str: str, spaces: int = 2) -> str:
|
107
|
+
"""Given a string with tabs in the form \t convert the to   which is a double space in markdown
|
108
|
+
|
109
|
+
Args:
|
110
|
+
regular_str (str): A string with tabs in it
|
111
|
+
spaces (int): the number of spaces per tab
|
112
|
+
|
113
|
+
Returns:
|
114
|
+
str: A string with sets of to represent the tabs in markdown
|
115
|
+
"""
|
116
|
+
return regular_str.replace("\t", "".join([" "] * spaces))
|
117
|
+
|
118
|
+
|
101
119
|
def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
|
102
120
|
"""Uses the golden angle to generate colors programmatically with minimum overlap between colors.
|
103
121
|
https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
|
@@ -129,13 +147,21 @@ def color_tuple_to_css(color: tuple[float, float, float]) -> str:
|
|
129
147
|
return f"rgb{(color[0] * 255, color[1] * 255, color[2] * 255)}"
|
130
148
|
|
131
149
|
|
150
|
+
def color_tuple_to_255(color: tuple[float, float, float]) -> tuple[float, float, float]:
|
151
|
+
return (
|
152
|
+
min(int(color[0] * 255), 255),
|
153
|
+
min(int(color[1] * 255), 255),
|
154
|
+
min(int(color[2] * 255), 255),
|
155
|
+
)
|
156
|
+
|
157
|
+
|
132
158
|
def gen_path(filename, folder="generic", suffix=".dat"):
|
133
159
|
path = Path(f"cachedir/{folder}/{filename}/")
|
134
160
|
path.mkdir(parents=True, exist_ok=True)
|
135
161
|
return f"{path.absolute().as_posix()}/{filename}_{uuid4()}{suffix}"
|
136
162
|
|
137
163
|
|
138
|
-
def gen_video_path(video_name: str = "vid", extension: str = ".
|
164
|
+
def gen_video_path(video_name: str = "vid", extension: str = ".mp4") -> str:
|
139
165
|
return gen_path(video_name, "vid", extension)
|
140
166
|
|
141
167
|
|
@@ -0,0 +1,67 @@
|
|
1
|
+
import numpy as np
|
2
|
+
import moviepy.video.io.ImageSequenceClip
|
3
|
+
from pathlib import Path
|
4
|
+
from .utils import gen_video_path, gen_image_path
|
5
|
+
|
6
|
+
import moviepy
|
7
|
+
from PIL import Image, ImageDraw
|
8
|
+
|
9
|
+
|
10
|
+
class VideoWriter:
|
11
|
+
def __init__(self, filename: str = "vid") -> None:
|
12
|
+
self.images = []
|
13
|
+
self.image_files = []
|
14
|
+
self.video_files = []
|
15
|
+
self.filename = gen_video_path(filename)
|
16
|
+
|
17
|
+
def append(self, img):
|
18
|
+
self.images.append(img)
|
19
|
+
|
20
|
+
def write(self) -> str:
|
21
|
+
if len(self.images) > 0:
|
22
|
+
clip = moviepy.video.io.ImageSequenceClip.ImageSequenceClip(
|
23
|
+
self.images, fps=30, with_mask=False, load_images=True
|
24
|
+
)
|
25
|
+
self.write_video_raw(clip)
|
26
|
+
return self.filename
|
27
|
+
|
28
|
+
@staticmethod
|
29
|
+
def create_label(label, width=None, height=16, color=(255, 255, 255)):
|
30
|
+
if width is None:
|
31
|
+
width = len(label) * 10
|
32
|
+
new_img = Image.new("RGB", (width, height), color=color)
|
33
|
+
# ImageDraw.Draw(new_img).text((width/2, 0), label, (0, 0, 0),align="center",achor="ms")
|
34
|
+
ImageDraw.Draw(new_img).text(
|
35
|
+
(width / 2.0, 0), label, (0, 0, 0), anchor="mt", font_size=height
|
36
|
+
)
|
37
|
+
|
38
|
+
return new_img
|
39
|
+
|
40
|
+
@staticmethod
|
41
|
+
def label_image(path: Path, label, padding=20, color=(255, 255, 255)) -> Path:
|
42
|
+
image = Image.open(path)
|
43
|
+
new_img = VideoWriter.create_label(
|
44
|
+
label, image.size[0], image.size[1] + padding, color=color
|
45
|
+
)
|
46
|
+
new_img.paste(image, (0, padding))
|
47
|
+
return new_img
|
48
|
+
|
49
|
+
def write_video_raw(self, video_clip: moviepy.video.VideoClip, fps: int = 30) -> str:
|
50
|
+
video_clip.write_videofile(
|
51
|
+
self.filename,
|
52
|
+
codec="libx264",
|
53
|
+
audio=False,
|
54
|
+
bitrate="0",
|
55
|
+
fps=fps,
|
56
|
+
ffmpeg_params=["-crf", "23"],
|
57
|
+
threads=8,
|
58
|
+
)
|
59
|
+
video_clip.close()
|
60
|
+
return self.filename
|
61
|
+
|
62
|
+
|
63
|
+
def add_image(np_array: np.ndarray, name: str = "img") -> str:
|
64
|
+
"""Creates a file on disk from a numpy array and returns the created image path"""
|
65
|
+
filename = gen_image_path(name)
|
66
|
+
Image.fromarray(np_array).save(filename)
|
67
|
+
return filename
|
@@ -0,0 +1,95 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: holobench
|
3
|
+
Version: 1.23.0
|
4
|
+
Summary: A package for benchmarking the performance of arbitrary functions
|
5
|
+
Author-email: Austin Gregg-Smith <blooop@gmail.com>
|
6
|
+
Maintainer: austin.gregg-smith
|
7
|
+
Maintainer-email: austin.gregg-smith@dyson.com
|
8
|
+
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
9
|
+
Project-URL: Home, https://github.com/dyson-ai/bencher
|
10
|
+
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
11
|
+
Description-Content-Type: text/markdown
|
12
|
+
License-File: LICENSE
|
13
|
+
Requires-Dist: holoviews<=1.18.3,>=1.15
|
14
|
+
Requires-Dist: numpy<=1.26.4,>=1.0
|
15
|
+
Requires-Dist: param<=2.1.0,>=1.13.0
|
16
|
+
Requires-Dist: hvplot<=0.10.0,>=0.8
|
17
|
+
Requires-Dist: matplotlib<=3.9.0,>=3.6.3
|
18
|
+
Requires-Dist: panel<=1.4.4,>=1.3.6
|
19
|
+
Requires-Dist: diskcache<=5.6.3,>=5.6
|
20
|
+
Requires-Dist: optuna<=3.6.1,>=3.2
|
21
|
+
Requires-Dist: xarray<=2024.5.0,>=2023.7
|
22
|
+
Requires-Dist: plotly<=5.22.0,>=5.15
|
23
|
+
Requires-Dist: sortedcontainers<=2.4,>=2.4
|
24
|
+
Requires-Dist: pandas<=2.2.2,>=2.0
|
25
|
+
Requires-Dist: strenum<=0.4.15,>=0.4.0
|
26
|
+
Requires-Dist: scikit-learn<=1.5.0,>=1.2
|
27
|
+
Requires-Dist: str2bool<=1.1,>=1.1
|
28
|
+
Requires-Dist: scoop<=0.7.2.0,>=0.7.0
|
29
|
+
Requires-Dist: moviepy<=1.0.3,>=1.0.3
|
30
|
+
Provides-Extra: test
|
31
|
+
Requires-Dist: black<=24.4.2,>=23; extra == "test"
|
32
|
+
Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
|
33
|
+
Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
|
34
|
+
Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
|
35
|
+
Requires-Dist: hypothesis<=6.103.0,>=6.82; extra == "test"
|
36
|
+
Requires-Dist: ruff<=0.4.7,>=0.3; extra == "test"
|
37
|
+
Requires-Dist: coverage<=7.5.3,>=7.2.7; extra == "test"
|
38
|
+
|
39
|
+
# Bencher
|
40
|
+
|
41
|
+
## Continuous Integration Status
|
42
|
+
|
43
|
+
[](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml?query=branch%3Amain)
|
44
|
+

|
45
|
+
[](https://codecov.io/gh/dyson-ai/bencher)
|
46
|
+
[](https://GitHub.com/dyson-ai/bencher/issues/)
|
47
|
+
[](https://github.com/dyson-ai/bencher/pulls?q=is%3Amerged)
|
48
|
+
[](https://pypi.org/project/holobench/)
|
49
|
+
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
50
|
+
[](https://opensource.org/license/mit/)
|
51
|
+
[](https://www.python.org/downloads/release/python-310/)
|
52
|
+
[](https://www.python.org/downloads/release/python-311/)
|
53
|
+
[](https://pixi.sh)
|
54
|
+
|
55
|
+
## Badges
|
56
|
+
|
57
|
+
### Pixi Badge
|
58
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
59
|
+
|
60
|
+
|
61
|
+
## Intro
|
62
|
+
|
63
|
+
Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics. It calculates the [cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) of a set of variables
|
64
|
+
|
65
|
+
Parameters for bencher are defined using the [param](https://param.holoviz.org/) library as a config class with extra metadata that describes the bounds of the search space you want to measure. You must define a benchmarking function that accepts an instance of the config class and return a dictionary with string metric names and float values.
|
66
|
+
|
67
|
+
Parameters are benchmarked by passing in a list N parameters, and an N-Dimensional tensor is returned. You can optionally sample each point multiple times to get back a distribution and also track its value over time. By default the data will be plotted automatically based on the types of parameters you are sampling (e.g, continous, discrete), but you can also pass in a callback to customize plotting.
|
68
|
+
|
69
|
+
The data is stored in a persistent database so that past performance is tracked.
|
70
|
+
|
71
|
+
## Assumptions
|
72
|
+
|
73
|
+
The input types should also be of one of the basic datatypes (bool, int, float, str, enum, datetime) so that the data can be easily hashed, cached and stored in the database and processed with seaborn and xarray plotting functions. You can use class inheritance to define hierarchical parameter configuration class types that can be reused in a bigger configuration classes.
|
74
|
+
|
75
|
+
Bencher is designed to work with stochastic pure functions with no side effects. It assumes that when the objective function is given the same inputs, it will return the same output +- random noise. This is because the function must be called multiple times to get a good statistical distribution of it and so each call must not be influenced by anything or the results will be corrupted.
|
76
|
+
|
77
|
+
### Pseudocode of bencher
|
78
|
+
|
79
|
+
Enumerate a list of all input parameter combinations
|
80
|
+
for each set of input parameters:
|
81
|
+
pass the inputs to the objective function and store results in the N-D array
|
82
|
+
|
83
|
+
get unique hash for the set of inputs parameters
|
84
|
+
look up previous results for that hash
|
85
|
+
if it exists:
|
86
|
+
load historical data
|
87
|
+
combine latest data with historical data
|
88
|
+
|
89
|
+
store the results using the input hash as a key
|
90
|
+
deduce the type of plot based on the input types
|
91
|
+
return data and plot
|
92
|
+
|
93
|
+
### Example Output
|
94
|
+
|
95
|
+
https://dyson-ai.github.io/bencher/
|
@@ -0,0 +1,47 @@
|
|
1
|
+
LICENSE
|
2
|
+
README.md
|
3
|
+
package.xml
|
4
|
+
pyproject.toml
|
5
|
+
setup.cfg
|
6
|
+
setup.py
|
7
|
+
bencher/__init__.py
|
8
|
+
bencher/bench_cfg.py
|
9
|
+
bencher/bench_plot_server.py
|
10
|
+
bencher/bench_report.py
|
11
|
+
bencher/bench_runner.py
|
12
|
+
bencher/bencher.py
|
13
|
+
bencher/caching.py
|
14
|
+
bencher/class_enum.py
|
15
|
+
bencher/job.py
|
16
|
+
bencher/optuna_conversions.py
|
17
|
+
bencher/utils.py
|
18
|
+
bencher/video_writer.py
|
19
|
+
bencher/worker_job.py
|
20
|
+
holobench.egg-info/PKG-INFO
|
21
|
+
holobench.egg-info/SOURCES.txt
|
22
|
+
holobench.egg-info/dependency_links.txt
|
23
|
+
holobench.egg-info/not-zip-safe
|
24
|
+
holobench.egg-info/requires.txt
|
25
|
+
holobench.egg-info/top_level.txt
|
26
|
+
resource/bencher
|
27
|
+
test/test_bench_examples.py
|
28
|
+
test/test_bench_report.py
|
29
|
+
test/test_bench_result_base.py
|
30
|
+
test/test_bench_runner.py
|
31
|
+
test/test_bencher.py
|
32
|
+
test/test_cache.py
|
33
|
+
test/test_class_enum.py
|
34
|
+
test/test_combinations.py
|
35
|
+
test/test_composable_container_base.py
|
36
|
+
test/test_composable_container_video.py
|
37
|
+
test/test_float_formatter.py
|
38
|
+
test/test_job.py
|
39
|
+
test/test_meta_tests.py
|
40
|
+
test/test_parametrized_sweep.py
|
41
|
+
test/test_plot_filter.py
|
42
|
+
test/test_plot_server.py
|
43
|
+
test/test_sample_cache.py
|
44
|
+
test/test_sweep_base.py
|
45
|
+
test/test_sweep_vars.py
|
46
|
+
test/test_utils.py
|
47
|
+
test/test_vars.py
|
@@ -0,0 +1 @@
|
|
1
|
+
|
@@ -0,0 +1 @@
|
|
1
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
holoviews<=1.18.3,>=1.15
|
2
|
+
numpy<=1.26.4,>=1.0
|
3
|
+
param<=2.1.0,>=1.13.0
|
4
|
+
hvplot<=0.10.0,>=0.8
|
5
|
+
matplotlib<=3.9.0,>=3.6.3
|
6
|
+
panel<=1.4.4,>=1.3.6
|
7
|
+
diskcache<=5.6.3,>=5.6
|
8
|
+
optuna<=3.6.1,>=3.2
|
9
|
+
xarray<=2024.5.0,>=2023.7
|
10
|
+
plotly<=5.22.0,>=5.15
|
11
|
+
sortedcontainers<=2.4,>=2.4
|
12
|
+
pandas<=2.2.2,>=2.0
|
13
|
+
strenum<=0.4.15,>=0.4.0
|
14
|
+
scikit-learn<=1.5.0,>=1.2
|
15
|
+
str2bool<=1.1,>=1.1
|
16
|
+
scoop<=0.7.2.0,>=0.7.0
|
17
|
+
moviepy<=1.0.3,>=1.0.3
|
18
|
+
|
19
|
+
[test]
|
20
|
+
black<=24.4.2,>=23
|
21
|
+
pylint<=3.2.2,>=2.17.7
|
22
|
+
pytest-cov<=5.0.0,>=4.1
|
23
|
+
pytest<=8.2.1,>=7.4
|
24
|
+
hypothesis<=6.103.0,>=6.82
|
25
|
+
ruff<=0.4.7,>=0.3
|
26
|
+
coverage<=7.5.3,>=7.2.7
|
@@ -0,0 +1 @@
|
|
1
|
+
bencher
|
@@ -0,0 +1,33 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<package format="3">
|
3
|
+
<name>bencher</name>
|
4
|
+
<version>0.1.0</version>
|
5
|
+
<description>A package for benchmarking the performance of arbitrary functions</description>
|
6
|
+
<maintainer email="austin.gregg-smith@dyson.com">Austin Gregg-Smith</maintainer>
|
7
|
+
<license>MIT</license>
|
8
|
+
|
9
|
+
<depend>python3-diskcache</depend>
|
10
|
+
|
11
|
+
<depend>python3-pandas</depend>
|
12
|
+
<depend>python3-seaborn</depend>
|
13
|
+
<depend>python3-matplotlib</depend>
|
14
|
+
<depend>python3-numpy</depend>
|
15
|
+
<depend>python3-pytest</depend>
|
16
|
+
<depend>python3-hypothesis</depend>
|
17
|
+
<depend>xarray</depend>
|
18
|
+
<depend>python3-zarr</depend>
|
19
|
+
<depend>python3-param</depend>
|
20
|
+
<depend>python3-panel</depend>
|
21
|
+
<depend>python3-hvplot</depend>
|
22
|
+
<depend>python3-optuna</depend>
|
23
|
+
<depend>python3-plotly</depend>
|
24
|
+
|
25
|
+
<!-- TO REMOVE WHEN WE UPGRADE TO PYTHON 3.11, This is in the standard library for python>=3.11 -->
|
26
|
+
<depend>strenum</depend>
|
27
|
+
|
28
|
+
<test_depend>python3-pytest-cov</test_depend>
|
29
|
+
|
30
|
+
<export>
|
31
|
+
<build_type>ament_python</build_type>
|
32
|
+
</export>
|
33
|
+
</package>
|