holobench 1.19.0__tar.gz → 1.22.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- holobench-1.22.2/LICENSE +21 -0
- {holobench-1.19.0 → holobench-1.22.2}/PKG-INFO +37 -29
- {holobench-1.19.0 → holobench-1.22.2}/README.md +9 -2
- {holobench-1.19.0 → holobench-1.22.2}/bencher/__init__.py +1 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/utils.py +13 -0
- holobench-1.22.2/holobench.egg-info/PKG-INFO +93 -0
- holobench-1.22.2/holobench.egg-info/SOURCES.txt +45 -0
- holobench-1.22.2/holobench.egg-info/dependency_links.txt +1 -0
- holobench-1.22.2/holobench.egg-info/not-zip-safe +1 -0
- holobench-1.22.2/holobench.egg-info/requires.txt +26 -0
- holobench-1.22.2/holobench.egg-info/top_level.txt +1 -0
- holobench-1.22.2/package.xml +33 -0
- {holobench-1.19.0 → holobench-1.22.2}/pyproject.toml +63 -33
- holobench-1.22.2/setup.cfg +10 -0
- holobench-1.22.2/setup.py +21 -0
- holobench-1.22.2/test/test_bench_examples.py +129 -0
- holobench-1.22.2/test/test_bench_report.py +55 -0
- holobench-1.22.2/test/test_bench_result_base.py +79 -0
- holobench-1.22.2/test/test_bench_runner.py +196 -0
- holobench-1.22.2/test/test_bencher.py +383 -0
- holobench-1.22.2/test/test_cache.py +46 -0
- holobench-1.22.2/test/test_class_enum.py +10 -0
- holobench-1.22.2/test/test_combinations.py +182 -0
- holobench-1.22.2/test/test_float_formatter.py +26 -0
- holobench-1.22.2/test/test_job.py +98 -0
- holobench-1.22.2/test/test_meta_tests.py +55 -0
- holobench-1.22.2/test/test_parametrized_sweep.py +74 -0
- holobench-1.22.2/test/test_plot_filter.py +91 -0
- holobench-1.22.2/test/test_plot_server.py +55 -0
- holobench-1.22.2/test/test_sample_cache.py +113 -0
- holobench-1.22.2/test/test_sweep_base.py +198 -0
- holobench-1.22.2/test/test_sweep_vars.py +176 -0
- holobench-1.22.2/test/test_utils.py +172 -0
- holobench-1.22.2/test/test_vars.py +69 -0
- holobench-1.19.0/bencher/example/benchmark_data.py +0 -200
- holobench-1.19.0/bencher/example/example_all.py +0 -45
- holobench-1.19.0/bencher/example/example_categorical.py +0 -99
- holobench-1.19.0/bencher/example/example_custom_sweep.py +0 -59
- holobench-1.19.0/bencher/example/example_docs.py +0 -34
- holobench-1.19.0/bencher/example/example_filepath.py +0 -27
- holobench-1.19.0/bencher/example/example_float3D.py +0 -101
- holobench-1.19.0/bencher/example/example_float_cat.py +0 -99
- holobench-1.19.0/bencher/example/example_floats.py +0 -89
- holobench-1.19.0/bencher/example/example_floats2D.py +0 -93
- holobench-1.19.0/bencher/example/example_holosweep.py +0 -98
- holobench-1.19.0/bencher/example/example_holosweep_objects.py +0 -111
- holobench-1.19.0/bencher/example/example_holosweep_tap.py +0 -144
- holobench-1.19.0/bencher/example/example_image.py +0 -138
- holobench-1.19.0/bencher/example/example_levels.py +0 -181
- holobench-1.19.0/bencher/example/example_pareto.py +0 -53
- holobench-1.19.0/bencher/example/example_sample_cache.py +0 -85
- holobench-1.19.0/bencher/example/example_sample_cache_context.py +0 -116
- holobench-1.19.0/bencher/example/example_simple.py +0 -134
- holobench-1.19.0/bencher/example/example_simple_bool.py +0 -35
- holobench-1.19.0/bencher/example/example_simple_cat.py +0 -48
- holobench-1.19.0/bencher/example/example_simple_float.py +0 -38
- holobench-1.19.0/bencher/example/example_strings.py +0 -46
- holobench-1.19.0/bencher/example/example_time_event.py +0 -63
- holobench-1.19.0/bencher/example/example_video.py +0 -127
- holobench-1.19.0/bencher/example/example_workflow.py +0 -189
- holobench-1.19.0/bencher/example/experimental/example_bokeh_plotly.py +0 -38
- holobench-1.19.0/bencher/example/experimental/example_hover_ex.py +0 -45
- holobench-1.19.0/bencher/example/experimental/example_hvplot_explorer.py +0 -39
- holobench-1.19.0/bencher/example/experimental/example_interactive.py +0 -75
- holobench-1.19.0/bencher/example/experimental/example_streamnd.py +0 -49
- holobench-1.19.0/bencher/example/experimental/example_streams.py +0 -36
- holobench-1.19.0/bencher/example/experimental/example_template.py +0 -40
- holobench-1.19.0/bencher/example/experimental/example_updates.py +0 -84
- holobench-1.19.0/bencher/example/experimental/example_vector.py +0 -84
- holobench-1.19.0/bencher/example/meta/example_meta.py +0 -171
- holobench-1.19.0/bencher/example/meta/example_meta_cat.py +0 -25
- holobench-1.19.0/bencher/example/meta/example_meta_float.py +0 -23
- holobench-1.19.0/bencher/example/meta/example_meta_levels.py +0 -26
- holobench-1.19.0/bencher/example/optuna/example_optuna.py +0 -78
- holobench-1.19.0/bencher/example/shelved/example_float2D_scatter.py +0 -109
- holobench-1.19.0/bencher/example/shelved/example_float3D_cone.py +0 -96
- holobench-1.19.0/bencher/example/shelved/example_kwargs.py +0 -63
- holobench-1.19.0/bencher/plotting/__init__.py +0 -0
- holobench-1.19.0/bencher/plotting/plot_filter.py +0 -110
- holobench-1.19.0/bencher/plotting/plt_cnt_cfg.py +0 -74
- holobench-1.19.0/bencher/results/__init__.py +0 -0
- holobench-1.19.0/bencher/results/bench_result.py +0 -94
- holobench-1.19.0/bencher/results/bench_result_base.py +0 -475
- holobench-1.19.0/bencher/results/composable_container/__init__.py +0 -0
- holobench-1.19.0/bencher/results/composable_container/composable_container_base.py +0 -60
- holobench-1.19.0/bencher/results/composable_container/composable_container_panel.py +0 -40
- holobench-1.19.0/bencher/results/composable_container/composable_container_video.py +0 -76
- holobench-1.19.0/bencher/results/float_formatter.py +0 -44
- holobench-1.19.0/bencher/results/holoview_result.py +0 -745
- holobench-1.19.0/bencher/results/optuna_result.py +0 -354
- holobench-1.19.0/bencher/results/panel_result.py +0 -41
- holobench-1.19.0/bencher/results/plotly_result.py +0 -65
- holobench-1.19.0/bencher/results/video_result.py +0 -38
- holobench-1.19.0/bencher/results/video_summary.py +0 -217
- holobench-1.19.0/bencher/variables/inputs.py +0 -179
- holobench-1.19.0/bencher/variables/parametrised_sweep.py +0 -208
- holobench-1.19.0/bencher/variables/results.py +0 -214
- holobench-1.19.0/bencher/variables/sweep_base.py +0 -162
- holobench-1.19.0/bencher/variables/time.py +0 -70
- {holobench-1.19.0 → holobench-1.22.2}/bencher/bench_cfg.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/bench_plot_server.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/bench_report.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/bench_runner.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/bencher.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/caching.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/class_enum.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/job.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/optuna_conversions.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/video_writer.py +0 -0
- {holobench-1.19.0 → holobench-1.22.2}/bencher/worker_job.py +0 -0
- /holobench-1.19.0/bencher/example/__init__.py → /holobench-1.22.2/resource/bencher +0 -0
holobench-1.22.2/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2023 Dyson AI
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
@@ -1,37 +1,38 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: holobench
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.22.2
|
4
4
|
Summary: A package for benchmarking the performance of arbitrary functions
|
5
5
|
Author-email: Austin Gregg-Smith <blooop@gmail.com>
|
6
|
-
Description-Content-Type: text/markdown
|
7
|
-
Requires-Dist: holoviews>=1.15,<=1.18.3
|
8
|
-
Requires-Dist: numpy>=1.0,<=1.26.4
|
9
|
-
Requires-Dist: param>=1.13.0,<=2.1.0
|
10
|
-
Requires-Dist: hvplot>=0.8,<=0.10.0
|
11
|
-
Requires-Dist: matplotlib>=3.6.3,<=3.8.4
|
12
|
-
Requires-Dist: panel>=1.3.6,<=1.4.2
|
13
|
-
Requires-Dist: diskcache>=5.6,<=5.6.3
|
14
|
-
Requires-Dist: optuna>=3.2,<=3.6.1
|
15
|
-
Requires-Dist: xarray>=2023.7,<=2024.5.0
|
16
|
-
Requires-Dist: plotly>=5.15,<=5.22.0
|
17
|
-
Requires-Dist: sortedcontainers>=2.4,<=2.4
|
18
|
-
Requires-Dist: pandas>=2.0,<=2.2.2
|
19
|
-
Requires-Dist: strenum>=0.4.0,<=0.4.15
|
20
|
-
Requires-Dist: scikit-learn>=1.2,<=1.4.2
|
21
|
-
Requires-Dist: str2bool>=1.1,<=1.1
|
22
|
-
Requires-Dist: scoop>=0.7.0,<=0.7.2.0
|
23
|
-
Requires-Dist: moviepy>=1.0.3,<=1.0.3
|
24
|
-
Requires-Dist: black>=23,<=24.4.2 ; extra == "test"
|
25
|
-
Requires-Dist: pylint>=2.16,<=3.1.0 ; extra == "test"
|
26
|
-
Requires-Dist: pytest-cov>=4.1,<=5.0.0 ; extra == "test"
|
27
|
-
Requires-Dist: pytest>=7.4,<=8.2.0 ; extra == "test"
|
28
|
-
Requires-Dist: hypothesis>=6.82,<=6.101.0 ; extra == "test"
|
29
|
-
Requires-Dist: ruff>=0.0.280,<=0.4.4 ; extra == "test"
|
30
|
-
Requires-Dist: coverage>=7.2.7,<=7.5.1 ; extra == "test"
|
31
|
-
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
32
|
-
Project-URL: Home, https://github.com/dyson-ai/bencher
|
33
6
|
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
7
|
+
Project-URL: Home, https://github.com/dyson-ai/bencher
|
8
|
+
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
9
|
+
Description-Content-Type: text/markdown
|
10
|
+
License-File: LICENSE
|
11
|
+
Requires-Dist: holoviews<=1.18.3,>=1.15
|
12
|
+
Requires-Dist: numpy<=1.26.4,>=1.0
|
13
|
+
Requires-Dist: param<=2.1.0,>=1.13.0
|
14
|
+
Requires-Dist: hvplot<=0.10.0,>=0.8
|
15
|
+
Requires-Dist: matplotlib<=3.9.0,>=3.6.3
|
16
|
+
Requires-Dist: panel<=1.4.3,>=1.3.6
|
17
|
+
Requires-Dist: diskcache<=5.6.3,>=5.6
|
18
|
+
Requires-Dist: optuna<=3.6.1,>=3.2
|
19
|
+
Requires-Dist: xarray<=2024.5.0,>=2023.7
|
20
|
+
Requires-Dist: plotly<=5.22.0,>=5.15
|
21
|
+
Requires-Dist: sortedcontainers<=2.4,>=2.4
|
22
|
+
Requires-Dist: pandas<=2.2.2,>=2.0
|
23
|
+
Requires-Dist: strenum<=0.4.15,>=0.4.0
|
24
|
+
Requires-Dist: scikit-learn<=1.5.0,>=1.2
|
25
|
+
Requires-Dist: str2bool<=1.1,>=1.1
|
26
|
+
Requires-Dist: scoop<=0.7.2.0,>=0.7.0
|
27
|
+
Requires-Dist: moviepy<=1.0.3,>=1.0.3
|
34
28
|
Provides-Extra: test
|
29
|
+
Requires-Dist: black<=24.4.2,>=23; extra == "test"
|
30
|
+
Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
|
31
|
+
Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
|
32
|
+
Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
|
33
|
+
Requires-Dist: hypothesis<=6.102.6,>=6.82; extra == "test"
|
34
|
+
Requires-Dist: ruff<=0.4.5,>=0.3; extra == "test"
|
35
|
+
Requires-Dist: coverage<=7.5.2,>=7.2.7; extra == "test"
|
35
36
|
|
36
37
|
# Bencher
|
37
38
|
|
@@ -45,7 +46,14 @@ Provides-Extra: test
|
|
45
46
|
[](https://pypi.org/project/holobench/)
|
46
47
|
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
47
48
|
[](https://opensource.org/license/mit/)
|
48
|
-
[](https://www.python.org/downloads/release/python-310/)
|
50
|
+
[](https://www.python.org/downloads/release/python-311/)
|
51
|
+
[](https://pixi.sh)
|
52
|
+
|
53
|
+
## Badges
|
54
|
+
|
55
|
+
### Pixi Badge
|
56
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
49
57
|
|
50
58
|
|
51
59
|
## Intro
|
@@ -10,7 +10,14 @@
|
|
10
10
|
[](https://pypi.org/project/holobench/)
|
11
11
|
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
12
12
|
[](https://opensource.org/license/mit/)
|
13
|
-
[](https://www.python.org/downloads/release/python-310/)
|
14
|
+
[](https://www.python.org/downloads/release/python-311/)
|
15
|
+
[](https://pixi.sh)
|
16
|
+
|
17
|
+
## Badges
|
18
|
+
|
19
|
+
### Pixi Badge
|
20
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
14
21
|
|
15
22
|
|
16
23
|
## Intro
|
@@ -47,4 +54,4 @@ Bencher is designed to work with stochastic pure functions with no side effects.
|
|
47
54
|
|
48
55
|
### Example Output
|
49
56
|
|
50
|
-
https://dyson-ai.github.io/bencher/
|
57
|
+
https://dyson-ai.github.io/bencher/
|
@@ -98,6 +98,19 @@ def un_camel(camel: str) -> str:
|
|
98
98
|
return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
|
99
99
|
|
100
100
|
|
101
|
+
def tabs_in_markdown(regular_str: str, spaces: int = 2) -> str:
|
102
|
+
"""Given a string with tabs in the form \t convert the to   which is a double space in markdown
|
103
|
+
|
104
|
+
Args:
|
105
|
+
regular_str (str): A string with tabs in it
|
106
|
+
spaces (int): the number of spaces per tab
|
107
|
+
|
108
|
+
Returns:
|
109
|
+
str: A string with sets of to represent the tabs in markdown
|
110
|
+
"""
|
111
|
+
return regular_str.replace("\t", "".join([" "] * spaces))
|
112
|
+
|
113
|
+
|
101
114
|
def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
|
102
115
|
"""Uses the golden angle to generate colors programmatically with minimum overlap between colors.
|
103
116
|
https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
|
@@ -0,0 +1,93 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: holobench
|
3
|
+
Version: 1.22.2
|
4
|
+
Summary: A package for benchmarking the performance of arbitrary functions
|
5
|
+
Author-email: Austin Gregg-Smith <blooop@gmail.com>
|
6
|
+
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
7
|
+
Project-URL: Home, https://github.com/dyson-ai/bencher
|
8
|
+
Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
|
9
|
+
Description-Content-Type: text/markdown
|
10
|
+
License-File: LICENSE
|
11
|
+
Requires-Dist: holoviews<=1.18.3,>=1.15
|
12
|
+
Requires-Dist: numpy<=1.26.4,>=1.0
|
13
|
+
Requires-Dist: param<=2.1.0,>=1.13.0
|
14
|
+
Requires-Dist: hvplot<=0.10.0,>=0.8
|
15
|
+
Requires-Dist: matplotlib<=3.9.0,>=3.6.3
|
16
|
+
Requires-Dist: panel<=1.4.3,>=1.3.6
|
17
|
+
Requires-Dist: diskcache<=5.6.3,>=5.6
|
18
|
+
Requires-Dist: optuna<=3.6.1,>=3.2
|
19
|
+
Requires-Dist: xarray<=2024.5.0,>=2023.7
|
20
|
+
Requires-Dist: plotly<=5.22.0,>=5.15
|
21
|
+
Requires-Dist: sortedcontainers<=2.4,>=2.4
|
22
|
+
Requires-Dist: pandas<=2.2.2,>=2.0
|
23
|
+
Requires-Dist: strenum<=0.4.15,>=0.4.0
|
24
|
+
Requires-Dist: scikit-learn<=1.5.0,>=1.2
|
25
|
+
Requires-Dist: str2bool<=1.1,>=1.1
|
26
|
+
Requires-Dist: scoop<=0.7.2.0,>=0.7.0
|
27
|
+
Requires-Dist: moviepy<=1.0.3,>=1.0.3
|
28
|
+
Provides-Extra: test
|
29
|
+
Requires-Dist: black<=24.4.2,>=23; extra == "test"
|
30
|
+
Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
|
31
|
+
Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
|
32
|
+
Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
|
33
|
+
Requires-Dist: hypothesis<=6.102.6,>=6.82; extra == "test"
|
34
|
+
Requires-Dist: ruff<=0.4.5,>=0.3; extra == "test"
|
35
|
+
Requires-Dist: coverage<=7.5.2,>=7.2.7; extra == "test"
|
36
|
+
|
37
|
+
# Bencher
|
38
|
+
|
39
|
+
## Continuous Integration Status
|
40
|
+
|
41
|
+
[](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml?query=branch%3Amain)
|
42
|
+

|
43
|
+
[](https://codecov.io/gh/dyson-ai/bencher)
|
44
|
+
[](https://GitHub.com/dyson-ai/bencher/issues/)
|
45
|
+
[](https://github.com/dyson-ai/bencher/pulls?q=is%3Amerged)
|
46
|
+
[](https://pypi.org/project/holobench/)
|
47
|
+
[](https://GitHub.com/dyson-ai/bencher/releases/)
|
48
|
+
[](https://opensource.org/license/mit/)
|
49
|
+
[](https://www.python.org/downloads/release/python-310/)
|
50
|
+
[](https://www.python.org/downloads/release/python-311/)
|
51
|
+
[](https://pixi.sh)
|
52
|
+
|
53
|
+
## Badges
|
54
|
+
|
55
|
+
### Pixi Badge
|
56
|
+
The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
|
57
|
+
|
58
|
+
|
59
|
+
## Intro
|
60
|
+
|
61
|
+
Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics. It calculates the [cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) of a set of variables
|
62
|
+
|
63
|
+
Parameters for bencher are defined using the [param](https://param.holoviz.org/) library as a config class with extra metadata that describes the bounds of the search space you want to measure. You must define a benchmarking function that accepts an instance of the config class and return a dictionary with string metric names and float values.
|
64
|
+
|
65
|
+
Parameters are benchmarked by passing in a list N parameters, and an N-Dimensional tensor is returned. You can optionally sample each point multiple times to get back a distribution and also track its value over time. By default the data will be plotted automatically based on the types of parameters you are sampling (e.g, continous, discrete), but you can also pass in a callback to customize plotting.
|
66
|
+
|
67
|
+
The data is stored in a persistent database so that past performance is tracked.
|
68
|
+
|
69
|
+
## Assumptions
|
70
|
+
|
71
|
+
The input types should also be of one of the basic datatypes (bool, int, float, str, enum, datetime) so that the data can be easily hashed, cached and stored in the database and processed with seaborn and xarray plotting functions. You can use class inheritance to define hierarchical parameter configuration class types that can be reused in a bigger configuration classes.
|
72
|
+
|
73
|
+
Bencher is designed to work with stochastic pure functions with no side effects. It assumes that when the objective function is given the same inputs, it will return the same output +- random noise. This is because the function must be called multiple times to get a good statistical distribution of it and so each call must not be influenced by anything or the results will be corrupted.
|
74
|
+
|
75
|
+
### Pseudocode of bencher
|
76
|
+
|
77
|
+
Enumerate a list of all input parameter combinations
|
78
|
+
for each set of input parameters:
|
79
|
+
pass the inputs to the objective function and store results in the N-D array
|
80
|
+
|
81
|
+
get unique hash for the set of inputs parameters
|
82
|
+
look up previous results for that hash
|
83
|
+
if it exists:
|
84
|
+
load historical data
|
85
|
+
combine latest data with historical data
|
86
|
+
|
87
|
+
store the results using the input hash as a key
|
88
|
+
deduce the type of plot based on the input types
|
89
|
+
return data and plot
|
90
|
+
|
91
|
+
### Example Output
|
92
|
+
|
93
|
+
https://dyson-ai.github.io/bencher/
|
@@ -0,0 +1,45 @@
|
|
1
|
+
LICENSE
|
2
|
+
README.md
|
3
|
+
package.xml
|
4
|
+
pyproject.toml
|
5
|
+
setup.cfg
|
6
|
+
setup.py
|
7
|
+
bencher/__init__.py
|
8
|
+
bencher/bench_cfg.py
|
9
|
+
bencher/bench_plot_server.py
|
10
|
+
bencher/bench_report.py
|
11
|
+
bencher/bench_runner.py
|
12
|
+
bencher/bencher.py
|
13
|
+
bencher/caching.py
|
14
|
+
bencher/class_enum.py
|
15
|
+
bencher/job.py
|
16
|
+
bencher/optuna_conversions.py
|
17
|
+
bencher/utils.py
|
18
|
+
bencher/video_writer.py
|
19
|
+
bencher/worker_job.py
|
20
|
+
holobench.egg-info/PKG-INFO
|
21
|
+
holobench.egg-info/SOURCES.txt
|
22
|
+
holobench.egg-info/dependency_links.txt
|
23
|
+
holobench.egg-info/not-zip-safe
|
24
|
+
holobench.egg-info/requires.txt
|
25
|
+
holobench.egg-info/top_level.txt
|
26
|
+
resource/bencher
|
27
|
+
test/test_bench_examples.py
|
28
|
+
test/test_bench_report.py
|
29
|
+
test/test_bench_result_base.py
|
30
|
+
test/test_bench_runner.py
|
31
|
+
test/test_bencher.py
|
32
|
+
test/test_cache.py
|
33
|
+
test/test_class_enum.py
|
34
|
+
test/test_combinations.py
|
35
|
+
test/test_float_formatter.py
|
36
|
+
test/test_job.py
|
37
|
+
test/test_meta_tests.py
|
38
|
+
test/test_parametrized_sweep.py
|
39
|
+
test/test_plot_filter.py
|
40
|
+
test/test_plot_server.py
|
41
|
+
test/test_sample_cache.py
|
42
|
+
test/test_sweep_base.py
|
43
|
+
test/test_sweep_vars.py
|
44
|
+
test/test_utils.py
|
45
|
+
test/test_vars.py
|
@@ -0,0 +1 @@
|
|
1
|
+
|
@@ -0,0 +1 @@
|
|
1
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
holoviews<=1.18.3,>=1.15
|
2
|
+
numpy<=1.26.4,>=1.0
|
3
|
+
param<=2.1.0,>=1.13.0
|
4
|
+
hvplot<=0.10.0,>=0.8
|
5
|
+
matplotlib<=3.9.0,>=3.6.3
|
6
|
+
panel<=1.4.3,>=1.3.6
|
7
|
+
diskcache<=5.6.3,>=5.6
|
8
|
+
optuna<=3.6.1,>=3.2
|
9
|
+
xarray<=2024.5.0,>=2023.7
|
10
|
+
plotly<=5.22.0,>=5.15
|
11
|
+
sortedcontainers<=2.4,>=2.4
|
12
|
+
pandas<=2.2.2,>=2.0
|
13
|
+
strenum<=0.4.15,>=0.4.0
|
14
|
+
scikit-learn<=1.5.0,>=1.2
|
15
|
+
str2bool<=1.1,>=1.1
|
16
|
+
scoop<=0.7.2.0,>=0.7.0
|
17
|
+
moviepy<=1.0.3,>=1.0.3
|
18
|
+
|
19
|
+
[test]
|
20
|
+
black<=24.4.2,>=23
|
21
|
+
pylint<=3.2.2,>=2.17.7
|
22
|
+
pytest-cov<=5.0.0,>=4.1
|
23
|
+
pytest<=8.2.1,>=7.4
|
24
|
+
hypothesis<=6.102.6,>=6.82
|
25
|
+
ruff<=0.4.5,>=0.3
|
26
|
+
coverage<=7.5.2,>=7.2.7
|
@@ -0,0 +1 @@
|
|
1
|
+
bencher
|
@@ -0,0 +1,33 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<package format="3">
|
3
|
+
<name>bencher</name>
|
4
|
+
<version>0.1.0</version>
|
5
|
+
<description>A package for benchmarking the performance of arbitrary functions</description>
|
6
|
+
<maintainer email="austin.gregg-smith@dyson.com">Austin Gregg-Smith</maintainer>
|
7
|
+
<license>MIT</license>
|
8
|
+
|
9
|
+
<depend>python3-diskcache</depend>
|
10
|
+
|
11
|
+
<depend>python3-pandas</depend>
|
12
|
+
<depend>python3-seaborn</depend>
|
13
|
+
<depend>python3-matplotlib</depend>
|
14
|
+
<depend>python3-numpy</depend>
|
15
|
+
<depend>python3-pytest</depend>
|
16
|
+
<depend>python3-hypothesis</depend>
|
17
|
+
<depend>xarray</depend>
|
18
|
+
<depend>python3-zarr</depend>
|
19
|
+
<depend>python3-param</depend>
|
20
|
+
<depend>python3-panel</depend>
|
21
|
+
<depend>python3-hvplot</depend>
|
22
|
+
<depend>python3-optuna</depend>
|
23
|
+
<depend>python3-plotly</depend>
|
24
|
+
|
25
|
+
<!-- TO REMOVE WHEN WE UPGRADE TO PYTHON 3.11, This is in the standard library for python>=3.11 -->
|
26
|
+
<depend>strenum</depend>
|
27
|
+
|
28
|
+
<test_depend>python3-pytest-cov</test_depend>
|
29
|
+
|
30
|
+
<export>
|
31
|
+
<build_type>ament_python</build_type>
|
32
|
+
</export>
|
33
|
+
</package>
|
@@ -1,22 +1,18 @@
|
|
1
1
|
[project]
|
2
2
|
name = "holobench"
|
3
|
-
version = "1.
|
3
|
+
version = "1.22.2"
|
4
4
|
|
5
5
|
authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
|
6
6
|
description = "A package for benchmarking the performance of arbitrary functions"
|
7
7
|
readme = "README.md"
|
8
8
|
|
9
|
-
#incompatible with setuptools
|
10
|
-
# requires-python = ">= 3.10"
|
11
|
-
|
12
|
-
|
13
9
|
dependencies = [
|
14
10
|
"holoviews>=1.15,<=1.18.3",
|
15
11
|
"numpy>=1.0,<=1.26.4",
|
16
12
|
"param>=1.13.0,<=2.1.0",
|
17
13
|
"hvplot>=0.8,<=0.10.0",
|
18
|
-
"matplotlib>=3.6.3,<=3.
|
19
|
-
"panel>=1.3.6,<=1.4.
|
14
|
+
"matplotlib>=3.6.3,<=3.9.0",
|
15
|
+
"panel>=1.3.6,<=1.4.3",
|
20
16
|
"diskcache>=5.6,<=5.6.3",
|
21
17
|
"optuna>=3.2,<=3.6.1",
|
22
18
|
"xarray>=2023.7,<=2024.5.0",
|
@@ -24,21 +20,37 @@ dependencies = [
|
|
24
20
|
"sortedcontainers>=2.4,<=2.4",
|
25
21
|
"pandas>=2.0,<=2.2.2",
|
26
22
|
"strenum>=0.4.0,<=0.4.15",
|
27
|
-
"scikit-learn>=1.2,<=1.
|
23
|
+
"scikit-learn>=1.2,<=1.5.0",
|
28
24
|
"str2bool>=1.1,<=1.1",
|
29
25
|
"scoop>=0.7.0,<=0.7.2.0",
|
30
26
|
"moviepy>=1.0.3,<=1.0.3",
|
31
27
|
]
|
32
28
|
|
29
|
+
[tool.pixi.project]
|
30
|
+
channels = ["conda-forge"]
|
31
|
+
platforms = ["linux-64"]
|
32
|
+
|
33
|
+
[tool.pixi.dependencies]
|
34
|
+
python = ">=3.10"
|
35
|
+
|
36
|
+
|
37
|
+
[tool.pixi.feature.py310.dependencies]
|
38
|
+
python = "3.10.*"
|
39
|
+
[tool.pixi.feature.py311.dependencies]
|
40
|
+
python = "3.11.*"
|
41
|
+
|
42
|
+
[tool.pixi.pypi-dependencies]
|
43
|
+
holobench = { path = ".", editable = true }
|
44
|
+
|
33
45
|
[project.optional-dependencies]
|
34
46
|
test = [
|
35
47
|
"black>=23,<=24.4.2",
|
36
|
-
"pylint>=2.
|
48
|
+
"pylint>=2.17.7,<=3.2.2",
|
37
49
|
"pytest-cov>=4.1,<=5.0.0",
|
38
|
-
"pytest>=7.4,<=8.2.
|
39
|
-
"hypothesis>=6.82,<=6.
|
40
|
-
"ruff>=0.
|
41
|
-
"coverage>=7.2.7,<=7.5.
|
50
|
+
"pytest>=7.4,<=8.2.1",
|
51
|
+
"hypothesis>=6.82,<=6.102.6",
|
52
|
+
"ruff>=0.3,<=0.4.5",
|
53
|
+
"coverage>=7.2.7,<=7.5.2",
|
42
54
|
]
|
43
55
|
|
44
56
|
[project.urls]
|
@@ -46,18 +58,39 @@ Repository = "https://github.com/dyson-ai/bencher"
|
|
46
58
|
Home = "https://github.com/dyson-ai/bencher"
|
47
59
|
Documentation = "https://bencher.readthedocs.io/en/latest/"
|
48
60
|
|
49
|
-
|
50
|
-
[tool.flit.module]
|
51
|
-
name = "bencher"
|
52
|
-
|
53
|
-
|
54
61
|
[build-system]
|
55
|
-
requires = ["
|
56
|
-
build-backend = "
|
57
|
-
|
58
|
-
|
59
|
-
[tool.
|
60
|
-
|
62
|
+
requires = ["setuptools"]
|
63
|
+
build-backend = "setuptools.build_meta"
|
64
|
+
|
65
|
+
# Environments
|
66
|
+
[tool.pixi.environments]
|
67
|
+
default = { features = ["test"], solve-group = "default" }
|
68
|
+
py310 = ["py310", "test"]
|
69
|
+
py311 = ["py311", "test"]
|
70
|
+
|
71
|
+
[tool.pixi.tasks]
|
72
|
+
format = "black ."
|
73
|
+
check-clean-workspace = "git diff --exit-code"
|
74
|
+
ruff-lint = "ruff check . --fix"
|
75
|
+
pylint = "pylint --version && echo 'running pylint...'&& pylint $(git ls-files '*.py')"
|
76
|
+
lint = { depends_on = ["ruff-lint", "pylint"] }
|
77
|
+
style = { depends_on = ["format", "lint"] }
|
78
|
+
test = "pytest"
|
79
|
+
coverage = "coverage run -m pytest && coverage xml -o coverage.xml"
|
80
|
+
coverage-report = "coverage report -m"
|
81
|
+
ci-no-cover = { depends_on = ["style", "test"] }
|
82
|
+
ci = { depends_on = [
|
83
|
+
"style",
|
84
|
+
"check-clean-workspace",
|
85
|
+
"coverage",
|
86
|
+
"coverage-report",
|
87
|
+
] }
|
88
|
+
update-lock-push = "pixi update; git commit -a -m'update pixi.lock';git push"
|
89
|
+
clear-pixi = "rm -rf .pixi pixi.lock"
|
90
|
+
|
91
|
+
|
92
|
+
[tool.setuptools.packages.find]
|
93
|
+
include = ["bencher"]
|
61
94
|
|
62
95
|
[tool.pylint]
|
63
96
|
extension-pkg-whitelist = ["numpy", "scipy"]
|
@@ -70,20 +103,17 @@ enable = "no-else-return,consider-using-in"
|
|
70
103
|
[tool.black]
|
71
104
|
line-length = 100
|
72
105
|
|
73
|
-
|
74
106
|
[tool.ruff]
|
75
|
-
|
76
|
-
#"F841" will auto remove unused variables which is annoying during development, pylint catches this anyway
|
77
|
-
lint.ignore = ["E501", "E902", "F841"]
|
78
|
-
|
79
|
-
|
80
|
-
# Same as Black.
|
81
|
-
line-length = 100
|
107
|
+
line-length = 100 # Same as Black.
|
82
108
|
|
83
109
|
target-version = "py310"
|
84
110
|
|
111
|
+
[tool.ruff.lint]
|
112
|
+
# Never enforce `E501` (line length violations).
|
113
|
+
#"F841" will auto remove unused variables which is annoying during development, pylint catches this anyway
|
114
|
+
ignore = ["E501", "E902", "F841"]
|
85
115
|
# Allow unused variables when underscore-prefixed.
|
86
|
-
|
116
|
+
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
87
117
|
|
88
118
|
# Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`.
|
89
119
|
[tool.ruff.lint.per-file-ignores]
|
@@ -0,0 +1,21 @@
|
|
1
|
+
from setuptools import setup, find_packages
|
2
|
+
|
3
|
+
package_name = "bencher"
|
4
|
+
|
5
|
+
setup(
|
6
|
+
name=package_name,
|
7
|
+
# version="0.0.1",
|
8
|
+
description="A library for benchmarking code and generating reports for analysis",
|
9
|
+
# maintainer="austin.gregg-smith",
|
10
|
+
# maintainer_email="austin.gregg-smith@dyson.com",
|
11
|
+
packages=find_packages(exclude=["test.*", "test"]),
|
12
|
+
data_files=[
|
13
|
+
("share/ament_index/resource_index/packages", ["resource/" + package_name]),
|
14
|
+
("share/" + package_name, ["package.xml"]),
|
15
|
+
],
|
16
|
+
include_package_data=False,
|
17
|
+
# install_requires=["setuptools"],
|
18
|
+
zip_safe=False,
|
19
|
+
test_suite="test",
|
20
|
+
tests_require=["pytest"],
|
21
|
+
)
|