holobench 1.19.0__tar.gz → 1.23.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. holobench-1.23.0/LICENSE +21 -0
  2. {holobench-1.19.0 → holobench-1.23.0}/PKG-INFO +39 -29
  3. {holobench-1.19.0 → holobench-1.23.0}/README.md +9 -2
  4. {holobench-1.19.0 → holobench-1.23.0}/bencher/__init__.py +10 -0
  5. {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_runner.py +1 -1
  6. {holobench-1.19.0 → holobench-1.23.0}/bencher/bencher.py +3 -1
  7. {holobench-1.19.0 → holobench-1.23.0}/bencher/optuna_conversions.py +3 -2
  8. {holobench-1.19.0 → holobench-1.23.0}/bencher/utils.py +28 -2
  9. holobench-1.23.0/bencher/video_writer.py +67 -0
  10. holobench-1.23.0/holobench.egg-info/PKG-INFO +95 -0
  11. holobench-1.23.0/holobench.egg-info/SOURCES.txt +47 -0
  12. holobench-1.23.0/holobench.egg-info/dependency_links.txt +1 -0
  13. holobench-1.23.0/holobench.egg-info/not-zip-safe +1 -0
  14. holobench-1.23.0/holobench.egg-info/requires.txt +26 -0
  15. holobench-1.23.0/holobench.egg-info/top_level.txt +1 -0
  16. holobench-1.23.0/package.xml +33 -0
  17. holobench-1.23.0/pyproject.toml +142 -0
  18. holobench-1.23.0/setup.cfg +10 -0
  19. holobench-1.23.0/setup.py +18 -0
  20. holobench-1.23.0/test/test_bench_examples.py +129 -0
  21. holobench-1.23.0/test/test_bench_report.py +55 -0
  22. holobench-1.23.0/test/test_bench_result_base.py +79 -0
  23. holobench-1.23.0/test/test_bench_runner.py +196 -0
  24. holobench-1.23.0/test/test_bencher.py +383 -0
  25. holobench-1.23.0/test/test_cache.py +46 -0
  26. holobench-1.23.0/test/test_class_enum.py +10 -0
  27. holobench-1.23.0/test/test_combinations.py +182 -0
  28. holobench-1.23.0/test/test_composable_container_base.py +77 -0
  29. holobench-1.23.0/test/test_composable_container_video.py +255 -0
  30. holobench-1.23.0/test/test_float_formatter.py +26 -0
  31. holobench-1.23.0/test/test_job.py +98 -0
  32. holobench-1.23.0/test/test_meta_tests.py +55 -0
  33. holobench-1.23.0/test/test_parametrized_sweep.py +74 -0
  34. holobench-1.23.0/test/test_plot_filter.py +91 -0
  35. holobench-1.23.0/test/test_plot_server.py +55 -0
  36. holobench-1.23.0/test/test_sample_cache.py +113 -0
  37. holobench-1.23.0/test/test_sweep_base.py +198 -0
  38. holobench-1.23.0/test/test_sweep_vars.py +176 -0
  39. holobench-1.23.0/test/test_utils.py +176 -0
  40. holobench-1.23.0/test/test_vars.py +69 -0
  41. holobench-1.19.0/bencher/example/benchmark_data.py +0 -200
  42. holobench-1.19.0/bencher/example/example_all.py +0 -45
  43. holobench-1.19.0/bencher/example/example_categorical.py +0 -99
  44. holobench-1.19.0/bencher/example/example_custom_sweep.py +0 -59
  45. holobench-1.19.0/bencher/example/example_docs.py +0 -34
  46. holobench-1.19.0/bencher/example/example_filepath.py +0 -27
  47. holobench-1.19.0/bencher/example/example_float3D.py +0 -101
  48. holobench-1.19.0/bencher/example/example_float_cat.py +0 -99
  49. holobench-1.19.0/bencher/example/example_floats.py +0 -89
  50. holobench-1.19.0/bencher/example/example_floats2D.py +0 -93
  51. holobench-1.19.0/bencher/example/example_holosweep.py +0 -98
  52. holobench-1.19.0/bencher/example/example_holosweep_objects.py +0 -111
  53. holobench-1.19.0/bencher/example/example_holosweep_tap.py +0 -144
  54. holobench-1.19.0/bencher/example/example_image.py +0 -138
  55. holobench-1.19.0/bencher/example/example_levels.py +0 -181
  56. holobench-1.19.0/bencher/example/example_pareto.py +0 -53
  57. holobench-1.19.0/bencher/example/example_sample_cache.py +0 -85
  58. holobench-1.19.0/bencher/example/example_sample_cache_context.py +0 -116
  59. holobench-1.19.0/bencher/example/example_simple.py +0 -134
  60. holobench-1.19.0/bencher/example/example_simple_bool.py +0 -35
  61. holobench-1.19.0/bencher/example/example_simple_cat.py +0 -48
  62. holobench-1.19.0/bencher/example/example_simple_float.py +0 -38
  63. holobench-1.19.0/bencher/example/example_strings.py +0 -46
  64. holobench-1.19.0/bencher/example/example_time_event.py +0 -63
  65. holobench-1.19.0/bencher/example/example_video.py +0 -127
  66. holobench-1.19.0/bencher/example/example_workflow.py +0 -189
  67. holobench-1.19.0/bencher/example/experimental/example_bokeh_plotly.py +0 -38
  68. holobench-1.19.0/bencher/example/experimental/example_hover_ex.py +0 -45
  69. holobench-1.19.0/bencher/example/experimental/example_hvplot_explorer.py +0 -39
  70. holobench-1.19.0/bencher/example/experimental/example_interactive.py +0 -75
  71. holobench-1.19.0/bencher/example/experimental/example_streamnd.py +0 -49
  72. holobench-1.19.0/bencher/example/experimental/example_streams.py +0 -36
  73. holobench-1.19.0/bencher/example/experimental/example_template.py +0 -40
  74. holobench-1.19.0/bencher/example/experimental/example_updates.py +0 -84
  75. holobench-1.19.0/bencher/example/experimental/example_vector.py +0 -84
  76. holobench-1.19.0/bencher/example/meta/example_meta.py +0 -171
  77. holobench-1.19.0/bencher/example/meta/example_meta_cat.py +0 -25
  78. holobench-1.19.0/bencher/example/meta/example_meta_float.py +0 -23
  79. holobench-1.19.0/bencher/example/meta/example_meta_levels.py +0 -26
  80. holobench-1.19.0/bencher/example/optuna/example_optuna.py +0 -78
  81. holobench-1.19.0/bencher/example/shelved/example_float2D_scatter.py +0 -109
  82. holobench-1.19.0/bencher/example/shelved/example_float3D_cone.py +0 -96
  83. holobench-1.19.0/bencher/example/shelved/example_kwargs.py +0 -63
  84. holobench-1.19.0/bencher/plotting/__init__.py +0 -0
  85. holobench-1.19.0/bencher/plotting/plot_filter.py +0 -110
  86. holobench-1.19.0/bencher/plotting/plt_cnt_cfg.py +0 -74
  87. holobench-1.19.0/bencher/results/__init__.py +0 -0
  88. holobench-1.19.0/bencher/results/bench_result.py +0 -94
  89. holobench-1.19.0/bencher/results/bench_result_base.py +0 -475
  90. holobench-1.19.0/bencher/results/composable_container/__init__.py +0 -0
  91. holobench-1.19.0/bencher/results/composable_container/composable_container_base.py +0 -60
  92. holobench-1.19.0/bencher/results/composable_container/composable_container_panel.py +0 -40
  93. holobench-1.19.0/bencher/results/composable_container/composable_container_video.py +0 -76
  94. holobench-1.19.0/bencher/results/float_formatter.py +0 -44
  95. holobench-1.19.0/bencher/results/holoview_result.py +0 -745
  96. holobench-1.19.0/bencher/results/optuna_result.py +0 -354
  97. holobench-1.19.0/bencher/results/panel_result.py +0 -41
  98. holobench-1.19.0/bencher/results/plotly_result.py +0 -65
  99. holobench-1.19.0/bencher/results/video_result.py +0 -38
  100. holobench-1.19.0/bencher/results/video_summary.py +0 -217
  101. holobench-1.19.0/bencher/variables/inputs.py +0 -179
  102. holobench-1.19.0/bencher/variables/parametrised_sweep.py +0 -208
  103. holobench-1.19.0/bencher/variables/results.py +0 -214
  104. holobench-1.19.0/bencher/variables/sweep_base.py +0 -162
  105. holobench-1.19.0/bencher/variables/time.py +0 -70
  106. holobench-1.19.0/bencher/video_writer.py +0 -121
  107. holobench-1.19.0/pyproject.toml +0 -106
  108. {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_cfg.py +0 -0
  109. {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_plot_server.py +0 -0
  110. {holobench-1.19.0 → holobench-1.23.0}/bencher/bench_report.py +0 -0
  111. {holobench-1.19.0 → holobench-1.23.0}/bencher/caching.py +0 -0
  112. {holobench-1.19.0 → holobench-1.23.0}/bencher/class_enum.py +0 -0
  113. {holobench-1.19.0 → holobench-1.23.0}/bencher/job.py +0 -0
  114. {holobench-1.19.0 → holobench-1.23.0}/bencher/worker_job.py +0 -0
  115. /holobench-1.19.0/bencher/example/__init__.py → /holobench-1.23.0/resource/bencher +0 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Dyson AI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -1,37 +1,40 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: holobench
3
- Version: 1.19.0
3
+ Version: 1.23.0
4
4
  Summary: A package for benchmarking the performance of arbitrary functions
5
5
  Author-email: Austin Gregg-Smith <blooop@gmail.com>
6
- Description-Content-Type: text/markdown
7
- Requires-Dist: holoviews>=1.15,<=1.18.3
8
- Requires-Dist: numpy>=1.0,<=1.26.4
9
- Requires-Dist: param>=1.13.0,<=2.1.0
10
- Requires-Dist: hvplot>=0.8,<=0.10.0
11
- Requires-Dist: matplotlib>=3.6.3,<=3.8.4
12
- Requires-Dist: panel>=1.3.6,<=1.4.2
13
- Requires-Dist: diskcache>=5.6,<=5.6.3
14
- Requires-Dist: optuna>=3.2,<=3.6.1
15
- Requires-Dist: xarray>=2023.7,<=2024.5.0
16
- Requires-Dist: plotly>=5.15,<=5.22.0
17
- Requires-Dist: sortedcontainers>=2.4,<=2.4
18
- Requires-Dist: pandas>=2.0,<=2.2.2
19
- Requires-Dist: strenum>=0.4.0,<=0.4.15
20
- Requires-Dist: scikit-learn>=1.2,<=1.4.2
21
- Requires-Dist: str2bool>=1.1,<=1.1
22
- Requires-Dist: scoop>=0.7.0,<=0.7.2.0
23
- Requires-Dist: moviepy>=1.0.3,<=1.0.3
24
- Requires-Dist: black>=23,<=24.4.2 ; extra == "test"
25
- Requires-Dist: pylint>=2.16,<=3.1.0 ; extra == "test"
26
- Requires-Dist: pytest-cov>=4.1,<=5.0.0 ; extra == "test"
27
- Requires-Dist: pytest>=7.4,<=8.2.0 ; extra == "test"
28
- Requires-Dist: hypothesis>=6.82,<=6.101.0 ; extra == "test"
29
- Requires-Dist: ruff>=0.0.280,<=0.4.4 ; extra == "test"
30
- Requires-Dist: coverage>=7.2.7,<=7.5.1 ; extra == "test"
31
- Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
32
- Project-URL: Home, https://github.com/dyson-ai/bencher
6
+ Maintainer: austin.gregg-smith
7
+ Maintainer-email: austin.gregg-smith@dyson.com
33
8
  Project-URL: Repository, https://github.com/dyson-ai/bencher
9
+ Project-URL: Home, https://github.com/dyson-ai/bencher
10
+ Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: holoviews<=1.18.3,>=1.15
14
+ Requires-Dist: numpy<=1.26.4,>=1.0
15
+ Requires-Dist: param<=2.1.0,>=1.13.0
16
+ Requires-Dist: hvplot<=0.10.0,>=0.8
17
+ Requires-Dist: matplotlib<=3.9.0,>=3.6.3
18
+ Requires-Dist: panel<=1.4.4,>=1.3.6
19
+ Requires-Dist: diskcache<=5.6.3,>=5.6
20
+ Requires-Dist: optuna<=3.6.1,>=3.2
21
+ Requires-Dist: xarray<=2024.5.0,>=2023.7
22
+ Requires-Dist: plotly<=5.22.0,>=5.15
23
+ Requires-Dist: sortedcontainers<=2.4,>=2.4
24
+ Requires-Dist: pandas<=2.2.2,>=2.0
25
+ Requires-Dist: strenum<=0.4.15,>=0.4.0
26
+ Requires-Dist: scikit-learn<=1.5.0,>=1.2
27
+ Requires-Dist: str2bool<=1.1,>=1.1
28
+ Requires-Dist: scoop<=0.7.2.0,>=0.7.0
29
+ Requires-Dist: moviepy<=1.0.3,>=1.0.3
34
30
  Provides-Extra: test
31
+ Requires-Dist: black<=24.4.2,>=23; extra == "test"
32
+ Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
33
+ Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
34
+ Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
35
+ Requires-Dist: hypothesis<=6.103.0,>=6.82; extra == "test"
36
+ Requires-Dist: ruff<=0.4.7,>=0.3; extra == "test"
37
+ Requires-Dist: coverage<=7.5.3,>=7.2.7; extra == "test"
35
38
 
36
39
  # Bencher
37
40
 
@@ -45,7 +48,14 @@ Provides-Extra: test
45
48
  [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
46
49
  [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
47
50
  [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
48
- [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-310/)
51
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
52
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
53
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
54
+
55
+ ## Badges
56
+
57
+ ### Pixi Badge
58
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
49
59
 
50
60
 
51
61
  ## Intro
@@ -10,7 +10,14 @@
10
10
  [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
11
11
  [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
12
12
  [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
13
- [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-310/)
13
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
14
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
15
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
16
+
17
+ ## Badges
18
+
19
+ ### Pixi Badge
20
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
14
21
 
15
22
 
16
23
  ## Intro
@@ -47,4 +54,4 @@ Bencher is designed to work with stochastic pure functions with no side effects.
47
54
 
48
55
  ### Example Output
49
56
 
50
- https://dyson-ai.github.io/bencher/
57
+ https://dyson-ai.github.io/bencher/
@@ -22,6 +22,15 @@ from .variables.results import (
22
22
  curve,
23
23
  )
24
24
 
25
+ from .results.composable_container.composable_container_base import (
26
+ ComposeType,
27
+ ComposableContainerBase,
28
+ )
29
+ from .results.composable_container.composable_container_video import (
30
+ ComposableContainerVideo,
31
+ RenderCfg,
32
+ )
33
+
25
34
  from .plotting.plot_filter import VarRange, PlotFilter
26
35
  from .utils import (
27
36
  hmap_canonical_input,
@@ -31,6 +40,7 @@ from .utils import (
31
40
  gen_image_path,
32
41
  gen_video_path,
33
42
  lerp,
43
+ tabs_in_markdown,
34
44
  )
35
45
  from .variables.parametrised_sweep import ParametrizedSweep
36
46
  from .caching import CachedParams
@@ -121,7 +121,7 @@ class BenchRunner:
121
121
  self.show_publish(report_level, show, publish, save, debug)
122
122
  return self.results
123
123
 
124
- def show_publish(self, report, show, publish, save, debug):
124
+ def show_publish(self, report: BenchReport, show: bool, publish: bool, save: bool, debug: bool):
125
125
  if save:
126
126
  report.save_index()
127
127
  if publish and self.publisher is not None:
@@ -220,9 +220,11 @@ class Bench(BenchPlotServer):
220
220
  relationship_cb=None,
221
221
  plot_callbacks: List | bool = None,
222
222
  ) -> List[BenchResult]:
223
- results = []
224
223
  if relationship_cb is None:
225
224
  relationship_cb = combinations
225
+ if input_vars is None:
226
+ input_vars = self.worker_class_instance.get_inputs_only()
227
+ results = []
226
228
  for it in range(iterations):
227
229
  for input_group in relationship_cb(input_vars, group_size):
228
230
  title_gen = title + "Sweeping " + " vs ".join(params_to_str(input_group))
@@ -1,4 +1,5 @@
1
1
  from typing import List
2
+ import logging
2
3
 
3
4
  import optuna
4
5
  import panel as pn
@@ -158,8 +159,8 @@ def summarise_optuna_study(study: optuna.study.Study) -> pn.pane.panel:
158
159
  row.append(plot_param_importances(study))
159
160
  try:
160
161
  row.append(plot_pareto_front(study))
161
- except Exception:
162
- pass
162
+ except Exception as e: # pylint: disable=broad-except
163
+ logging.exception(e)
163
164
 
164
165
  row.append(
165
166
  pn.pane.Markdown(f"```\nBest value: {study.best_value}\nParams: {study.best_params}```")
@@ -8,8 +8,9 @@ from colorsys import hsv_to_rgb
8
8
  from pathlib import Path
9
9
  from uuid import uuid4
10
10
  from functools import partial
11
- from typing import Callable, Any, List
11
+ from typing import Callable, Any, List, Tuple
12
12
  import param
13
+ import numpy as np
13
14
 
14
15
 
15
16
  def hmap_canonical_input(dic: dict) -> tuple:
@@ -98,6 +99,23 @@ def un_camel(camel: str) -> str:
98
99
  return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
99
100
 
100
101
 
102
+ def mult_tuple(inp: Tuple[float], val: float) -> Tuple[float]:
103
+ return tuple(np.array(inp) * val)
104
+
105
+
106
+ def tabs_in_markdown(regular_str: str, spaces: int = 2) -> str:
107
+ """Given a string with tabs in the form \t convert the to &ensp; which is a double space in markdown
108
+
109
+ Args:
110
+ regular_str (str): A string with tabs in it
111
+ spaces (int): the number of spaces per tab
112
+
113
+ Returns:
114
+ str: A string with sets of &nbsp; to represent the tabs in markdown
115
+ """
116
+ return regular_str.replace("\t", "".join(["&nbsp;"] * spaces))
117
+
118
+
101
119
  def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
102
120
  """Uses the golden angle to generate colors programmatically with minimum overlap between colors.
103
121
  https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
@@ -129,13 +147,21 @@ def color_tuple_to_css(color: tuple[float, float, float]) -> str:
129
147
  return f"rgb{(color[0] * 255, color[1] * 255, color[2] * 255)}"
130
148
 
131
149
 
150
+ def color_tuple_to_255(color: tuple[float, float, float]) -> tuple[float, float, float]:
151
+ return (
152
+ min(int(color[0] * 255), 255),
153
+ min(int(color[1] * 255), 255),
154
+ min(int(color[2] * 255), 255),
155
+ )
156
+
157
+
132
158
  def gen_path(filename, folder="generic", suffix=".dat"):
133
159
  path = Path(f"cachedir/{folder}/{filename}/")
134
160
  path.mkdir(parents=True, exist_ok=True)
135
161
  return f"{path.absolute().as_posix()}/{filename}_{uuid4()}{suffix}"
136
162
 
137
163
 
138
- def gen_video_path(video_name: str = "vid", extension: str = ".webm") -> str:
164
+ def gen_video_path(video_name: str = "vid", extension: str = ".mp4") -> str:
139
165
  return gen_path(video_name, "vid", extension)
140
166
 
141
167
 
@@ -0,0 +1,67 @@
1
+ import numpy as np
2
+ import moviepy.video.io.ImageSequenceClip
3
+ from pathlib import Path
4
+ from .utils import gen_video_path, gen_image_path
5
+
6
+ import moviepy
7
+ from PIL import Image, ImageDraw
8
+
9
+
10
+ class VideoWriter:
11
+ def __init__(self, filename: str = "vid") -> None:
12
+ self.images = []
13
+ self.image_files = []
14
+ self.video_files = []
15
+ self.filename = gen_video_path(filename)
16
+
17
+ def append(self, img):
18
+ self.images.append(img)
19
+
20
+ def write(self) -> str:
21
+ if len(self.images) > 0:
22
+ clip = moviepy.video.io.ImageSequenceClip.ImageSequenceClip(
23
+ self.images, fps=30, with_mask=False, load_images=True
24
+ )
25
+ self.write_video_raw(clip)
26
+ return self.filename
27
+
28
+ @staticmethod
29
+ def create_label(label, width=None, height=16, color=(255, 255, 255)):
30
+ if width is None:
31
+ width = len(label) * 10
32
+ new_img = Image.new("RGB", (width, height), color=color)
33
+ # ImageDraw.Draw(new_img).text((width/2, 0), label, (0, 0, 0),align="center",achor="ms")
34
+ ImageDraw.Draw(new_img).text(
35
+ (width / 2.0, 0), label, (0, 0, 0), anchor="mt", font_size=height
36
+ )
37
+
38
+ return new_img
39
+
40
+ @staticmethod
41
+ def label_image(path: Path, label, padding=20, color=(255, 255, 255)) -> Path:
42
+ image = Image.open(path)
43
+ new_img = VideoWriter.create_label(
44
+ label, image.size[0], image.size[1] + padding, color=color
45
+ )
46
+ new_img.paste(image, (0, padding))
47
+ return new_img
48
+
49
+ def write_video_raw(self, video_clip: moviepy.video.VideoClip, fps: int = 30) -> str:
50
+ video_clip.write_videofile(
51
+ self.filename,
52
+ codec="libx264",
53
+ audio=False,
54
+ bitrate="0",
55
+ fps=fps,
56
+ ffmpeg_params=["-crf", "23"],
57
+ threads=8,
58
+ )
59
+ video_clip.close()
60
+ return self.filename
61
+
62
+
63
+ def add_image(np_array: np.ndarray, name: str = "img") -> str:
64
+ """Creates a file on disk from a numpy array and returns the created image path"""
65
+ filename = gen_image_path(name)
66
+ Image.fromarray(np_array).save(filename)
67
+ return filename
@@ -0,0 +1,95 @@
1
+ Metadata-Version: 2.1
2
+ Name: holobench
3
+ Version: 1.23.0
4
+ Summary: A package for benchmarking the performance of arbitrary functions
5
+ Author-email: Austin Gregg-Smith <blooop@gmail.com>
6
+ Maintainer: austin.gregg-smith
7
+ Maintainer-email: austin.gregg-smith@dyson.com
8
+ Project-URL: Repository, https://github.com/dyson-ai/bencher
9
+ Project-URL: Home, https://github.com/dyson-ai/bencher
10
+ Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: holoviews<=1.18.3,>=1.15
14
+ Requires-Dist: numpy<=1.26.4,>=1.0
15
+ Requires-Dist: param<=2.1.0,>=1.13.0
16
+ Requires-Dist: hvplot<=0.10.0,>=0.8
17
+ Requires-Dist: matplotlib<=3.9.0,>=3.6.3
18
+ Requires-Dist: panel<=1.4.4,>=1.3.6
19
+ Requires-Dist: diskcache<=5.6.3,>=5.6
20
+ Requires-Dist: optuna<=3.6.1,>=3.2
21
+ Requires-Dist: xarray<=2024.5.0,>=2023.7
22
+ Requires-Dist: plotly<=5.22.0,>=5.15
23
+ Requires-Dist: sortedcontainers<=2.4,>=2.4
24
+ Requires-Dist: pandas<=2.2.2,>=2.0
25
+ Requires-Dist: strenum<=0.4.15,>=0.4.0
26
+ Requires-Dist: scikit-learn<=1.5.0,>=1.2
27
+ Requires-Dist: str2bool<=1.1,>=1.1
28
+ Requires-Dist: scoop<=0.7.2.0,>=0.7.0
29
+ Requires-Dist: moviepy<=1.0.3,>=1.0.3
30
+ Provides-Extra: test
31
+ Requires-Dist: black<=24.4.2,>=23; extra == "test"
32
+ Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
33
+ Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
34
+ Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
35
+ Requires-Dist: hypothesis<=6.103.0,>=6.82; extra == "test"
36
+ Requires-Dist: ruff<=0.4.7,>=0.3; extra == "test"
37
+ Requires-Dist: coverage<=7.5.3,>=7.2.7; extra == "test"
38
+
39
+ # Bencher
40
+
41
+ ## Continuous Integration Status
42
+
43
+ [![Ci](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml?query=branch%3Amain)
44
+ ![Read the Docs](https://img.shields.io/readthedocs/bencher)
45
+ [![Codecov](https://codecov.io/gh/dyson-ai/bencher/branch/main/graph/badge.svg?token=Y212GW1PG6)](https://codecov.io/gh/dyson-ai/bencher)
46
+ [![GitHub issues](https://img.shields.io/github/issues/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/issues/)
47
+ [![GitHub pull-requests merged](https://badgen.net/github/merged-prs/dyson-ai/bencher)](https://github.com/dyson-ai/bencher/pulls?q=is%3Amerged)
48
+ [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
49
+ [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
50
+ [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
51
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
52
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
53
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
54
+
55
+ ## Badges
56
+
57
+ ### Pixi Badge
58
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
59
+
60
+
61
+ ## Intro
62
+
63
+ Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics. It calculates the [cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) of a set of variables
64
+
65
+ Parameters for bencher are defined using the [param](https://param.holoviz.org/) library as a config class with extra metadata that describes the bounds of the search space you want to measure. You must define a benchmarking function that accepts an instance of the config class and return a dictionary with string metric names and float values.
66
+
67
+ Parameters are benchmarked by passing in a list N parameters, and an N-Dimensional tensor is returned. You can optionally sample each point multiple times to get back a distribution and also track its value over time. By default the data will be plotted automatically based on the types of parameters you are sampling (e.g, continous, discrete), but you can also pass in a callback to customize plotting.
68
+
69
+ The data is stored in a persistent database so that past performance is tracked.
70
+
71
+ ## Assumptions
72
+
73
+ The input types should also be of one of the basic datatypes (bool, int, float, str, enum, datetime) so that the data can be easily hashed, cached and stored in the database and processed with seaborn and xarray plotting functions. You can use class inheritance to define hierarchical parameter configuration class types that can be reused in a bigger configuration classes.
74
+
75
+ Bencher is designed to work with stochastic pure functions with no side effects. It assumes that when the objective function is given the same inputs, it will return the same output +- random noise. This is because the function must be called multiple times to get a good statistical distribution of it and so each call must not be influenced by anything or the results will be corrupted.
76
+
77
+ ### Pseudocode of bencher
78
+
79
+ Enumerate a list of all input parameter combinations
80
+ for each set of input parameters:
81
+ pass the inputs to the objective function and store results in the N-D array
82
+
83
+ get unique hash for the set of inputs parameters
84
+ look up previous results for that hash
85
+ if it exists:
86
+ load historical data
87
+ combine latest data with historical data
88
+
89
+ store the results using the input hash as a key
90
+ deduce the type of plot based on the input types
91
+ return data and plot
92
+
93
+ ### Example Output
94
+
95
+ https://dyson-ai.github.io/bencher/
@@ -0,0 +1,47 @@
1
+ LICENSE
2
+ README.md
3
+ package.xml
4
+ pyproject.toml
5
+ setup.cfg
6
+ setup.py
7
+ bencher/__init__.py
8
+ bencher/bench_cfg.py
9
+ bencher/bench_plot_server.py
10
+ bencher/bench_report.py
11
+ bencher/bench_runner.py
12
+ bencher/bencher.py
13
+ bencher/caching.py
14
+ bencher/class_enum.py
15
+ bencher/job.py
16
+ bencher/optuna_conversions.py
17
+ bencher/utils.py
18
+ bencher/video_writer.py
19
+ bencher/worker_job.py
20
+ holobench.egg-info/PKG-INFO
21
+ holobench.egg-info/SOURCES.txt
22
+ holobench.egg-info/dependency_links.txt
23
+ holobench.egg-info/not-zip-safe
24
+ holobench.egg-info/requires.txt
25
+ holobench.egg-info/top_level.txt
26
+ resource/bencher
27
+ test/test_bench_examples.py
28
+ test/test_bench_report.py
29
+ test/test_bench_result_base.py
30
+ test/test_bench_runner.py
31
+ test/test_bencher.py
32
+ test/test_cache.py
33
+ test/test_class_enum.py
34
+ test/test_combinations.py
35
+ test/test_composable_container_base.py
36
+ test/test_composable_container_video.py
37
+ test/test_float_formatter.py
38
+ test/test_job.py
39
+ test/test_meta_tests.py
40
+ test/test_parametrized_sweep.py
41
+ test/test_plot_filter.py
42
+ test/test_plot_server.py
43
+ test/test_sample_cache.py
44
+ test/test_sweep_base.py
45
+ test/test_sweep_vars.py
46
+ test/test_utils.py
47
+ test/test_vars.py
@@ -0,0 +1,26 @@
1
+ holoviews<=1.18.3,>=1.15
2
+ numpy<=1.26.4,>=1.0
3
+ param<=2.1.0,>=1.13.0
4
+ hvplot<=0.10.0,>=0.8
5
+ matplotlib<=3.9.0,>=3.6.3
6
+ panel<=1.4.4,>=1.3.6
7
+ diskcache<=5.6.3,>=5.6
8
+ optuna<=3.6.1,>=3.2
9
+ xarray<=2024.5.0,>=2023.7
10
+ plotly<=5.22.0,>=5.15
11
+ sortedcontainers<=2.4,>=2.4
12
+ pandas<=2.2.2,>=2.0
13
+ strenum<=0.4.15,>=0.4.0
14
+ scikit-learn<=1.5.0,>=1.2
15
+ str2bool<=1.1,>=1.1
16
+ scoop<=0.7.2.0,>=0.7.0
17
+ moviepy<=1.0.3,>=1.0.3
18
+
19
+ [test]
20
+ black<=24.4.2,>=23
21
+ pylint<=3.2.2,>=2.17.7
22
+ pytest-cov<=5.0.0,>=4.1
23
+ pytest<=8.2.1,>=7.4
24
+ hypothesis<=6.103.0,>=6.82
25
+ ruff<=0.4.7,>=0.3
26
+ coverage<=7.5.3,>=7.2.7
@@ -0,0 +1 @@
1
+ bencher
@@ -0,0 +1,33 @@
1
+ <?xml version="1.0"?>
2
+ <package format="3">
3
+ <name>bencher</name>
4
+ <version>0.1.0</version>
5
+ <description>A package for benchmarking the performance of arbitrary functions</description>
6
+ <maintainer email="austin.gregg-smith@dyson.com">Austin Gregg-Smith</maintainer>
7
+ <license>MIT</license>
8
+
9
+ <depend>python3-diskcache</depend>
10
+
11
+ <depend>python3-pandas</depend>
12
+ <depend>python3-seaborn</depend>
13
+ <depend>python3-matplotlib</depend>
14
+ <depend>python3-numpy</depend>
15
+ <depend>python3-pytest</depend>
16
+ <depend>python3-hypothesis</depend>
17
+ <depend>xarray</depend>
18
+ <depend>python3-zarr</depend>
19
+ <depend>python3-param</depend>
20
+ <depend>python3-panel</depend>
21
+ <depend>python3-hvplot</depend>
22
+ <depend>python3-optuna</depend>
23
+ <depend>python3-plotly</depend>
24
+
25
+ <!-- TO REMOVE WHEN WE UPGRADE TO PYTHON 3.11, This is in the standard library for python>=3.11 -->
26
+ <depend>strenum</depend>
27
+
28
+ <test_depend>python3-pytest-cov</test_depend>
29
+
30
+ <export>
31
+ <build_type>ament_python</build_type>
32
+ </export>
33
+ </package>