holobench 1.18.0__tar.gz → 1.22.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. holobench-1.22.2/LICENSE +21 -0
  2. {holobench-1.18.0 → holobench-1.22.2}/PKG-INFO +37 -29
  3. {holobench-1.18.0 → holobench-1.22.2}/README.md +9 -2
  4. {holobench-1.18.0 → holobench-1.22.2}/bencher/__init__.py +2 -0
  5. {holobench-1.18.0 → holobench-1.22.2}/bencher/bench_cfg.py +1 -1
  6. {holobench-1.18.0 → holobench-1.22.2}/bencher/bencher.py +14 -5
  7. {holobench-1.18.0 → holobench-1.22.2}/bencher/utils.py +14 -1
  8. holobench-1.22.2/holobench.egg-info/PKG-INFO +93 -0
  9. holobench-1.22.2/holobench.egg-info/SOURCES.txt +45 -0
  10. holobench-1.22.2/holobench.egg-info/dependency_links.txt +1 -0
  11. holobench-1.22.2/holobench.egg-info/not-zip-safe +1 -0
  12. holobench-1.22.2/holobench.egg-info/requires.txt +26 -0
  13. holobench-1.22.2/holobench.egg-info/top_level.txt +1 -0
  14. holobench-1.22.2/package.xml +33 -0
  15. {holobench-1.18.0 → holobench-1.22.2}/pyproject.toml +63 -33
  16. holobench-1.22.2/setup.cfg +10 -0
  17. holobench-1.22.2/setup.py +21 -0
  18. holobench-1.22.2/test/test_bench_examples.py +129 -0
  19. holobench-1.22.2/test/test_bench_report.py +55 -0
  20. holobench-1.22.2/test/test_bench_result_base.py +79 -0
  21. holobench-1.22.2/test/test_bench_runner.py +196 -0
  22. holobench-1.22.2/test/test_bencher.py +383 -0
  23. holobench-1.22.2/test/test_cache.py +46 -0
  24. holobench-1.22.2/test/test_class_enum.py +10 -0
  25. holobench-1.22.2/test/test_combinations.py +182 -0
  26. holobench-1.22.2/test/test_float_formatter.py +26 -0
  27. holobench-1.22.2/test/test_job.py +98 -0
  28. holobench-1.22.2/test/test_meta_tests.py +55 -0
  29. holobench-1.22.2/test/test_parametrized_sweep.py +74 -0
  30. holobench-1.22.2/test/test_plot_filter.py +91 -0
  31. holobench-1.22.2/test/test_plot_server.py +55 -0
  32. holobench-1.22.2/test/test_sample_cache.py +113 -0
  33. holobench-1.22.2/test/test_sweep_base.py +198 -0
  34. holobench-1.22.2/test/test_sweep_vars.py +176 -0
  35. holobench-1.22.2/test/test_utils.py +172 -0
  36. holobench-1.22.2/test/test_vars.py +69 -0
  37. holobench-1.18.0/bencher/example/benchmark_data.py +0 -200
  38. holobench-1.18.0/bencher/example/example_all.py +0 -45
  39. holobench-1.18.0/bencher/example/example_categorical.py +0 -99
  40. holobench-1.18.0/bencher/example/example_custom_sweep.py +0 -59
  41. holobench-1.18.0/bencher/example/example_docs.py +0 -34
  42. holobench-1.18.0/bencher/example/example_float3D.py +0 -101
  43. holobench-1.18.0/bencher/example/example_float_cat.py +0 -99
  44. holobench-1.18.0/bencher/example/example_floats.py +0 -89
  45. holobench-1.18.0/bencher/example/example_floats2D.py +0 -93
  46. holobench-1.18.0/bencher/example/example_holosweep.py +0 -98
  47. holobench-1.18.0/bencher/example/example_holosweep_objects.py +0 -111
  48. holobench-1.18.0/bencher/example/example_holosweep_tap.py +0 -144
  49. holobench-1.18.0/bencher/example/example_image.py +0 -138
  50. holobench-1.18.0/bencher/example/example_levels.py +0 -181
  51. holobench-1.18.0/bencher/example/example_pareto.py +0 -53
  52. holobench-1.18.0/bencher/example/example_sample_cache.py +0 -85
  53. holobench-1.18.0/bencher/example/example_sample_cache_context.py +0 -116
  54. holobench-1.18.0/bencher/example/example_simple.py +0 -134
  55. holobench-1.18.0/bencher/example/example_simple_bool.py +0 -35
  56. holobench-1.18.0/bencher/example/example_simple_cat.py +0 -48
  57. holobench-1.18.0/bencher/example/example_simple_float.py +0 -38
  58. holobench-1.18.0/bencher/example/example_strings.py +0 -46
  59. holobench-1.18.0/bencher/example/example_time_event.py +0 -63
  60. holobench-1.18.0/bencher/example/example_video.py +0 -127
  61. holobench-1.18.0/bencher/example/example_workflow.py +0 -189
  62. holobench-1.18.0/bencher/example/experimental/example_bokeh_plotly.py +0 -38
  63. holobench-1.18.0/bencher/example/experimental/example_hover_ex.py +0 -45
  64. holobench-1.18.0/bencher/example/experimental/example_hvplot_explorer.py +0 -39
  65. holobench-1.18.0/bencher/example/experimental/example_interactive.py +0 -75
  66. holobench-1.18.0/bencher/example/experimental/example_streamnd.py +0 -49
  67. holobench-1.18.0/bencher/example/experimental/example_streams.py +0 -36
  68. holobench-1.18.0/bencher/example/experimental/example_template.py +0 -40
  69. holobench-1.18.0/bencher/example/experimental/example_updates.py +0 -84
  70. holobench-1.18.0/bencher/example/experimental/example_vector.py +0 -84
  71. holobench-1.18.0/bencher/example/meta/example_meta.py +0 -171
  72. holobench-1.18.0/bencher/example/meta/example_meta_cat.py +0 -25
  73. holobench-1.18.0/bencher/example/meta/example_meta_float.py +0 -23
  74. holobench-1.18.0/bencher/example/meta/example_meta_levels.py +0 -26
  75. holobench-1.18.0/bencher/example/optuna/example_optuna.py +0 -78
  76. holobench-1.18.0/bencher/example/shelved/example_float2D_scatter.py +0 -109
  77. holobench-1.18.0/bencher/example/shelved/example_float3D_cone.py +0 -96
  78. holobench-1.18.0/bencher/example/shelved/example_kwargs.py +0 -63
  79. holobench-1.18.0/bencher/plotting/__init__.py +0 -0
  80. holobench-1.18.0/bencher/plotting/plot_filter.py +0 -110
  81. holobench-1.18.0/bencher/plotting/plt_cnt_cfg.py +0 -74
  82. holobench-1.18.0/bencher/results/__init__.py +0 -0
  83. holobench-1.18.0/bencher/results/bench_result.py +0 -94
  84. holobench-1.18.0/bencher/results/bench_result_base.py +0 -468
  85. holobench-1.18.0/bencher/results/composable_container/__init__.py +0 -0
  86. holobench-1.18.0/bencher/results/composable_container/composable_container_base.py +0 -60
  87. holobench-1.18.0/bencher/results/composable_container/composable_container_panel.py +0 -40
  88. holobench-1.18.0/bencher/results/composable_container/composable_container_video.py +0 -76
  89. holobench-1.18.0/bencher/results/float_formatter.py +0 -44
  90. holobench-1.18.0/bencher/results/holoview_result.py +0 -745
  91. holobench-1.18.0/bencher/results/optuna_result.py +0 -354
  92. holobench-1.18.0/bencher/results/panel_result.py +0 -43
  93. holobench-1.18.0/bencher/results/plotly_result.py +0 -65
  94. holobench-1.18.0/bencher/results/video_result.py +0 -38
  95. holobench-1.18.0/bencher/results/video_summary.py +0 -217
  96. holobench-1.18.0/bencher/variables/inputs.py +0 -179
  97. holobench-1.18.0/bencher/variables/parametrised_sweep.py +0 -226
  98. holobench-1.18.0/bencher/variables/results.py +0 -184
  99. holobench-1.18.0/bencher/variables/sweep_base.py +0 -162
  100. holobench-1.18.0/bencher/variables/time.py +0 -70
  101. {holobench-1.18.0 → holobench-1.22.2}/bencher/bench_plot_server.py +0 -0
  102. {holobench-1.18.0 → holobench-1.22.2}/bencher/bench_report.py +0 -0
  103. {holobench-1.18.0 → holobench-1.22.2}/bencher/bench_runner.py +0 -0
  104. {holobench-1.18.0 → holobench-1.22.2}/bencher/caching.py +0 -0
  105. {holobench-1.18.0 → holobench-1.22.2}/bencher/class_enum.py +0 -0
  106. {holobench-1.18.0 → holobench-1.22.2}/bencher/job.py +0 -0
  107. {holobench-1.18.0 → holobench-1.22.2}/bencher/optuna_conversions.py +0 -0
  108. {holobench-1.18.0 → holobench-1.22.2}/bencher/video_writer.py +0 -0
  109. {holobench-1.18.0 → holobench-1.22.2}/bencher/worker_job.py +0 -0
  110. /holobench-1.18.0/bencher/example/__init__.py → /holobench-1.22.2/resource/bencher +0 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Dyson AI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -1,37 +1,38 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: holobench
3
- Version: 1.18.0
3
+ Version: 1.22.2
4
4
  Summary: A package for benchmarking the performance of arbitrary functions
5
5
  Author-email: Austin Gregg-Smith <blooop@gmail.com>
6
- Description-Content-Type: text/markdown
7
- Requires-Dist: holoviews>=1.15,<=1.18.3
8
- Requires-Dist: numpy>=1.0,<=1.26.4
9
- Requires-Dist: param>=1.13.0,<=2.1.0
10
- Requires-Dist: hvplot>=0.8,<=0.10.0
11
- Requires-Dist: matplotlib>=3.6.3,<=3.8.4
12
- Requires-Dist: panel>=1.3.6,<=1.4.2
13
- Requires-Dist: diskcache>=5.6,<=5.6.3
14
- Requires-Dist: optuna>=3.2,<=3.6.1
15
- Requires-Dist: xarray>=2023.7,<=2024.5.0
16
- Requires-Dist: plotly>=5.15,<=5.22.0
17
- Requires-Dist: sortedcontainers>=2.4,<=2.4
18
- Requires-Dist: pandas>=2.0,<=2.2.2
19
- Requires-Dist: strenum>=0.4.0,<=0.4.15
20
- Requires-Dist: scikit-learn>=1.2,<=1.4.2
21
- Requires-Dist: str2bool>=1.1,<=1.1
22
- Requires-Dist: scoop>=0.7.0,<=0.7.2.0
23
- Requires-Dist: moviepy>=1.0.3,<=1.0.3
24
- Requires-Dist: black>=23,<=24.4.2 ; extra == "test"
25
- Requires-Dist: pylint>=2.16,<=3.1.0 ; extra == "test"
26
- Requires-Dist: pytest-cov>=4.1,<=5.0.0 ; extra == "test"
27
- Requires-Dist: pytest>=7.4,<=8.2.0 ; extra == "test"
28
- Requires-Dist: hypothesis>=6.82,<=6.101.0 ; extra == "test"
29
- Requires-Dist: ruff>=0.0.280,<=0.4.4 ; extra == "test"
30
- Requires-Dist: coverage>=7.2.7,<=7.5.1 ; extra == "test"
31
- Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
32
- Project-URL: Home, https://github.com/dyson-ai/bencher
33
6
  Project-URL: Repository, https://github.com/dyson-ai/bencher
7
+ Project-URL: Home, https://github.com/dyson-ai/bencher
8
+ Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: holoviews<=1.18.3,>=1.15
12
+ Requires-Dist: numpy<=1.26.4,>=1.0
13
+ Requires-Dist: param<=2.1.0,>=1.13.0
14
+ Requires-Dist: hvplot<=0.10.0,>=0.8
15
+ Requires-Dist: matplotlib<=3.9.0,>=3.6.3
16
+ Requires-Dist: panel<=1.4.3,>=1.3.6
17
+ Requires-Dist: diskcache<=5.6.3,>=5.6
18
+ Requires-Dist: optuna<=3.6.1,>=3.2
19
+ Requires-Dist: xarray<=2024.5.0,>=2023.7
20
+ Requires-Dist: plotly<=5.22.0,>=5.15
21
+ Requires-Dist: sortedcontainers<=2.4,>=2.4
22
+ Requires-Dist: pandas<=2.2.2,>=2.0
23
+ Requires-Dist: strenum<=0.4.15,>=0.4.0
24
+ Requires-Dist: scikit-learn<=1.5.0,>=1.2
25
+ Requires-Dist: str2bool<=1.1,>=1.1
26
+ Requires-Dist: scoop<=0.7.2.0,>=0.7.0
27
+ Requires-Dist: moviepy<=1.0.3,>=1.0.3
34
28
  Provides-Extra: test
29
+ Requires-Dist: black<=24.4.2,>=23; extra == "test"
30
+ Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
31
+ Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
32
+ Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
33
+ Requires-Dist: hypothesis<=6.102.6,>=6.82; extra == "test"
34
+ Requires-Dist: ruff<=0.4.5,>=0.3; extra == "test"
35
+ Requires-Dist: coverage<=7.5.2,>=7.2.7; extra == "test"
35
36
 
36
37
  # Bencher
37
38
 
@@ -45,7 +46,14 @@ Provides-Extra: test
45
46
  [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
46
47
  [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
47
48
  [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
48
- [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-310/)
49
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
50
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
51
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
52
+
53
+ ## Badges
54
+
55
+ ### Pixi Badge
56
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
49
57
 
50
58
 
51
59
  ## Intro
@@ -10,7 +10,14 @@
10
10
  [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
11
11
  [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
12
12
  [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
13
- [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-310/)
13
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
14
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
15
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
16
+
17
+ ## Badges
18
+
19
+ ### Pixi Badge
20
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
14
21
 
15
22
 
16
23
  ## Intro
@@ -47,4 +54,4 @@ Bencher is designed to work with stochastic pure functions with no side effects.
47
54
 
48
55
  ### Example Output
49
56
 
50
- https://dyson-ai.github.io/bencher/
57
+ https://dyson-ai.github.io/bencher/
@@ -11,6 +11,7 @@ from .variables.results import (
11
11
  ResultVar,
12
12
  ResultVec,
13
13
  ResultHmap,
14
+ ResultPath,
14
15
  ResultVideo,
15
16
  ResultImage,
16
17
  ResultString,
@@ -30,6 +31,7 @@ from .utils import (
30
31
  gen_image_path,
31
32
  gen_video_path,
32
33
  lerp,
34
+ tabs_in_markdown,
33
35
  )
34
36
  from .variables.parametrised_sweep import ParametrizedSweep
35
37
  from .caching import CachedParams
@@ -383,7 +383,7 @@ class BenchCfg(BenchRunCfg):
383
383
  benchmark_sampling_str.append(f" use_cache: {self.use_cache}")
384
384
  benchmark_sampling_str.append(f" use_sample_cache: {self.use_sample_cache}")
385
385
  benchmark_sampling_str.append(f" only_hash_tag: {self.only_hash_tag}")
386
- benchmark_sampling_str.append(f" parallel: {self.executor}")
386
+ benchmark_sampling_str.append(f" executor: {self.executor}")
387
387
 
388
388
  for mv in self.meta_vars:
389
389
  benchmark_sampling_str.extend(describe_variable(mv, True))
@@ -24,6 +24,7 @@ from bencher.variables.results import (
24
24
  ResultVar,
25
25
  ResultVec,
26
26
  ResultHmap,
27
+ ResultPath,
27
28
  ResultVideo,
28
29
  ResultImage,
29
30
  ResultString,
@@ -557,7 +558,7 @@ class Bench(BenchPlotServer):
557
558
  time_src (datetime | str): a representation of the sample time
558
559
 
559
560
  Returns:
560
- _type_: _description_
561
+ tuple[BenchResult, List, List]: bench_result, function intputs, dimension names
561
562
  """
562
563
 
563
564
  if time_src is None:
@@ -565,9 +566,7 @@ class Bench(BenchPlotServer):
565
566
  bench_cfg.meta_vars = self.define_extra_vars(bench_cfg, bench_cfg.repeats, time_src)
566
567
 
567
568
  bench_cfg.all_vars = bench_cfg.input_vars + bench_cfg.meta_vars
568
-
569
569
  # bench_cfg.all_vars = bench_cfg.iv_time + bench_cfg.input_vars +[ bench_cfg.iv_repeat]
570
-
571
570
  # bench_cfg.all_vars = [ bench_cfg.iv_repeat] +bench_cfg.input_vars + bench_cfg.iv_time
572
571
 
573
572
  for i in bench_cfg.all_vars:
@@ -587,7 +586,9 @@ class Bench(BenchPlotServer):
587
586
  if isinstance(rv, ResultReference):
588
587
  result_data = np.full(dims_cfg.dims_size, -1, dtype=int)
589
588
  data_vars[rv.name] = (dims_cfg.dims_name, result_data)
590
- if isinstance(rv, (ResultVideo, ResultImage, ResultString, ResultContainer)):
589
+ if isinstance(
590
+ rv, (ResultPath, ResultVideo, ResultImage, ResultString, ResultContainer)
591
+ ):
591
592
  result_data = np.full(dims_cfg.dims_size, "NAN", dtype=object)
592
593
  data_vars[rv.name] = (dims_cfg.dims_name, result_data)
593
594
  elif type(rv) == ResultVec:
@@ -724,7 +725,15 @@ class Bench(BenchPlotServer):
724
725
  logging.info(f"{rv.name}: {result_value}")
725
726
 
726
727
  if isinstance(
727
- rv, (ResultVar, ResultVideo, ResultImage, ResultString, ResultContainer)
728
+ rv,
729
+ (
730
+ ResultVar,
731
+ ResultVideo,
732
+ ResultImage,
733
+ ResultString,
734
+ ResultContainer,
735
+ ResultPath,
736
+ ),
728
737
  ):
729
738
  set_xarray_multidim(bench_res.ds[rv.name], worker_job.index_tuple, result_value)
730
739
  elif isinstance(rv, ResultReference):
@@ -98,6 +98,19 @@ def un_camel(camel: str) -> str:
98
98
  return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
99
99
 
100
100
 
101
+ def tabs_in_markdown(regular_str: str, spaces: int = 2) -> str:
102
+ """Given a string with tabs in the form \t convert the to &ensp; which is a double space in markdown
103
+
104
+ Args:
105
+ regular_str (str): A string with tabs in it
106
+ spaces (int): the number of spaces per tab
107
+
108
+ Returns:
109
+ str: A string with sets of &nbsp; to represent the tabs in markdown
110
+ """
111
+ return regular_str.replace("\t", "".join(["&nbsp;"] * spaces))
112
+
113
+
101
114
  def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
102
115
  """Uses the golden angle to generate colors programmatically with minimum overlap between colors.
103
116
  https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
@@ -129,7 +142,7 @@ def color_tuple_to_css(color: tuple[float, float, float]) -> str:
129
142
  return f"rgb{(color[0] * 255, color[1] * 255, color[2] * 255)}"
130
143
 
131
144
 
132
- def gen_path(filename, folder, suffix):
145
+ def gen_path(filename, folder="generic", suffix=".dat"):
133
146
  path = Path(f"cachedir/{folder}/{filename}/")
134
147
  path.mkdir(parents=True, exist_ok=True)
135
148
  return f"{path.absolute().as_posix()}/{filename}_{uuid4()}{suffix}"
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.1
2
+ Name: holobench
3
+ Version: 1.22.2
4
+ Summary: A package for benchmarking the performance of arbitrary functions
5
+ Author-email: Austin Gregg-Smith <blooop@gmail.com>
6
+ Project-URL: Repository, https://github.com/dyson-ai/bencher
7
+ Project-URL: Home, https://github.com/dyson-ai/bencher
8
+ Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: holoviews<=1.18.3,>=1.15
12
+ Requires-Dist: numpy<=1.26.4,>=1.0
13
+ Requires-Dist: param<=2.1.0,>=1.13.0
14
+ Requires-Dist: hvplot<=0.10.0,>=0.8
15
+ Requires-Dist: matplotlib<=3.9.0,>=3.6.3
16
+ Requires-Dist: panel<=1.4.3,>=1.3.6
17
+ Requires-Dist: diskcache<=5.6.3,>=5.6
18
+ Requires-Dist: optuna<=3.6.1,>=3.2
19
+ Requires-Dist: xarray<=2024.5.0,>=2023.7
20
+ Requires-Dist: plotly<=5.22.0,>=5.15
21
+ Requires-Dist: sortedcontainers<=2.4,>=2.4
22
+ Requires-Dist: pandas<=2.2.2,>=2.0
23
+ Requires-Dist: strenum<=0.4.15,>=0.4.0
24
+ Requires-Dist: scikit-learn<=1.5.0,>=1.2
25
+ Requires-Dist: str2bool<=1.1,>=1.1
26
+ Requires-Dist: scoop<=0.7.2.0,>=0.7.0
27
+ Requires-Dist: moviepy<=1.0.3,>=1.0.3
28
+ Provides-Extra: test
29
+ Requires-Dist: black<=24.4.2,>=23; extra == "test"
30
+ Requires-Dist: pylint<=3.2.2,>=2.17.7; extra == "test"
31
+ Requires-Dist: pytest-cov<=5.0.0,>=4.1; extra == "test"
32
+ Requires-Dist: pytest<=8.2.1,>=7.4; extra == "test"
33
+ Requires-Dist: hypothesis<=6.102.6,>=6.82; extra == "test"
34
+ Requires-Dist: ruff<=0.4.5,>=0.3; extra == "test"
35
+ Requires-Dist: coverage<=7.5.2,>=7.2.7; extra == "test"
36
+
37
+ # Bencher
38
+
39
+ ## Continuous Integration Status
40
+
41
+ [![Ci](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/dyson-ai/bencher/actions/workflows/ci.yml?query=branch%3Amain)
42
+ ![Read the Docs](https://img.shields.io/readthedocs/bencher)
43
+ [![Codecov](https://codecov.io/gh/dyson-ai/bencher/branch/main/graph/badge.svg?token=Y212GW1PG6)](https://codecov.io/gh/dyson-ai/bencher)
44
+ [![GitHub issues](https://img.shields.io/github/issues/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/issues/)
45
+ [![GitHub pull-requests merged](https://badgen.net/github/merged-prs/dyson-ai/bencher)](https://github.com/dyson-ai/bencher/pulls?q=is%3Amerged)
46
+ [![PyPI](https://img.shields.io/pypi/v/holobench)](https://pypi.org/project/holobench/)
47
+ [![GitHub release](https://img.shields.io/github/release/dyson-ai/bencher.svg)](https://GitHub.com/dyson-ai/bencher/releases/)
48
+ [![License](https://img.shields.io/pypi/l/bencher)](https://opensource.org/license/mit/)
49
+ [![Python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/downloads/release/python-310/)
50
+ [![Python](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/downloads/release/python-311/)
51
+ [![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
52
+
53
+ ## Badges
54
+
55
+ ### Pixi Badge
56
+ The Pixi badge indicates that this project is optimized for performance and follows best practices as evaluated by the Pixi tool. For more information, visit [Pixi](https://pixi.sh).
57
+
58
+
59
+ ## Intro
60
+
61
+ Bencher is a tool to make it easy to benchmark the interactions between the input parameters to your algorithm and its resulting performance on a set of metrics. It calculates the [cartesian product](https://en.wikipedia.org/wiki/Cartesian_product) of a set of variables
62
+
63
+ Parameters for bencher are defined using the [param](https://param.holoviz.org/) library as a config class with extra metadata that describes the bounds of the search space you want to measure. You must define a benchmarking function that accepts an instance of the config class and return a dictionary with string metric names and float values.
64
+
65
+ Parameters are benchmarked by passing in a list N parameters, and an N-Dimensional tensor is returned. You can optionally sample each point multiple times to get back a distribution and also track its value over time. By default the data will be plotted automatically based on the types of parameters you are sampling (e.g, continous, discrete), but you can also pass in a callback to customize plotting.
66
+
67
+ The data is stored in a persistent database so that past performance is tracked.
68
+
69
+ ## Assumptions
70
+
71
+ The input types should also be of one of the basic datatypes (bool, int, float, str, enum, datetime) so that the data can be easily hashed, cached and stored in the database and processed with seaborn and xarray plotting functions. You can use class inheritance to define hierarchical parameter configuration class types that can be reused in a bigger configuration classes.
72
+
73
+ Bencher is designed to work with stochastic pure functions with no side effects. It assumes that when the objective function is given the same inputs, it will return the same output +- random noise. This is because the function must be called multiple times to get a good statistical distribution of it and so each call must not be influenced by anything or the results will be corrupted.
74
+
75
+ ### Pseudocode of bencher
76
+
77
+ Enumerate a list of all input parameter combinations
78
+ for each set of input parameters:
79
+ pass the inputs to the objective function and store results in the N-D array
80
+
81
+ get unique hash for the set of inputs parameters
82
+ look up previous results for that hash
83
+ if it exists:
84
+ load historical data
85
+ combine latest data with historical data
86
+
87
+ store the results using the input hash as a key
88
+ deduce the type of plot based on the input types
89
+ return data and plot
90
+
91
+ ### Example Output
92
+
93
+ https://dyson-ai.github.io/bencher/
@@ -0,0 +1,45 @@
1
+ LICENSE
2
+ README.md
3
+ package.xml
4
+ pyproject.toml
5
+ setup.cfg
6
+ setup.py
7
+ bencher/__init__.py
8
+ bencher/bench_cfg.py
9
+ bencher/bench_plot_server.py
10
+ bencher/bench_report.py
11
+ bencher/bench_runner.py
12
+ bencher/bencher.py
13
+ bencher/caching.py
14
+ bencher/class_enum.py
15
+ bencher/job.py
16
+ bencher/optuna_conversions.py
17
+ bencher/utils.py
18
+ bencher/video_writer.py
19
+ bencher/worker_job.py
20
+ holobench.egg-info/PKG-INFO
21
+ holobench.egg-info/SOURCES.txt
22
+ holobench.egg-info/dependency_links.txt
23
+ holobench.egg-info/not-zip-safe
24
+ holobench.egg-info/requires.txt
25
+ holobench.egg-info/top_level.txt
26
+ resource/bencher
27
+ test/test_bench_examples.py
28
+ test/test_bench_report.py
29
+ test/test_bench_result_base.py
30
+ test/test_bench_runner.py
31
+ test/test_bencher.py
32
+ test/test_cache.py
33
+ test/test_class_enum.py
34
+ test/test_combinations.py
35
+ test/test_float_formatter.py
36
+ test/test_job.py
37
+ test/test_meta_tests.py
38
+ test/test_parametrized_sweep.py
39
+ test/test_plot_filter.py
40
+ test/test_plot_server.py
41
+ test/test_sample_cache.py
42
+ test/test_sweep_base.py
43
+ test/test_sweep_vars.py
44
+ test/test_utils.py
45
+ test/test_vars.py
@@ -0,0 +1,26 @@
1
+ holoviews<=1.18.3,>=1.15
2
+ numpy<=1.26.4,>=1.0
3
+ param<=2.1.0,>=1.13.0
4
+ hvplot<=0.10.0,>=0.8
5
+ matplotlib<=3.9.0,>=3.6.3
6
+ panel<=1.4.3,>=1.3.6
7
+ diskcache<=5.6.3,>=5.6
8
+ optuna<=3.6.1,>=3.2
9
+ xarray<=2024.5.0,>=2023.7
10
+ plotly<=5.22.0,>=5.15
11
+ sortedcontainers<=2.4,>=2.4
12
+ pandas<=2.2.2,>=2.0
13
+ strenum<=0.4.15,>=0.4.0
14
+ scikit-learn<=1.5.0,>=1.2
15
+ str2bool<=1.1,>=1.1
16
+ scoop<=0.7.2.0,>=0.7.0
17
+ moviepy<=1.0.3,>=1.0.3
18
+
19
+ [test]
20
+ black<=24.4.2,>=23
21
+ pylint<=3.2.2,>=2.17.7
22
+ pytest-cov<=5.0.0,>=4.1
23
+ pytest<=8.2.1,>=7.4
24
+ hypothesis<=6.102.6,>=6.82
25
+ ruff<=0.4.5,>=0.3
26
+ coverage<=7.5.2,>=7.2.7
@@ -0,0 +1 @@
1
+ bencher
@@ -0,0 +1,33 @@
1
+ <?xml version="1.0"?>
2
+ <package format="3">
3
+ <name>bencher</name>
4
+ <version>0.1.0</version>
5
+ <description>A package for benchmarking the performance of arbitrary functions</description>
6
+ <maintainer email="austin.gregg-smith@dyson.com">Austin Gregg-Smith</maintainer>
7
+ <license>MIT</license>
8
+
9
+ <depend>python3-diskcache</depend>
10
+
11
+ <depend>python3-pandas</depend>
12
+ <depend>python3-seaborn</depend>
13
+ <depend>python3-matplotlib</depend>
14
+ <depend>python3-numpy</depend>
15
+ <depend>python3-pytest</depend>
16
+ <depend>python3-hypothesis</depend>
17
+ <depend>xarray</depend>
18
+ <depend>python3-zarr</depend>
19
+ <depend>python3-param</depend>
20
+ <depend>python3-panel</depend>
21
+ <depend>python3-hvplot</depend>
22
+ <depend>python3-optuna</depend>
23
+ <depend>python3-plotly</depend>
24
+
25
+ <!-- TO REMOVE WHEN WE UPGRADE TO PYTHON 3.11, This is in the standard library for python>=3.11 -->
26
+ <depend>strenum</depend>
27
+
28
+ <test_depend>python3-pytest-cov</test_depend>
29
+
30
+ <export>
31
+ <build_type>ament_python</build_type>
32
+ </export>
33
+ </package>
@@ -1,22 +1,18 @@
1
1
  [project]
2
2
  name = "holobench"
3
- version = "1.18.0"
3
+ version = "1.22.2"
4
4
 
5
5
  authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
6
6
  description = "A package for benchmarking the performance of arbitrary functions"
7
7
  readme = "README.md"
8
8
 
9
- #incompatible with setuptools
10
- # requires-python = ">= 3.10"
11
-
12
-
13
9
  dependencies = [
14
10
  "holoviews>=1.15,<=1.18.3",
15
11
  "numpy>=1.0,<=1.26.4",
16
12
  "param>=1.13.0,<=2.1.0",
17
13
  "hvplot>=0.8,<=0.10.0",
18
- "matplotlib>=3.6.3,<=3.8.4",
19
- "panel>=1.3.6,<=1.4.2",
14
+ "matplotlib>=3.6.3,<=3.9.0",
15
+ "panel>=1.3.6,<=1.4.3",
20
16
  "diskcache>=5.6,<=5.6.3",
21
17
  "optuna>=3.2,<=3.6.1",
22
18
  "xarray>=2023.7,<=2024.5.0",
@@ -24,21 +20,37 @@ dependencies = [
24
20
  "sortedcontainers>=2.4,<=2.4",
25
21
  "pandas>=2.0,<=2.2.2",
26
22
  "strenum>=0.4.0,<=0.4.15",
27
- "scikit-learn>=1.2,<=1.4.2",
23
+ "scikit-learn>=1.2,<=1.5.0",
28
24
  "str2bool>=1.1,<=1.1",
29
25
  "scoop>=0.7.0,<=0.7.2.0",
30
26
  "moviepy>=1.0.3,<=1.0.3",
31
27
  ]
32
28
 
29
+ [tool.pixi.project]
30
+ channels = ["conda-forge"]
31
+ platforms = ["linux-64"]
32
+
33
+ [tool.pixi.dependencies]
34
+ python = ">=3.10"
35
+
36
+
37
+ [tool.pixi.feature.py310.dependencies]
38
+ python = "3.10.*"
39
+ [tool.pixi.feature.py311.dependencies]
40
+ python = "3.11.*"
41
+
42
+ [tool.pixi.pypi-dependencies]
43
+ holobench = { path = ".", editable = true }
44
+
33
45
  [project.optional-dependencies]
34
46
  test = [
35
47
  "black>=23,<=24.4.2",
36
- "pylint>=2.16,<=3.1.0",
48
+ "pylint>=2.17.7,<=3.2.2",
37
49
  "pytest-cov>=4.1,<=5.0.0",
38
- "pytest>=7.4,<=8.2.0",
39
- "hypothesis>=6.82,<=6.101.0",
40
- "ruff>=0.0.280,<=0.4.4",
41
- "coverage>=7.2.7,<=7.5.1",
50
+ "pytest>=7.4,<=8.2.1",
51
+ "hypothesis>=6.82,<=6.102.6",
52
+ "ruff>=0.3,<=0.4.5",
53
+ "coverage>=7.2.7,<=7.5.2",
42
54
  ]
43
55
 
44
56
  [project.urls]
@@ -46,18 +58,39 @@ Repository = "https://github.com/dyson-ai/bencher"
46
58
  Home = "https://github.com/dyson-ai/bencher"
47
59
  Documentation = "https://bencher.readthedocs.io/en/latest/"
48
60
 
49
-
50
- [tool.flit.module]
51
- name = "bencher"
52
-
53
-
54
61
  [build-system]
55
- requires = ["flit_core >=3.2,<4"]
56
- build-backend = "flit_core.buildapi"
57
-
58
-
59
- [tool.setuptools_scm]
60
- # lol just to keep colcon happy
62
+ requires = ["setuptools"]
63
+ build-backend = "setuptools.build_meta"
64
+
65
+ # Environments
66
+ [tool.pixi.environments]
67
+ default = { features = ["test"], solve-group = "default" }
68
+ py310 = ["py310", "test"]
69
+ py311 = ["py311", "test"]
70
+
71
+ [tool.pixi.tasks]
72
+ format = "black ."
73
+ check-clean-workspace = "git diff --exit-code"
74
+ ruff-lint = "ruff check . --fix"
75
+ pylint = "pylint --version && echo 'running pylint...'&& pylint $(git ls-files '*.py')"
76
+ lint = { depends_on = ["ruff-lint", "pylint"] }
77
+ style = { depends_on = ["format", "lint"] }
78
+ test = "pytest"
79
+ coverage = "coverage run -m pytest && coverage xml -o coverage.xml"
80
+ coverage-report = "coverage report -m"
81
+ ci-no-cover = { depends_on = ["style", "test"] }
82
+ ci = { depends_on = [
83
+ "style",
84
+ "check-clean-workspace",
85
+ "coverage",
86
+ "coverage-report",
87
+ ] }
88
+ update-lock-push = "pixi update; git commit -a -m'update pixi.lock';git push"
89
+ clear-pixi = "rm -rf .pixi pixi.lock"
90
+
91
+
92
+ [tool.setuptools.packages.find]
93
+ include = ["bencher"]
61
94
 
62
95
  [tool.pylint]
63
96
  extension-pkg-whitelist = ["numpy", "scipy"]
@@ -70,20 +103,17 @@ enable = "no-else-return,consider-using-in"
70
103
  [tool.black]
71
104
  line-length = 100
72
105
 
73
-
74
106
  [tool.ruff]
75
- # Never enforce `E501` (line length violations).
76
- #"F841" will auto remove unused variables which is annoying during development, pylint catches this anyway
77
- lint.ignore = ["E501", "E902", "F841"]
78
-
79
-
80
- # Same as Black.
81
- line-length = 100
107
+ line-length = 100 # Same as Black.
82
108
 
83
109
  target-version = "py310"
84
110
 
111
+ [tool.ruff.lint]
112
+ # Never enforce `E501` (line length violations).
113
+ #"F841" will auto remove unused variables which is annoying during development, pylint catches this anyway
114
+ ignore = ["E501", "E902", "F841"]
85
115
  # Allow unused variables when underscore-prefixed.
86
- lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
116
+ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
87
117
 
88
118
  # Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`.
89
119
  [tool.ruff.lint.per-file-ignores]
@@ -0,0 +1,10 @@
1
+ [develop]
2
+ script_dir = $base/lib/bencher
3
+
4
+ [install]
5
+ install_scripts = $base/lib/bencher
6
+
7
+ [egg_info]
8
+ tag_build =
9
+ tag_date = 0
10
+
@@ -0,0 +1,21 @@
1
+ from setuptools import setup, find_packages
2
+
3
+ package_name = "bencher"
4
+
5
+ setup(
6
+ name=package_name,
7
+ # version="0.0.1",
8
+ description="A library for benchmarking code and generating reports for analysis",
9
+ # maintainer="austin.gregg-smith",
10
+ # maintainer_email="austin.gregg-smith@dyson.com",
11
+ packages=find_packages(exclude=["test.*", "test"]),
12
+ data_files=[
13
+ ("share/ament_index/resource_index/packages", ["resource/" + package_name]),
14
+ ("share/" + package_name, ["package.xml"]),
15
+ ],
16
+ include_package_data=False,
17
+ # install_requires=["setuptools"],
18
+ zip_safe=False,
19
+ test_suite="test",
20
+ tests_require=["pytest"],
21
+ )