holobench 1.5.0__tar.gz → 1.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {holobench-1.5.0 → holobench-1.7.0}/PKG-INFO +13 -13
  2. {holobench-1.5.0 → holobench-1.7.0}/bencher/bench_cfg.py +5 -0
  3. {holobench-1.5.0 → holobench-1.7.0}/bencher/bench_plot_server.py +1 -0
  4. {holobench-1.5.0 → holobench-1.7.0}/bencher/bench_report.py +14 -14
  5. {holobench-1.5.0 → holobench-1.7.0}/bencher/bench_runner.py +1 -1
  6. {holobench-1.5.0 → holobench-1.7.0}/bencher/bencher.py +46 -33
  7. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_float_cat.py +1 -0
  8. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_image.py +18 -31
  9. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_simple_bool.py +1 -0
  10. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_simple_cat.py +1 -0
  11. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_time_event.py +1 -0
  12. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/bench_result.py +9 -2
  13. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/bench_result_base.py +55 -44
  14. holobench-1.7.0/bencher/results/composable_container/__init__.py +0 -0
  15. holobench-1.7.0/bencher/results/composable_container/composable_container_base.py +58 -0
  16. holobench-1.7.0/bencher/results/composable_container/composable_container_panel.py +40 -0
  17. holobench-1.7.0/bencher/results/composable_container/composable_container_video.py +54 -0
  18. holobench-1.7.0/bencher/results/panel_result.py +32 -0
  19. holobench-1.7.0/bencher/results/video_summary.py +198 -0
  20. {holobench-1.5.0 → holobench-1.7.0}/bencher/utils.py +2 -2
  21. {holobench-1.5.0 → holobench-1.7.0}/bencher/variables/parametrised_sweep.py +18 -1
  22. {holobench-1.5.0 → holobench-1.7.0}/bencher/video_writer.py +46 -22
  23. {holobench-1.5.0 → holobench-1.7.0}/pyproject.toml +13 -13
  24. holobench-1.5.0/bencher/results/panel_result.py +0 -63
  25. holobench-1.5.0/bencher/results/video_summary.py +0 -77
  26. {holobench-1.5.0 → holobench-1.7.0}/README.md +0 -0
  27. {holobench-1.5.0 → holobench-1.7.0}/bencher/__init__.py +0 -0
  28. {holobench-1.5.0 → holobench-1.7.0}/bencher/caching.py +0 -0
  29. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/__init__.py +0 -0
  30. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/benchmark_data.py +0 -0
  31. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_all.py +0 -0
  32. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_categorical.py +0 -0
  33. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_custom_sweep.py +0 -0
  34. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_docs.py +0 -0
  35. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_float3D.py +0 -0
  36. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_floats.py +0 -0
  37. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_floats2D.py +0 -0
  38. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_holosweep.py +0 -0
  39. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_holosweep_objects.py +0 -0
  40. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_holosweep_tap.py +0 -0
  41. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_levels.py +0 -0
  42. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_pareto.py +0 -0
  43. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_sample_cache.py +0 -0
  44. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_sample_cache_context.py +0 -0
  45. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_simple.py +0 -0
  46. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_simple_float.py +0 -0
  47. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_strings.py +0 -0
  48. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_video.py +0 -0
  49. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/example_workflow.py +0 -0
  50. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_bokeh_plotly.py +0 -0
  51. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_hover_ex.py +0 -0
  52. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_hvplot_explorer.py +0 -0
  53. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_interactive.py +0 -0
  54. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_streamnd.py +0 -0
  55. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_streams.py +0 -0
  56. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_template.py +0 -0
  57. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_updates.py +0 -0
  58. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/experimental/example_vector.py +0 -0
  59. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/meta/example_meta.py +0 -0
  60. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/meta/example_meta_cat.py +0 -0
  61. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/meta/example_meta_float.py +0 -0
  62. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/meta/example_meta_levels.py +0 -0
  63. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/optuna/example_optuna.py +0 -0
  64. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/shelved/example_float2D_scatter.py +0 -0
  65. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/shelved/example_float3D_cone.py +0 -0
  66. {holobench-1.5.0 → holobench-1.7.0}/bencher/example/shelved/example_kwargs.py +0 -0
  67. {holobench-1.5.0 → holobench-1.7.0}/bencher/job.py +0 -0
  68. {holobench-1.5.0 → holobench-1.7.0}/bencher/optuna_conversions.py +0 -0
  69. {holobench-1.5.0 → holobench-1.7.0}/bencher/plotting/__init__.py +0 -0
  70. {holobench-1.5.0 → holobench-1.7.0}/bencher/plotting/plot_filter.py +0 -0
  71. {holobench-1.5.0 → holobench-1.7.0}/bencher/plotting/plt_cnt_cfg.py +0 -0
  72. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/__init__.py +0 -0
  73. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/float_formatter.py +0 -0
  74. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/holoview_result.py +0 -0
  75. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/optuna_result.py +0 -0
  76. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/plotly_result.py +0 -0
  77. {holobench-1.5.0 → holobench-1.7.0}/bencher/results/video_result.py +0 -0
  78. {holobench-1.5.0 → holobench-1.7.0}/bencher/variables/inputs.py +0 -0
  79. {holobench-1.5.0 → holobench-1.7.0}/bencher/variables/results.py +0 -0
  80. {holobench-1.5.0 → holobench-1.7.0}/bencher/variables/sweep_base.py +0 -0
  81. {holobench-1.5.0 → holobench-1.7.0}/bencher/variables/time.py +0 -0
  82. {holobench-1.5.0 → holobench-1.7.0}/bencher/worker_job.py +0 -0
@@ -1,33 +1,33 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: holobench
3
- Version: 1.5.0
3
+ Version: 1.7.0
4
4
  Summary: A package for benchmarking the performance of arbitrary functions
5
5
  Author-email: Austin Gregg-Smith <blooop@gmail.com>
6
6
  Description-Content-Type: text/markdown
7
7
  Requires-Dist: holoviews>=1.15,<=1.18.1
8
- Requires-Dist: numpy>=1.0,<=1.26.2
9
- Requires-Dist: param>=1.13.0,<=2.0.1
10
- Requires-Dist: hvplot>=0.8,<=0.9.1
8
+ Requires-Dist: numpy>=1.0,<=1.26.3
9
+ Requires-Dist: param>=1.13.0,<=2.0.2
10
+ Requires-Dist: hvplot>=0.8,<=0.9.2
11
11
  Requires-Dist: matplotlib>=3.6.3,<=3.8.2
12
- Requires-Dist: panel>=1.3.6,<=1.3.6
12
+ Requires-Dist: panel>=1.3.6,<=1.3.8
13
13
  Requires-Dist: diskcache>=5.6,<=5.6.3
14
14
  Requires-Dist: optuna>=3.2,<=3.5.0
15
- Requires-Dist: xarray>=2023.7,<=2023.12.0
15
+ Requires-Dist: xarray>=2023.7,<=2024.1.1
16
16
  Requires-Dist: plotly>=5.15,<=5.18.0
17
17
  Requires-Dist: sortedcontainers>=2.4,<=2.4
18
- Requires-Dist: pandas>=2.0,<=2.1.4
18
+ Requires-Dist: pandas>=2.0,<=2.2.0
19
19
  Requires-Dist: strenum>=0.4.0,<=0.4.15
20
- Requires-Dist: scikit-learn>=1.2,<=1.3.2
20
+ Requires-Dist: scikit-learn>=1.2,<=1.4.0
21
21
  Requires-Dist: str2bool>=1.1,<=1.1
22
22
  Requires-Dist: scoop>=0.7.0,<=0.7.2.0
23
23
  Requires-Dist: moviepy>=1.0.3,<=1.0.3
24
- Requires-Dist: black>=23,<=23.12.1 ; extra == "test"
24
+ Requires-Dist: black>=23,<=24.1.1 ; extra == "test"
25
25
  Requires-Dist: pylint>=2.16,<=3.0.3 ; extra == "test"
26
26
  Requires-Dist: pytest-cov>=4.1,<=4.1 ; extra == "test"
27
- Requires-Dist: pytest>=7.4,<=7.4.4 ; extra == "test"
28
- Requires-Dist: hypothesis>=6.82,<=6.92.2 ; extra == "test"
29
- Requires-Dist: ruff>=0.0.280,<=0.1.9 ; extra == "test"
30
- Requires-Dist: coverage>=7.2.7,<=7.4.0 ; extra == "test"
27
+ Requires-Dist: pytest>=7.4,<=8.0.0 ; extra == "test"
28
+ Requires-Dist: hypothesis>=6.82,<=6.97.4 ; extra == "test"
29
+ Requires-Dist: ruff>=0.0.280,<=0.1.15 ; extra == "test"
30
+ Requires-Dist: coverage>=7.2.7,<=7.4.1 ; extra == "test"
31
31
  Project-URL: Documentation, https://bencher.readthedocs.io/en/latest/
32
32
  Project-URL: Home, https://github.com/dyson-ai/bencher
33
33
  Project-URL: Repository, https://github.com/dyson-ai/bencher
@@ -303,6 +303,11 @@ class BenchCfg(BenchRunCfg):
303
303
  doc="store the hash value of the config to avoid having to hash multiple times",
304
304
  )
305
305
 
306
+ plot_callbacks = param.List(
307
+ None,
308
+ doc="A callable that takes a BenchResult and returns panel representation of the results",
309
+ )
310
+
306
311
  def __init__(self, **params):
307
312
  super().__init__(**params)
308
313
  self.plot_lib = None
@@ -1,4 +1,5 @@
1
1
  """A server for display plots of benchmark results"""
2
+
2
3
  import logging
3
4
  import os
4
5
  from typing import List, Tuple
@@ -3,7 +3,8 @@ from typing import Callable
3
3
  import os
4
4
  import panel as pn
5
5
  from pathlib import Path
6
- import shutil
6
+ import tempfile
7
+
7
8
  from threading import Thread
8
9
 
9
10
  from bencher.results.bench_result import BenchResult
@@ -47,7 +48,7 @@ class BenchReport(BenchPlotServer):
47
48
  self.pane.append(col)
48
49
 
49
50
  def append_result(self, bench_res: BenchResult) -> None:
50
- self.append_tab(bench_res.to_auto_plots(), bench_res.bench_cfg.title)
51
+ self.append_tab(bench_res.plot(), bench_res.bench_cfg.title)
51
52
 
52
53
  def append_tab(self, pane: pn.panel, name: str = None) -> None:
53
54
  if pane is not None:
@@ -137,24 +138,23 @@ class BenchReport(BenchPlotServer):
137
138
 
138
139
  remote, publish_url = remote_callback(branch_name)
139
140
 
140
- directory = "tmpgit"
141
- report_path = self.save(directory, filename="index.html", in_html_folder=False)
142
- logging.info(f"created report at: {report_path.absolute()}")
141
+ with tempfile.TemporaryDirectory() as td:
142
+ directory = td
143
+ report_path = self.save(directory, filename="index.html", in_html_folder=False)
144
+ logging.info(f"created report at: {report_path.absolute()}")
143
145
 
144
- cd_dir = f"cd {directory} &&"
146
+ cd_dir = f"cd {directory} &&"
145
147
 
146
- os.system(f"{cd_dir} git init")
147
- os.system(f"{cd_dir} git checkout -b {branch_name}")
148
- os.system(f"{cd_dir} git add index.html")
149
- os.system(f'{cd_dir} git commit -m "publish {branch_name}"')
150
- os.system(f"{cd_dir} git remote add origin {remote}")
151
- os.system(f"{cd_dir} git push --set-upstream origin {branch_name} -f")
148
+ os.system(f"{cd_dir} git init")
149
+ os.system(f"{cd_dir} git checkout -b {branch_name}")
150
+ os.system(f"{cd_dir} git add index.html")
151
+ os.system(f'{cd_dir} git commit -m "publish {branch_name}"')
152
+ os.system(f"{cd_dir} git remote add origin {remote}")
153
+ os.system(f"{cd_dir} git push --set-upstream origin {branch_name} -f")
152
154
 
153
155
  logging.info("Published report @")
154
156
  logging.info(publish_url)
155
157
 
156
- shutil.rmtree(directory)
157
-
158
158
  return publish_url
159
159
 
160
160
 
@@ -103,7 +103,7 @@ class BenchRunner:
103
103
  for r in range(1, repeats + 1):
104
104
  for lvl in range(min_level, max_level + 1):
105
105
  if grouped:
106
- report_level = BenchReport(self.name)
106
+ report_level = BenchReport(f"{self.name}_{run_cfg.run_tag}")
107
107
 
108
108
  for bch_fn in self.bench_fns:
109
109
  run_lvl = deepcopy(run_cfg)
@@ -10,6 +10,7 @@ import xarray as xr
10
10
  from diskcache import Cache
11
11
  from contextlib import suppress
12
12
  from functools import partial
13
+ import panel as pn
13
14
 
14
15
  from bencher.worker_job import WorkerJob
15
16
 
@@ -165,6 +166,17 @@ class Bench(BenchPlotServer):
165
166
 
166
167
  self.cache_size = int(100e9) # default to 100gb
167
168
 
169
+ # self.bench_cfg = BenchCfg()
170
+
171
+ # Maybe put this in SweepCfg
172
+ self.input_vars = None
173
+ self.result_vars = None
174
+ self.const_vars = None
175
+ self.plot_callbacks = []
176
+
177
+ def add_plot_callback(self, callback: Callable[[BenchResult], pn.panel]) -> None:
178
+ self.plot_callbacks.append(callback)
179
+
168
180
  def set_worker(self, worker: Callable, worker_input_cfg: ParametrizedSweep = None) -> None:
169
181
  """Set the benchmark worker function and optionally the type the worker expects
170
182
 
@@ -188,34 +200,6 @@ class Bench(BenchPlotServer):
188
200
  logging.info(f"setting worker {worker}")
189
201
  self.worker_input_cfg = worker_input_cfg
190
202
 
191
- def sweep(
192
- self,
193
- input_vars: List[ParametrizedSweep] = None,
194
- result_vars: List[ParametrizedSweep] = None,
195
- const_vars: List[ParametrizedSweep] = None,
196
- time_src: datetime = None,
197
- description: str = None,
198
- post_description: str = None,
199
- pass_repeat: bool = False,
200
- tag: str = "",
201
- run_cfg: BenchRunCfg = None,
202
- plot: bool = False,
203
- ) -> BenchResult:
204
- title = "Sweeping " + " vs ".join(params_to_str(input_vars))
205
- return self.plot_sweep(
206
- title,
207
- input_vars=input_vars,
208
- result_vars=result_vars,
209
- const_vars=const_vars,
210
- time_src=time_src,
211
- description=description,
212
- post_description=post_description,
213
- pass_repeat=pass_repeat,
214
- tag=tag,
215
- run_cfg=run_cfg,
216
- plot=plot,
217
- )
218
-
219
203
  def sweep_sequential(
220
204
  self,
221
205
  title="",
@@ -264,6 +248,7 @@ class Bench(BenchPlotServer):
264
248
  tag: str = "",
265
249
  run_cfg: BenchRunCfg = None,
266
250
  plot: bool = True,
251
+ plot_callbacks=None,
267
252
  ) -> BenchResult:
268
253
  """The all in 1 function benchmarker and results plotter.
269
254
 
@@ -278,7 +263,8 @@ class Bench(BenchPlotServer):
278
263
  pass_repeat (bool,optional) By default do not pass the kwarg 'repeat' to the benchmark function. Set to true if
279
264
  you want the benchmark function to be passed the repeat number
280
265
  tag (str,optional): Use tags to group different benchmarks together.
281
- run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run and plotted
266
+ run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run
267
+ plot_callbacks: A list of plot callbacks to clal on the results
282
268
  Raises:
283
269
  ValueError: If a result variable is not set
284
270
 
@@ -291,18 +277,27 @@ class Bench(BenchPlotServer):
291
277
  logging.info(
292
278
  "No input variables passed, using all param variables in bench class as inputs"
293
279
  )
294
- input_vars = self.worker_class_instance.get_inputs_only()
280
+ if self.input_vars is None:
281
+ input_vars = self.worker_class_instance.get_inputs_only()
282
+ else:
283
+ input_vars = self.input_vars
295
284
  for i in input_vars:
296
285
  logging.info(f"input var: {i.name}")
297
286
  if result_vars is None:
298
287
  logging.info(
299
288
  "No results variables passed, using all result variables in bench class:"
300
289
  )
301
- result_vars = self.worker_class_instance.get_results_only()
290
+ if self.result_vars is None:
291
+ result_vars = self.worker_class_instance.get_results_only()
292
+ else:
293
+ result_vars = self.result_vars
302
294
  for r in result_vars:
303
295
  logging.info(f"result var: {r.name}")
304
296
  if const_vars is None:
305
- const_vars = self.worker_class_instance.get_input_defaults()
297
+ if self.const_vars is None:
298
+ const_vars = self.worker_class_instance.get_input_defaults()
299
+ else:
300
+ const_vars = self.const_vars
306
301
  else:
307
302
  if input_vars is None:
308
303
  input_vars = []
@@ -381,6 +376,12 @@ class Bench(BenchPlotServer):
381
376
  "## Results Description\nPlease set post_description to explain these results"
382
377
  )
383
378
 
379
+ if plot_callbacks is None:
380
+ if len(self.plot_callbacks) == 0:
381
+ plot_callbacks = [BenchResult.to_auto_plots]
382
+ else:
383
+ plot_callbacks = self.plot_callbacks
384
+
384
385
  bench_cfg = BenchCfg(
385
386
  input_vars=input_vars,
386
387
  result_vars=result_vars_only,
@@ -392,7 +393,14 @@ class Bench(BenchPlotServer):
392
393
  title=title,
393
394
  pass_repeat=pass_repeat,
394
395
  tag=run_cfg.run_tag + tag,
396
+ auto_plot=plot,
397
+ plot_callbacks=plot_callbacks,
395
398
  )
399
+ return self.run_sweep(bench_cfg, run_cfg, time_src)
400
+
401
+ def run_sweep(
402
+ self, bench_cfg: BenchCfg, run_cfg: BenchRunCfg, time_src: datetime
403
+ ) -> BenchResult:
396
404
  print("tag", bench_cfg.tag)
397
405
 
398
406
  bench_cfg.param.update(run_cfg.param.values())
@@ -447,8 +455,9 @@ class Bench(BenchPlotServer):
447
455
 
448
456
  bench_res.post_setup()
449
457
 
450
- if plot and bench_res.bench_cfg.auto_plot:
458
+ if bench_cfg.auto_plot:
451
459
  self.report.append_result(bench_res)
460
+
452
461
  self.results.append(bench_res)
453
462
  return bench_res
454
463
 
@@ -799,3 +808,7 @@ class Bench(BenchPlotServer):
799
808
 
800
809
  def get_result(self, index: int = -1) -> BenchResult:
801
810
  return self.results[index]
811
+
812
+ def publish(self, remote_callback: Callable) -> str:
813
+ branch_name = f"{self.bench_name}_{self.run_cfg.run_tag}"
814
+ return self.report.publish(remote_callback, branch_name=branch_name)
@@ -1,4 +1,5 @@
1
1
  """Example of how to perform a parameter sweep for categorical variables"""
2
+
2
3
  import bencher as bch
3
4
 
4
5
 
@@ -44,7 +44,7 @@ class BenchPolygons(bch.ParametrizedSweep):
44
44
 
45
45
  ax.set_aspect("equal")
46
46
  fig.add_axes(ax)
47
- fig.savefig(filename, dpi=50)
47
+ fig.savefig(filename, dpi=30)
48
48
 
49
49
  return filename
50
50
 
@@ -54,24 +54,9 @@ def example_image(
54
54
  ) -> bch.Bench:
55
55
  bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
56
56
 
57
- for s in [
58
- [BenchPolygons.param.sides],
59
- [BenchPolygons.param.sides, BenchPolygons.param.linewidth],
60
- [BenchPolygons.param.sides, BenchPolygons.param.linewidth, BenchPolygons.param.linestyle],
61
- [
62
- BenchPolygons.param.sides,
63
- BenchPolygons.param.linewidth,
64
- BenchPolygons.param.linestyle,
65
- BenchPolygons.param.color,
66
- ],
67
- [
68
- BenchPolygons.param.sides,
69
- BenchPolygons.param.linewidth,
70
- BenchPolygons.param.linestyle,
71
- BenchPolygons.param.color,
72
- BenchPolygons.param.radius,
73
- ],
74
- ]:
57
+ sweep_vars = ["sides", "radius", "linewidth", "color"]
58
+ for i in range(1, len(sweep_vars)):
59
+ s = sweep_vars[:i]
75
60
  bench.plot_sweep(
76
61
  f"Polygons Sweeping {len(s)} Parameters",
77
62
  input_vars=s,
@@ -81,24 +66,26 @@ def example_image(
81
66
  return bench
82
67
 
83
68
 
84
- if __name__ == "__main__":
69
+ def example_image_vid(
70
+ run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
71
+ ) -> bch.Bench:
72
+ bench = BenchPolygons().to_bench(run_cfg, report)
73
+ bench.add_plot_callback(bch.BenchResult.to_sweep_summary)
74
+ bench.add_plot_callback(bch.BenchResult.to_video_grid)
75
+ bench.plot_sweep(input_vars=["sides"])
76
+ bench.plot_sweep(input_vars=["sides", "radius"])
77
+ return bench
85
78
 
86
- def example_image_vid(
87
- run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
88
- ) -> bch.Bench:
89
- bench = BenchPolygons().to_bench(run_cfg, report)
90
- bench.plot_sweep(input_vars=["sides", "radius", "color"], plot=True)
91
- return bench
79
+
80
+ if __name__ == "__main__":
92
81
 
93
82
  def example_image_vid_sequential(
94
83
  run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
95
84
  ) -> bch.Bench:
96
85
  bench = BenchPolygons().to_bench(run_cfg, report)
97
- res_list = bench.sweep_sequential(
98
- input_vars=["radius", "sides", "linewidth", "color"], group_size=3
99
- )
100
- for r in res_list:
101
- bench.report.append(r.to_video_summary())
86
+ bench.add_plot_callback(bch.BenchResult.to_title)
87
+ bench.add_plot_callback(bch.BenchResult.to_video_grid)
88
+ bench.sweep_sequential(input_vars=["radius", "sides", "linewidth", "color"], group_size=1)
102
89
  return bench
103
90
 
104
91
  # def example_image_pairs()
@@ -1,4 +1,5 @@
1
1
  """This file has some examples for how to perform basic benchmarking parameter sweeps"""
2
+
2
3
  import bencher as bch
3
4
 
4
5
  # All the examples will be using the data structures and benchmark function defined in this file
@@ -1,4 +1,5 @@
1
1
  """This file has some examples for how to perform basic benchmarking parameter sweeps"""
2
+
2
3
  # pylint: disable=duplicate-code
3
4
 
4
5
  import bencher as bch
@@ -1,4 +1,5 @@
1
1
  """This file has some examples for how to perform basic benchmarking parameter sweeps"""
2
+
2
3
  # pylint: disable=duplicate-code
3
4
 
4
5
  import bencher as bch
@@ -11,7 +11,6 @@ from bencher.utils import listify
11
11
 
12
12
 
13
13
  class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult):
14
-
15
14
  """Contains the results of the benchmark and has methods to cast the results to various datatypes and graphical representations"""
16
15
 
17
16
  def __init__(self, bench_cfg) -> None:
@@ -36,6 +35,14 @@ class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult):
36
35
  def plotly_callbacks():
37
36
  return [HoloviewResult.to_surface, PlotlyResult.to_volume]
38
37
 
38
+ def plot(self) -> pn.panel:
39
+ """Plots the benchresult using the plot callbacks defined by the bench run
40
+
41
+ Returns:
42
+ pn.panel: A panel representation of the results
43
+ """
44
+ return pn.Column(*[cb(self) for cb in self.bench_cfg.plot_callbacks])
45
+
39
46
  def to_auto(
40
47
  self,
41
48
  plot_list: List[callable] = None,
@@ -68,7 +75,7 @@ class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult):
68
75
  )
69
76
  return row.pane
70
77
 
71
- def to_auto_plots(self, **kwargs) -> List[pn.panel]:
78
+ def to_auto_plots(self, **kwargs) -> pn.panel:
72
79
  """Given the dataset result of a benchmark run, automatically dedeuce how to plot the data based on the types of variables that were sampled
73
80
 
74
81
  Args:
@@ -5,6 +5,8 @@ import xarray as xr
5
5
  from param import Parameter
6
6
  import holoviews as hv
7
7
  from functools import partial
8
+ import panel as pn
9
+
8
10
  from bencher.utils import int_to_col, color_tuple_to_css, callable_name
9
11
 
10
12
  from bencher.variables.parametrised_sweep import ParametrizedSweep
@@ -12,10 +14,13 @@ from bencher.variables.results import OptDir
12
14
  from copy import deepcopy
13
15
  from bencher.results.optuna_result import OptunaResult
14
16
  from bencher.variables.results import ResultVar
15
- from bencher.results.float_formatter import FormatFloat
16
17
  from bencher.plotting.plot_filter import VarRange, PlotFilter
17
- import panel as pn
18
18
 
19
+ from bencher.variables.results import (
20
+ ResultReference,
21
+ )
22
+
23
+ from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
19
24
 
20
25
  # todo add plugins
21
26
  # https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
@@ -333,29 +338,24 @@ class BenchResultBase(OptunaResult):
333
338
  time_dim_delta = 0
334
339
 
335
340
  if num_dims > (target_dimension + time_dim_delta) and num_dims != 0:
336
- dim_sel = dims[-1]
341
+ selected_dim = dims[-1]
337
342
  # print(f"selected dim {dim_sel}")
338
-
339
343
  dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
340
344
 
341
- background_col = dim_color
342
- name = " vs ".join(dims)
343
-
344
- container_args = {"name": name, "styles": {"background": background_col}}
345
- outer_container = (
346
- pn.Row(**container_args) if horizontal else pn.Column(**container_args)
345
+ outer_container = ComposableContainerPanel(
346
+ name=" vs ".join(dims), background_col=dim_color, horizontal=not horizontal
347
347
  )
348
-
349
348
  max_len = 0
350
-
351
- for i in range(dataset.sizes[dim_sel]):
352
- sliced = dataset.isel({dim_sel: i})
353
-
354
- lable_val = sliced.coords[dim_sel].values.item()
355
- if isinstance(lable_val, (int, float)):
356
- lable_val = FormatFloat()(lable_val)
357
-
358
- label = f"{dim_sel}={lable_val}"
349
+ for i in range(dataset.sizes[selected_dim]):
350
+ sliced = dataset.isel({selected_dim: i})
351
+ label_val = sliced.coords[selected_dim].values.item()
352
+ inner_container = ComposableContainerPanel(
353
+ outer_container.name,
354
+ width=num_dims - target_dimension,
355
+ var_name=selected_dim,
356
+ var_value=label_val,
357
+ horizontal=horizontal,
358
+ )
359
359
 
360
360
  panes = self._to_panes_da(
361
361
  sliced,
@@ -364,35 +364,46 @@ class BenchResultBase(OptunaResult):
364
364
  horizontal=len(sliced.sizes) <= target_dimension + 1,
365
365
  result_var=result_var,
366
366
  )
367
- width = num_dims - target_dimension
368
-
369
- container_args = {
370
- "name": name,
371
- "styles": {"border-bottom": f"{width}px solid grey"},
372
- }
373
-
374
- if horizontal:
375
- inner_container = pn.Column(**container_args)
376
- align = ("center", "center")
377
- else:
378
- inner_container = pn.Row(**container_args)
379
- align = ("end", "center")
380
-
381
- label_len = len(label)
382
- if label_len > max_len:
383
- max_len = label_len
384
- side = pn.pane.Markdown(label, align=align)
385
-
386
- inner_container.append(side)
367
+
368
+ if inner_container.label_len > max_len:
369
+ max_len = inner_container.label_len
387
370
  inner_container.append(panes)
388
- outer_container.append(inner_container)
389
- # outer_container.append(pn.Row(inner_container, align="center"))
390
- for c in outer_container:
371
+ outer_container.append(inner_container.container)
372
+ for c in outer_container.container:
391
373
  c[0].width = max_len * 7
392
374
  else:
393
375
  return plot_callback(dataset=dataset, result_var=result_var, **kwargs)
394
376
 
395
- return outer_container
377
+ return outer_container.container
378
+
379
+ def zero_dim_da_to_val(self, da_ds: xr.DataArray | xr.Dataset) -> Any:
380
+ # todo this is really horrible, need to improve
381
+ dim = None
382
+ if isinstance(da_ds, xr.Dataset):
383
+ dim = list(da_ds.keys())[0]
384
+ da = da_ds[dim]
385
+ else:
386
+ da = da_ds
387
+
388
+ for k in da.coords.keys():
389
+ dim = k
390
+ break
391
+ if dim is None:
392
+ return da_ds.values.squeeze().item()
393
+ return da.expand_dims(dim).values[0]
394
+
395
+ def ds_to_container(
396
+ self, dataset: xr.Dataset, result_var: Parameter, container, **kwargs
397
+ ) -> Any:
398
+ val = self.zero_dim_da_to_val(dataset[result_var.name])
399
+ if isinstance(result_var, ResultReference):
400
+ ref = self.object_index[val]
401
+ val = ref.obj
402
+ if ref.container is not None:
403
+ return ref.container(val, **kwargs)
404
+ if container is not None:
405
+ return container(val, styles={"background": "white"}, **kwargs)
406
+ return val
396
407
 
397
408
  # MAPPING TO LOWER LEVEL BENCHCFG functions so they are available at a top level.
398
409
  def to_sweep_summary(self, **kwargs):
@@ -0,0 +1,58 @@
1
+ from enum import Enum, auto
2
+ from typing import Any
3
+ from bencher.results.float_formatter import FormatFloat
4
+
5
+
6
+ # TODO enable these options
7
+ class ComposeType(Enum):
8
+ right = auto() # append the container to the right (creates a row)
9
+ down = auto() # append the container below (creates a column)
10
+ overlay = auto() # overlay on top of the current container (alpha blending)
11
+ sequence = auto() # display the container after (in time)
12
+
13
+
14
+ class ComposableContainerBase:
15
+ """A base class for renderer backends. A composable renderr"""
16
+
17
+ @staticmethod
18
+ def label_formatter(var_name: str, var_value: int | float | str) -> str:
19
+ """Take a variable name and values and return a pretty version with approximate fixed width
20
+
21
+ Args:
22
+ var_name (str): The name of the variable, usually a dimension
23
+ var_value (int | float | str): The value of the dimension
24
+
25
+ Returns:
26
+ str: Pretty string representation with fixed width
27
+ """
28
+
29
+ if isinstance(var_value, (int, float)):
30
+ var_value = FormatFloat()(var_value)
31
+ if var_name is not None and var_value is not None:
32
+ return f"{var_name}={var_value}"
33
+ if var_name is not None:
34
+ return f"{var_name}"
35
+ if var_value is not None:
36
+ return f"{var_value}"
37
+ return None
38
+
39
+ def __init__(self, horizontal: bool = True) -> None:
40
+ self.horizontal: bool = horizontal
41
+ self.compose_method = ComposeType.right
42
+ self.container = []
43
+
44
+ def append(self, obj: Any) -> None:
45
+ """Add an object to the container. The relationship between the objects is defined by the ComposeType
46
+
47
+ Args:
48
+ obj (Any): Object to add to the container
49
+ """
50
+ self.container.append(obj)
51
+
52
+ def render(self):
53
+ """Return a representation of the container that can be composed with other render() results. This function can also be used to defer layout and rending options until all the information about the container content is known. You may need to ovverride this method depending on the container. See composable_container_video as an example.
54
+
55
+ Returns:
56
+ Any: Visual representation of the container that can be combined with other containers
57
+ """
58
+ return self.container
@@ -0,0 +1,40 @@
1
+ import panel as pn
2
+ from bencher.results.composable_container.composable_container_base import ComposableContainerBase
3
+
4
+
5
+ class ComposableContainerPanel(ComposableContainerBase):
6
+ def __init__(
7
+ self,
8
+ name: str = None,
9
+ var_name: str = None,
10
+ var_value: str = None,
11
+ width: int = None,
12
+ background_col: str = None,
13
+ horizontal: bool = True,
14
+ ) -> None:
15
+ super().__init__(horizontal)
16
+
17
+ container_args = {
18
+ "name": name,
19
+ "styles": {},
20
+ }
21
+
22
+ self.name = name
23
+
24
+ if width is not None:
25
+ container_args["styles"]["border-bottom"] = f"{width}px solid grey"
26
+ if background_col is not None:
27
+ container_args["styles"]["background"] = background_col
28
+
29
+ if horizontal:
30
+ self.container = pn.Column(**container_args)
31
+ align = ("center", "center")
32
+ else:
33
+ self.container = pn.Row(**container_args)
34
+ align = ("end", "center")
35
+
36
+ label = self.label_formatter(var_name, var_value)
37
+ if label is not None:
38
+ self.label_len = len(label)
39
+ side = pn.pane.Markdown(label, align=align)
40
+ self.append(side)