holobench 1.34.0__tar.gz → 1.36.0__tar.gz

{holobench-1.34.0 → holobench-1.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: holobench
-Version: 1.34.0
+Version: 1.36.0
 Summary: A package for benchmarking the performance of arbitrary functions
 Project-URL: Repository, https://github.com/dyson-ai/bencher
 Project-URL: Home, https://github.com/dyson-ai/bencher

{holobench-1.34.0 → holobench-1.36.0}/bencher/__init__.py

@@ -56,7 +56,13 @@ from .utils import (
     publish_file,
     github_content,
 )
-from .utils_rerun import publish_and_view_rrd, rrd_to_pane, capture_rerun_window
+
+try:
+    from .utils_rerun import publish_and_view_rrd, rrd_to_pane, capture_rerun_window
+    from .flask_server import run_flask_in_thread
+except ModuleNotFoundError as e:
+    pass
+
 
 from .plotting.plot_filter import VarRange, PlotFilter
 from .variables.parametrised_sweep import ParametrizedSweep
@@ -68,5 +74,3 @@ from .bench_report import BenchReport, GithubPagesCfg
 from .job import Executors
 from .video_writer import VideoWriter, add_image
 from .class_enum import ClassEnum, ExampleEnum
-
-from .flask_server import run_flask_in_thread

{holobench-1.34.0 → holobench-1.36.0}/bencher/bench_cfg.py

@@ -83,16 +83,16 @@ class BenchRunCfg(BenchPlotSrvCfg):
 
     raise_duplicate_exception: bool = param.Boolean(False, doc=" Used to debug unique plot names.")
 
-    use_cache: bool = param.Boolean(
+    cache_results: bool = param.Boolean(
         False,
-        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the use_sample_cache option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
+        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the cache_samples option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
 
     clear_cache: bool = param.Boolean(
         False, doc=" Clear the cache of saved input->output mappings."
     )
 
-    use_sample_cache: bool = param.Boolean(
+    cache_samples: bool = param.Boolean(
         False,
         doc="If true, every time the benchmark function is called, bencher will check if that value has been calculated before and if so load the from the cache.  Note that the sample level cache is different from the benchmark level cache which only caches the aggregate of all the results at the end of the benchmark. This cache lets you stop a benchmark halfway through and continue. However, beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
@@ -182,7 +182,7 @@ class BenchRunCfg(BenchPlotSrvCfg):
         parser.add_argument(
             "--use-cache",
             action="store_true",
-            help=BenchRunCfg.param.use_cache.doc,
+            help=BenchRunCfg.param.cache_results.doc,
         )
 
         parser.add_argument(
@@ -380,8 +380,8 @@ class BenchCfg(BenchRunCfg):
             benchmark_sampling_str.append(f"    run tag: {self.run_tag}")
         if self.level is not None:
             benchmark_sampling_str.append(f"    bench level: {self.level}")
-        benchmark_sampling_str.append(f"    use_cache: {self.use_cache}")
-        benchmark_sampling_str.append(f"    use_sample_cache: {self.use_sample_cache}")
+        benchmark_sampling_str.append(f"    cache_results: {self.cache_results}")
+        benchmark_sampling_str.append(f"    cache_samples {self.cache_samples}")
         benchmark_sampling_str.append(f"    only_hash_tag: {self.only_hash_tag}")
         benchmark_sampling_str.append(f"    executor: {self.executor}")
 

{holobench-1.34.0 → holobench-1.36.0}/bencher/bench_runner.py

@@ -33,11 +33,11 @@ class BenchRunner:
 
     @staticmethod
     def setup_run_cfg(
-        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, use_cache=True
+        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, cache_results=True
     ) -> BenchRunCfg:
         run_cfg_out = deepcopy(run_cfg)
-        run_cfg_out.use_sample_cache = use_cache
-        run_cfg_out.only_hash_tag = use_cache
+        run_cfg_out.cache_samples = cache_results
+        run_cfg_out.only_hash_tag = cache_results
         run_cfg_out.level = level
         return run_cfg_out
 
@@ -78,9 +78,9 @@ class BenchRunner:
         show: bool = False,
         save: bool = False,
         grouped: bool = True,
-        use_cache: bool = True,
+        cache_results: bool = True,
     ) -> List[Bench]:
-        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default use_cache=True so that previous values are reused.
+        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default cache_results=True so that previous values are reused.
 
         Args:
             min_level (int, optional): The minimum level to start sampling at. Defaults to 2.
@@ -93,14 +93,14 @@ class BenchRunner:
             show (bool, optional): show the results in the local web browser. Defaults to False.
             save (bool, optional): save the results to disk in index.html. Defaults to False.
             grouped (bool, optional): Produce a single html page with all the benchmarks included. Defaults to True.
-            use_cache (bool, optional): Use the sample cache to reused previous results. Defaults to True.
+            cache_results (bool, optional): Use the sample cache to reused previous results. Defaults to True.
 
         Returns:
             List[BenchCfg]: A list of bencher instances
         """
         if run_cfg is None:
             run_cfg = deepcopy(self.run_cfg)
-        run_cfg = BenchRunner.setup_run_cfg(run_cfg, use_cache=use_cache)
+        run_cfg = BenchRunner.setup_run_cfg(run_cfg, cache_results=cache_results)
 
         if level is not None:
             min_level = level

{holobench-1.34.0 → holobench-1.36.0}/bencher/bencher.py

@@ -327,7 +327,7 @@ class Bench(BenchPlotServer):
                 logging.info("Copy run cfg from bench class")
 
         if run_cfg.only_plot:
-            run_cfg.use_cache = True
+            run_cfg.cache_results = True
 
         self.last_run_cfg = run_cfg
 
@@ -371,7 +371,7 @@ class Bench(BenchPlotServer):
                     title += "s"
                 title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in const_vars_in])
             else:
-                raise RuntimeError("you must pass a title, or define inputs or consts")
+                title = "Recording: " + ", ".join([i.name for i in result_vars_in])
 
         if run_cfg.level > 0:
             inputs = []
@@ -448,7 +448,7 @@ class Bench(BenchPlotServer):
             if run_cfg.clear_cache:
                 c.delete(bench_cfg_hash)
                 logging.info("cleared cache")
-            elif run_cfg.use_cache:
+            elif run_cfg.cache_results:
                 logging.info(
                     f"checking for previously calculated results with key: {bench_cfg_hash}"
                 )
@@ -813,7 +813,7 @@ class Bench(BenchPlotServer):
             cache_name="sample_cache",
             tag_index=True,
             size_limit=self.cache_size,
-            use_cache=run_cfg.use_sample_cache,
+            cache_results=run_cfg.cache_samples,
         )
 
     def clear_tag_from_sample_cache(self, tag: str, run_cfg):
@@ -871,6 +871,9 @@ class Bench(BenchPlotServer):
     def get_result(self, index: int = -1) -> BenchResult:
         return self.results[index]
 
+    def get_ds(self, index: int = -1) -> xr.Dataset:
+        return self.get_result(index).to_xarray()
+
     def publish(self, remote_callback: Callable) -> str:
         branch_name = f"{self.bench_name}_{self.run_cfg.run_tag}"
         return self.report.publish(remote_callback, branch_name=branch_name)

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_composable_container.py

@@ -98,7 +98,7 @@ def example_composable_container_video(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = False
+    ex_run_cfg.cache_samples = False
     # ex_run_cfg.level = 2
     ex_report = bch.BenchReport()
     example_composable_container_image(ex_run_cfg, report=ex_report)

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_composable_container2.py

@@ -144,7 +144,7 @@ def example_composable_container_image(
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_sample_cache = False
+#     ex_run_cfg.cache_samples = False
 #     # ex_run_cfg.level = 2
 #     ex_report = bch.BenchReport()
 #     example_composable_container_image(ex_run_cfg, report=ex_report)
@@ -157,4 +157,4 @@ if __name__ == "__main__":
     # bench_runner.add_run(bench_image)
     bench_runner.add_run(example_composable_container_image)
 
-    bench_runner.run(level=6, show=True, use_cache=False)
+    bench_runner.run(level=6, show=True, cache_results=False)

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_holosweep.py

@@ -95,4 +95,4 @@ if __name__ == "__main__":
     PlotFunctions().to_gui()
     bench_run = bch.BenchRunner("bench_runner_test")
     bench_run.add_run(example_holosweep)
-    bench_run.run(level=6, show=True, use_cache=False)
+    bench_run.run(level=6, show=True, cache_results=False)

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_image.py

@@ -58,7 +58,7 @@ class BenchPolygons(bch.ParametrizedSweep):
 def example_image(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
-    run_cfg.use_cache = False
+    run_cfg.cache_results = False
     bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
 
     bench.result_vars = ["polygon", "area"]
@@ -142,7 +142,7 @@ if __name__ == "__main__":
     # def example_image_pairs()
 
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     # ex_run_cfg.debug = True
     # ex_run_cfg.repeats = 2
     ex_run_cfg.level = 4

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_image1.py

@@ -73,7 +73,7 @@ def example_image_vid_sequential1(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.overwrite_sample_cache = True
     ex_run_cfg.level = 3
 

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_sample_cache.py

@@ -2,7 +2,7 @@ import bencher as bch
 
 
 class UnreliableClass(bch.ParametrizedSweep):
-    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
+    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.cache_samples you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
 
     input_val = bch.IntSweep(
         default=0,
@@ -31,7 +31,7 @@ def example_sample_cache(
     report: bch.BenchReport = bch.BenchReport(),
     trigger_crash: bool = False,
 ) -> bch.Bench:
-    """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
+    """This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
 
     Args:
         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
@@ -50,7 +50,7 @@ def example_sample_cache(
         title="Example Crashy Function with the sample_cache",
         input_vars=[UnreliableClass.param.input_val],
         result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
-        description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
+        description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
         run_cfg=run_cfg,
         post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
     )
@@ -63,7 +63,7 @@ if __name__ == "__main__":
     ex_run_cfg.executor = bch.Executors.SCOOP
 
     # this will store the result of of every call to crashy_fn
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.clear_sample_cache = True
 
     try:

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_sample_cache_context.py

@@ -51,7 +51,7 @@ def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_ca
 
 def example_cache_context() -> bch.Bench:
     run_cfg = bch.BenchRunCfg()
-    run_cfg.use_sample_cache = True
+    run_cfg.cache_samples = True
     run_cfg.only_hash_tag = True
     run_cfg.repeats = 2
     run_cfg.parallel = False

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/example_video.py

@@ -79,7 +79,7 @@ def example_video(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
     # run_cfg.auto_plot = False
-    # run_cfg.use_sample_cache = True
+    # run_cfg.cache_samples = True
     bench = bch.Bench("example_video", TuringPattern(), run_cfg=run_cfg, report=report)
 
     bench.plot_sweep(
@@ -111,7 +111,7 @@ def example_video_tap(
 if __name__ == "__main__":
     run_cfg_ex = bch.BenchRunCfg()
     run_cfg_ex.level = 2
-    run_cfg_ex.use_sample_cache = True
+    run_cfg_ex.cache_samples = True
     run_cfg_ex.only_hash_tag = True
 
     # example_video(run_cfg_ex).report.show()

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/experimental/example_hvplot_explorer.py

@@ -30,7 +30,7 @@ if __name__ == "__main__":
         post_description="Here you can see the output plot of sin theta between 0 and pi.  In the tabs at the top you can also view 3 tabular representations of the data",
         run_cfg=bch.BenchRunCfg(
             auto_plot=True,
-            use_cache=False,
+            cache_results=False,
             repeats=2,
         ),
     )

holobench-1.36.0/bencher/example/inputs_0D/example_0D.py

@@ -0,0 +1,34 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+import random
+
+
+class SimpleFloat0D(bch.ParametrizedSweep):
+    """This class has 0 input dimensions and 1 output dimensions.  It samples from a gaussian distribution"""
+
+    # This defines a variable that we want to plot
+    output = bch.ResultVar(units="ul", doc="a sample from a gaussian distribution")
+
+    def __call__(self, **kwargs) -> dict:
+        """Generate a sample from a uniform distribution
+
+        Returns:
+            dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
+        """
+
+        self.output = random.gauss(mu=0.0, sigma=1.0)
+        return super().__call__(**kwargs)
+
+
+def example_0D(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = SimpleFloat0D().to_bench(run_cfg, report)
+    bench.plot_sweep()
+    return bench
+
+
+if __name__ == "__main__":
+    run_config = bch.BenchRunCfg(repeats=100)
+    example_0D(run_config).report.show()

holobench-1.36.0/bencher/example/inputs_1D/example_1D.py

@@ -0,0 +1,62 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+
+
+class DataSource:
+    def __init__(self):
+        self.data = [
+            [0, 0, 0, 0],
+            [1, 1, 1, 1],
+            [1, 1, 1, 1],
+            [2, 1, 1, 0],
+            [2, 2, 0, 0],
+            [2, 2, 1, 1],
+        ]
+
+    def call(self, index, repeat):
+        return self.data[index][repeat - 1]
+
+
+class Example1D(bch.ParametrizedSweep):
+    index = bch.IntSweep(default=0, bounds=[0, 5], doc="Input angle", units="rad", samples=30)
+    output = bch.ResultVar(units="v", doc="sin of theta")
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        self.output = DataSource().call(self.index, kwargs["repeat"])
+        return super().__call__(**kwargs)
+
+
+def example_1D_float_repeats(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = Example1D().to_bench(run_cfg, report)
+    # bench.plot_sweep(pass_repeat=True,plot_callbacks=False)
+
+    # res = bench.get_result()
+    bench.run_cfg = bch.BenchRunCfg(repeats=4)
+    # bench.plot_sweep(pass_repeat=True, plot_callbacks=False)
+    bench.plot_sweep(pass_repeat=True)
+
+    res = bench.get_result()
+    bench.report.append(res.to_curve())
+    # bench.report.append(hv.Table(res.to_hv_dataset(bch.ReduceType.MINMAX)))
+    # bench.report.append(res.to_curve() + res.to_scatter_jitter(override=True))
+    # bench.report.append(res.to_line())
+    bench.report.append(res.to_scatter_jitter(override=True))
+    # bench.report.append(res.to_error_bar())
+    # bench.report.append(res.to_explorer())
+    # bench.report.append(res.to_error_bar()
+
+    # bench.report.append(res.to_dataset())
+    # bench.report.append(res.to_xarray().hvplot.plot(kind="andrews_curves"))
+    # print(res.to_xarray())
+    # bench.report.append()
+    return bench
+
+
+if __name__ == "__main__":
+    example_1D_float_repeats().report.show()

{holobench-1.34.0 → holobench-1.36.0}/bencher/example/shelved/example_float3D_cone.py

@@ -92,5 +92,5 @@
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_cache = True
+#     ex_run_cfg.cache_results = True
 #     example_cone(ex_run_cfg).report.show()

{holobench-1.34.0 → holobench-1.36.0}/bencher/job.py

@@ -59,7 +59,7 @@ class Executors(StrEnum):
     # THREADS=auto() #not that useful as most bench code is cpu bound
 
     @staticmethod
-    def factory(provider: Executors) -> Future():
+    def factory(provider: Executors) -> Future:
         providers = {
             Executors.SERIAL: None,
             Executors.MULTIPROCESSING: ProcessPoolExecutor(),
@@ -78,11 +78,11 @@ class FutureCache:
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(20e9),  # 20 GB
-        use_cache=True,
+        cache_results=True,
     ):
         self.executor_type = executor
         self.executor = None
-        if use_cache:
+        if cache_results:
             self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
             logging.info(f"cache dir: {self.cache.directory}")
         else:

{holobench-1.34.0 → holobench-1.36.0}/bencher/plotting/plot_filter.py

@@ -64,16 +64,24 @@ class PlotFilter:
     repeats_range: VarRange = VarRange(1, None)
     input_range: VarRange = VarRange(1, None)
 
-    def matches_result(self, plt_cnt_cfg: PltCntCfg, plot_name: str) -> PlotMatchesResult:
+    def matches_result(
+        self, plt_cnt_cfg: PltCntCfg, plot_name: str, override: bool = False
+    ) -> PlotMatchesResult:
         """Checks if the result data signature matches the type of data the plot is able to display."""
-        return PlotMatchesResult(self, plt_cnt_cfg, plot_name)
+        return PlotMatchesResult(self, plt_cnt_cfg, plot_name, override)
 
 
 # @dataclass
 class PlotMatchesResult:
     """Stores information about which properties match the requirements of a particular plotter"""
 
-    def __init__(self, plot_filter: PlotFilter, plt_cnt_cfg: PltCntCfg, plot_name: str):
+    def __init__(
+        self,
+        plot_filter: PlotFilter,
+        plt_cnt_cfg: PltCntCfg,
+        plot_name: str,
+        override: bool = False,
+    ):
         match_info = []
         matches = []
 
@@ -92,8 +100,11 @@ class PlotMatchesResult:
             matches.append(match)
             if not match:
                 match_info.append(info)
-
-        self.overall = all(matches)
+        if override:
+            match_info.append(f"override: {override}")
+            self.overall = True
+        else:
+            self.overall = all(matches)
 
         match_info.insert(0, f"plot {plot_name} matches: {self.overall}")
         self.matches_info = "\n".join(match_info).strip()

{holobench-1.34.0 → holobench-1.36.0}/bencher/results/bench_result.py

@@ -7,11 +7,12 @@ from bencher.results.video_summary import VideoSummaryResult
 from bencher.results.panel_result import PanelResult
 from bencher.results.plotly_result import PlotlyResult
 from bencher.results.holoview_result import HoloviewResult
+from bencher.results.hvplot_result import HvplotResult
 from bencher.results.dataset_result import DataSetResult
 from bencher.utils import listify
 
 
-class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult, DataSetResult):
+class BenchResult(PlotlyResult, HoloviewResult, HvplotResult, VideoSummaryResult, DataSetResult):  # noqa pylint: disable=too-many-ancestors
     """Contains the results of the benchmark and has methods to cast the results to various datatypes and graphical representations"""
 
     def __init__(self, bench_cfg) -> None:
@@ -28,6 +29,7 @@ class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult, DataSetResul
             HoloviewResult.to_curve,
             HoloviewResult.to_line,
             HoloviewResult.to_heatmap,
+            HvplotResult.to_histogram,
             PlotlyResult.to_volume,
             # PanelResult.to_video,
             PanelResult.to_panes,

{holobench-1.34.0 → holobench-1.36.0}/bencher/results/bench_result_base.py

@@ -21,7 +21,9 @@ from bencher.utils import listify
 
 from bencher.variables.results import ResultReference, ResultDataSet
 
-from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
+from bencher.results.composable_container.composable_container_panel import (
+    ComposableContainerPanel,
+)
 
 # todo add plugins
 # https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
@@ -31,7 +33,8 @@ from bencher.results.composable_container.composable_container_panel import Comp
 class ReduceType(Enum):
     AUTO = auto()  # automatically determine the best way to reduce the dataset
     SQUEEZE = auto()  # remove any dimensions of length 1
-    REDUCE = auto()  # get the mean and std dev of the the "repeat" dimension
+    REDUCE = auto()  # get the mean and std dev of the data along the "repeat" dimension
+    MINMAX = auto()  # get the minimum and maximum of data along the "repeat" dimension
     NONE = auto()  # don't reduce
 
 
@@ -55,7 +58,10 @@ class BenchResultBase(OptunaResult):
         return self.ds.count()
 
     def to_hv_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+        self,
+        reduce: ReduceType = ReduceType.AUTO,
+        result_var: ResultVar = None,
+        level: int = None,
     ) -> hv.Dataset:
         """Generate a holoviews dataset from the xarray dataset.
 
@@ -72,7 +78,10 @@ class BenchResultBase(OptunaResult):
         return hv.Dataset(self.to_dataset(reduce, result_var, level))
 
     def to_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+        self,
+        reduce: ReduceType = ReduceType.AUTO,
+        result_var: ResultVar = None,
+        level: int = None,
     ) -> xr.Dataset:
         """Generate a summarised xarray dataset.
 
@@ -85,16 +94,35 @@ class BenchResultBase(OptunaResult):
         if reduce == ReduceType.AUTO:
             reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
 
-        ds_out = self.ds if result_var is None else self.ds[result_var.name]
+        ds_out = self.ds.copy()
+
+        if result_var is not None:
+            ds_out = ds_out[result_var.name]
+
+        def rename_ds(dataset: xr.Dataset, suffix: str):
+            # var_name =
+            rename_dict = {var: f"{var}_{suffix}" for var in dataset.data_vars}
+            ds = dataset.rename_vars(rename_dict)
+            return ds
 
         match reduce:
             case ReduceType.REDUCE:
                 ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
-                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=True)
-
-                for v in ds_reduce_mean.data_vars:
-                    ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
-                ds_out = ds_reduce_mean
+                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=False)
+                ds_reduce_std = rename_ds(ds_reduce_std, "std")
+                ds_out = xr.merge([ds_reduce_mean, ds_reduce_std])
+                ds_out = xr.merge(
+                    [
+                        ds_reduce_mean,
+                        ds_reduce_std,
+                    ]
+                )
+            case ReduceType.MINMAX:  # TODO, need to pass mean, center of minmax, and minmax
+                ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
+                ds_reduce_min = ds_out.min(dim="repeat")
+                ds_reduce_max = ds_out.max(dim="repeat")
+                ds_reduce_range = rename_ds(ds_reduce_max - ds_reduce_min, "range")
+                ds_out = xr.merge([ds_reduce_mean, ds_reduce_range])
             case ReduceType.SQUEEZE:
                 ds_out = ds_out.squeeze(drop=True)
         if level is not None:
@@ -331,6 +359,7 @@ class BenchResultBase(OptunaResult):
         result_var: ResultVar = None,
         result_types=None,
         pane_collection: pn.pane = None,
+        override=False,
         **kwargs,
     ):
         plot_filter = PlotFilter(
@@ -342,7 +371,9 @@ class BenchResultBase(OptunaResult):
             repeats_range=repeats_range,
             input_range=input_range,
         )
-        matches_res = plot_filter.matches_result(self.plt_cnt_cfg, callable_name(plot_callback))
+        matches_res = plot_filter.matches_result(
+            self.plt_cnt_cfg, callable_name(plot_callback), override
+        )
         if matches_res.overall:
             return self.map_plot_panes(
                 plot_callback=plot_callback,
@@ -400,7 +431,9 @@ class BenchResultBase(OptunaResult):
             dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
 
             outer_container = ComposableContainerPanel(
-                name=" vs ".join(dims), background_col=dim_color, horizontal=not horizontal
+                name=" vs ".join(dims),
+                background_col=dim_color,
+                horizontal=not horizontal,
             )
             max_len = 0
             for i in range(dataset.sizes[selected_dim]):

{holobench-1.34.0 → holobench-1.36.0}/bencher/results/holoview_result.py

@@ -6,6 +6,7 @@ import holoviews as hv
 from param import Parameter
 from functools import partial
 import hvplot.xarray  # noqa pylint: disable=duplicate-code,unused-import
+import hvplot.pandas  # noqa pylint: disable=duplicate-code,unused-import
 import xarray as xr
 
 from bencher.utils import (
@@ -86,7 +87,9 @@ class HoloviewResult(PanelResult):
 
         # return time_widget_args
 
-    def to_bar(self, result_var: Parameter = None, **kwargs) -> Optional[pn.panel]:
+    def to_bar(
+        self, result_var: Parameter = None, override: bool = False, **kwargs
+    ) -> Optional[pn.panel]:
         return self.filter(
             self.to_bar_ds,
             float_range=VarRange(0, 0),
@@ -97,6 +100,7 @@ class HoloviewResult(PanelResult):
             target_dimension=2,
             result_var=result_var,
             result_types=(ResultVar),
+            override=override,
             **kwargs,
         )
 
@@ -186,6 +190,7 @@ class HoloviewResult(PanelResult):
             cat_range=VarRange(0, None),
             repeats_range=VarRange(2, None),
             reduce=ReduceType.REDUCE,
+            # reduce=ReduceType.MINMAX,
             target_dimension=2,
             result_var=result_var,
             result_types=(ResultVar),
@@ -196,7 +201,6 @@ class HoloviewResult(PanelResult):
         self, dataset: xr.Dataset, result_var: Parameter, **kwargs
     ) -> Optional[hv.Curve]:
         hvds = hv.Dataset(dataset)
-        # result_var = self.get_results_var_list(result_var)[0]
         title = self.title_from_ds(dataset, result_var, **kwargs)
         pt = hvds.to(hv.Curve).opts(title=title, **kwargs)
         pt *= hvds.to(hv.Spread).opts(alpha=0.2)
@@ -541,17 +545,22 @@ class HoloviewResult(PanelResult):
     def to_scatter_jitter(
         self,
         result_var: Parameter = None,
+        override: bool = False,
         **kwargs,  # pylint: disable=unused-argument
     ) -> List[hv.Scatter]:
-        return self.overlay_plots(partial(self.to_scatter_jitter_single, **kwargs))
+        return self.overlay_plots(
+            partial(self.to_scatter_jitter_single, override=override, **kwargs)
+        )
 
-    def to_scatter_jitter_single(self, result_var: Parameter, **kwargs) -> Optional[hv.Scatter]:
+    def to_scatter_jitter_single(
+        self, result_var: Parameter, override: bool = True, **kwargs
+    ) -> Optional[hv.Scatter]:
         matches = PlotFilter(
             float_range=VarRange(0, 0),
             cat_range=VarRange(0, None),
             repeats_range=VarRange(2, None),
             input_range=VarRange(1, None),
-        ).matches_result(self.plt_cnt_cfg, "to_scatter_jitter")
+        ).matches_result(self.plt_cnt_cfg, "to_scatter_jitter", override)
         if matches.overall:
             ds = self.to_hv_dataset(ReduceType.NONE)
             pt = (
@@ -640,9 +649,6 @@ class HoloviewResult(PanelResult):
 
         return hv.DynamicMap(cb, kdims=kdims)
 
-    def to_explorer(self):
-        return self.to_xarray().hvplot.explorer()
-
     def to_grid(self, inputs=None):
         if inputs is None:
             inputs = self.bench_cfg.inputs_as_str()
@@ -653,7 +659,13 @@ class HoloviewResult(PanelResult):
     def to_table(self):
         return self.to(hv.Table, ReduceType.SQUEEZE)
 
-    def to_surface(self, result_var: Parameter = None, **kwargs) -> Optional[pn.panel]:
+    def to_tabulator(self, **kwargs):
+        """Passes the data to the panel Tabulator type to display an interactive table
+        see https://panel.holoviz.org/reference/widgets/Tabulator.html for extra options
+        """
+        return pn.widgets.Tabulator(self.to_pandas(), **kwargs)
+
+    def to_surface(self, result_var: Parameter = None, **kwargs) -> Optional[pn.pane.Pane]:
         return self.filter(
             self.to_surface_ds,
             float_range=VarRange(2, None),

holobench-1.36.0/bencher/results/hvplot_result.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+from typing import Optional
+import panel as pn
+from param import Parameter
+import hvplot.xarray  # noqa pylint: disable=duplicate-code,unused-import
+import hvplot.pandas  # noqa pylint: disable=duplicate-code,unused-import
+import xarray as xr
+
+from bencher.results.panel_result import PanelResult
+from bencher.results.bench_result_base import ReduceType
+
+from bencher.plotting.plot_filter import VarRange
+from bencher.variables.results import ResultVar
+
+
+class HvplotResult(PanelResult):
+    def to_explorer(self) -> pn.pane.Pane:
+        """Produces a hvplot explorer instance to explore the generated dataset
+        see: https://hvplot.holoviz.org/getting_started/explorer.html
+
+        Returns:
+            pn.pane.Pane: A dynamic pane for exploring a dataset
+        """
+
+        if len(self.bench_cfg.input_vars) > 0:
+            return self.to_xarray().hvplot.explorer()
+
+        # For some reason hvplot doesn't like 1D datasets in xarray, so convert to pandas which it has no problem with
+        # TODO look into why this is, its probably due to how I am setting up the indexing in xarray.
+        return self.to_pandas().hvplot.explorer()
+
+    def to_histogram(self, result_var: Parameter = None, **kwargs) -> Optional[pn.pane.Pane]:
+        return self.filter(
+            self.to_histogram_ds,
+            float_range=VarRange(0, 0),
+            cat_range=VarRange(0, None),
+            input_range=VarRange(0, 0),
+            reduce=ReduceType.NONE,
+            target_dimension=2,
+            result_var=result_var,
+            result_types=(ResultVar),
+            **kwargs,
+        )
+
+    def to_histogram_ds(self, dataset: xr.Dataset, result_var: Parameter, **kwargs):
+        return dataset.hvplot(
+            kind="hist",
+            y=[result_var.name],
+            ylabel="count",
+            legend="bottom_right",
+            widget_location="bottom",
+            title=f"{result_var.name} vs Count",
+            **kwargs,
+        )

{holobench-1.34.0 → holobench-1.36.0}/bencher/variables/parametrised_sweep.py

@@ -192,7 +192,12 @@ class ParametrizedSweep(Parameterized):
             )
         )
 
-    def __call__(self, **kwargs):
+    def __call__(self, **kwargs) -> dict:
+        """This is the function that is called to record data samples in the benchmarking function.  It should be overridden with your custom logic and then call the parent method  "return super().__call__(**kwargs)"
+
+        Returns:
+            dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
+        """
         return self.get_results_values_as_dict()
 
     def plot_hmap(self, **kwargs):

{holobench-1.34.0 → holobench-1.36.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.34.0"
+version = "1.36.0"
 
 authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
 description = "A package for benchmarking the performance of arbitrary functions"
@@ -97,7 +97,7 @@ coverage-report = "coverage report -m"
 update-lock = "pixi update && git commit -a -m'update pixi.lock' || true"
 push = "git push"
 update-lock-push = { depends-on = ["update-lock", "push"] }
-fix = { depends-on = ["update-lock", "format", "ruff-lint"] }
+fix = { depends-on = ["update-lock", "format", "ruff-lint", "pre-commit"] }
 fix-commit-push = { depends-on = ["fix", "commit-format", "update-lock-push"] }
 ci-no-cover = { depends-on = ["style", "test"] }
 ci = { depends-on = [