holobench 1.35.0__tar.gz → 1.36.0__tar.gz

{holobench-1.35.0 → holobench-1.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: holobench
-Version: 1.35.0
+Version: 1.36.0
 Summary: A package for benchmarking the performance of arbitrary functions
 Project-URL: Repository, https://github.com/dyson-ai/bencher
 Project-URL: Home, https://github.com/dyson-ai/bencher

{holobench-1.35.0 → holobench-1.36.0}/bencher/bench_cfg.py

@@ -83,16 +83,16 @@ class BenchRunCfg(BenchPlotSrvCfg):
 
     raise_duplicate_exception: bool = param.Boolean(False, doc=" Used to debug unique plot names.")
 
-    use_cache: bool = param.Boolean(
+    cache_results: bool = param.Boolean(
         False,
-        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the use_sample_cache option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
+        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the cache_samples option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
 
     clear_cache: bool = param.Boolean(
         False, doc=" Clear the cache of saved input->output mappings."
     )
 
-    use_sample_cache: bool = param.Boolean(
+    cache_samples: bool = param.Boolean(
         False,
         doc="If true, every time the benchmark function is called, bencher will check if that value has been calculated before and if so load the from the cache.  Note that the sample level cache is different from the benchmark level cache which only caches the aggregate of all the results at the end of the benchmark. This cache lets you stop a benchmark halfway through and continue. However, beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
@@ -182,7 +182,7 @@ class BenchRunCfg(BenchPlotSrvCfg):
         parser.add_argument(
             "--use-cache",
             action="store_true",
-            help=BenchRunCfg.param.use_cache.doc,
+            help=BenchRunCfg.param.cache_results.doc,
         )
 
         parser.add_argument(
@@ -380,8 +380,8 @@ class BenchCfg(BenchRunCfg):
             benchmark_sampling_str.append(f"    run tag: {self.run_tag}")
         if self.level is not None:
             benchmark_sampling_str.append(f"    bench level: {self.level}")
-        benchmark_sampling_str.append(f"    use_cache: {self.use_cache}")
-        benchmark_sampling_str.append(f"    use_sample_cache: {self.use_sample_cache}")
+        benchmark_sampling_str.append(f"    cache_results: {self.cache_results}")
+        benchmark_sampling_str.append(f"    cache_samples {self.cache_samples}")
         benchmark_sampling_str.append(f"    only_hash_tag: {self.only_hash_tag}")
         benchmark_sampling_str.append(f"    executor: {self.executor}")
 

{holobench-1.35.0 → holobench-1.36.0}/bencher/bench_runner.py

@@ -33,11 +33,11 @@ class BenchRunner:
 
     @staticmethod
     def setup_run_cfg(
-        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, use_cache=True
+        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, cache_results=True
     ) -> BenchRunCfg:
         run_cfg_out = deepcopy(run_cfg)
-        run_cfg_out.use_sample_cache = use_cache
-        run_cfg_out.only_hash_tag = use_cache
+        run_cfg_out.cache_samples = cache_results
+        run_cfg_out.only_hash_tag = cache_results
         run_cfg_out.level = level
         return run_cfg_out
 
@@ -78,9 +78,9 @@ class BenchRunner:
         show: bool = False,
         save: bool = False,
         grouped: bool = True,
-        use_cache: bool = True,
+        cache_results: bool = True,
     ) -> List[Bench]:
-        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default use_cache=True so that previous values are reused.
+        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default cache_results=True so that previous values are reused.
 
         Args:
             min_level (int, optional): The minimum level to start sampling at. Defaults to 2.
@@ -93,14 +93,14 @@ class BenchRunner:
             show (bool, optional): show the results in the local web browser. Defaults to False.
             save (bool, optional): save the results to disk in index.html. Defaults to False.
             grouped (bool, optional): Produce a single html page with all the benchmarks included. Defaults to True.
-            use_cache (bool, optional): Use the sample cache to reused previous results. Defaults to True.
+            cache_results (bool, optional): Use the sample cache to reused previous results. Defaults to True.
 
         Returns:
             List[BenchCfg]: A list of bencher instances
         """
         if run_cfg is None:
             run_cfg = deepcopy(self.run_cfg)
-        run_cfg = BenchRunner.setup_run_cfg(run_cfg, use_cache=use_cache)
+        run_cfg = BenchRunner.setup_run_cfg(run_cfg, cache_results=cache_results)
 
         if level is not None:
             min_level = level

{holobench-1.35.0 → holobench-1.36.0}/bencher/bencher.py

@@ -327,7 +327,7 @@ class Bench(BenchPlotServer):
                 logging.info("Copy run cfg from bench class")
 
         if run_cfg.only_plot:
-            run_cfg.use_cache = True
+            run_cfg.cache_results = True
 
         self.last_run_cfg = run_cfg
 
@@ -371,7 +371,7 @@ class Bench(BenchPlotServer):
                     title += "s"
                 title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in const_vars_in])
             else:
-                title = " ".join([i.name for i in result_vars_in])
+                title = "Recording: " + ", ".join([i.name for i in result_vars_in])
 
         if run_cfg.level > 0:
             inputs = []
@@ -448,7 +448,7 @@ class Bench(BenchPlotServer):
             if run_cfg.clear_cache:
                 c.delete(bench_cfg_hash)
                 logging.info("cleared cache")
-            elif run_cfg.use_cache:
+            elif run_cfg.cache_results:
                 logging.info(
                     f"checking for previously calculated results with key: {bench_cfg_hash}"
                 )
@@ -813,7 +813,7 @@ class Bench(BenchPlotServer):
             cache_name="sample_cache",
             tag_index=True,
             size_limit=self.cache_size,
-            use_cache=run_cfg.use_sample_cache,
+            cache_results=run_cfg.cache_samples,
         )
 
     def clear_tag_from_sample_cache(self, tag: str, run_cfg):

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_composable_container.py

@@ -98,7 +98,7 @@ def example_composable_container_video(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = False
+    ex_run_cfg.cache_samples = False
     # ex_run_cfg.level = 2
     ex_report = bch.BenchReport()
     example_composable_container_image(ex_run_cfg, report=ex_report)

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_composable_container2.py

@@ -144,7 +144,7 @@ def example_composable_container_image(
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_sample_cache = False
+#     ex_run_cfg.cache_samples = False
 #     # ex_run_cfg.level = 2
 #     ex_report = bch.BenchReport()
 #     example_composable_container_image(ex_run_cfg, report=ex_report)
@@ -157,4 +157,4 @@ if __name__ == "__main__":
     # bench_runner.add_run(bench_image)
     bench_runner.add_run(example_composable_container_image)
 
-    bench_runner.run(level=6, show=True, use_cache=False)
+    bench_runner.run(level=6, show=True, cache_results=False)

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_holosweep.py

@@ -95,4 +95,4 @@ if __name__ == "__main__":
     PlotFunctions().to_gui()
     bench_run = bch.BenchRunner("bench_runner_test")
     bench_run.add_run(example_holosweep)
-    bench_run.run(level=6, show=True, use_cache=False)
+    bench_run.run(level=6, show=True, cache_results=False)

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_image.py

@@ -58,7 +58,7 @@ class BenchPolygons(bch.ParametrizedSweep):
 def example_image(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
-    run_cfg.use_cache = False
+    run_cfg.cache_results = False
     bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
 
     bench.result_vars = ["polygon", "area"]
@@ -142,7 +142,7 @@ if __name__ == "__main__":
     # def example_image_pairs()
 
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     # ex_run_cfg.debug = True
     # ex_run_cfg.repeats = 2
     ex_run_cfg.level = 4

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_image1.py

@@ -73,7 +73,7 @@ def example_image_vid_sequential1(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.overwrite_sample_cache = True
     ex_run_cfg.level = 3
 

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_sample_cache.py

@@ -2,7 +2,7 @@ import bencher as bch
 
 
 class UnreliableClass(bch.ParametrizedSweep):
-    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
+    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.cache_samples you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
 
     input_val = bch.IntSweep(
         default=0,
@@ -31,7 +31,7 @@ def example_sample_cache(
     report: bch.BenchReport = bch.BenchReport(),
     trigger_crash: bool = False,
 ) -> bch.Bench:
-    """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
+    """This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
 
     Args:
         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
@@ -50,7 +50,7 @@ def example_sample_cache(
         title="Example Crashy Function with the sample_cache",
         input_vars=[UnreliableClass.param.input_val],
         result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
-        description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
+        description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
         run_cfg=run_cfg,
         post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
     )
@@ -63,7 +63,7 @@ if __name__ == "__main__":
     ex_run_cfg.executor = bch.Executors.SCOOP
 
     # this will store the result of of every call to crashy_fn
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.clear_sample_cache = True
 
     try:

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_sample_cache_context.py

@@ -51,7 +51,7 @@ def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_ca
 
 def example_cache_context() -> bch.Bench:
     run_cfg = bch.BenchRunCfg()
-    run_cfg.use_sample_cache = True
+    run_cfg.cache_samples = True
     run_cfg.only_hash_tag = True
     run_cfg.repeats = 2
     run_cfg.parallel = False

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/example_video.py

@@ -79,7 +79,7 @@ def example_video(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
     # run_cfg.auto_plot = False
-    # run_cfg.use_sample_cache = True
+    # run_cfg.cache_samples = True
     bench = bch.Bench("example_video", TuringPattern(), run_cfg=run_cfg, report=report)
 
     bench.plot_sweep(
@@ -111,7 +111,7 @@ def example_video_tap(
 if __name__ == "__main__":
     run_cfg_ex = bch.BenchRunCfg()
     run_cfg_ex.level = 2
-    run_cfg_ex.use_sample_cache = True
+    run_cfg_ex.cache_samples = True
     run_cfg_ex.only_hash_tag = True
 
     # example_video(run_cfg_ex).report.show()

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/experimental/example_hvplot_explorer.py

@@ -30,7 +30,7 @@ if __name__ == "__main__":
         post_description="Here you can see the output plot of sin theta between 0 and pi.  In the tabs at the top you can also view 3 tabular representations of the data",
         run_cfg=bch.BenchRunCfg(
             auto_plot=True,
-            use_cache=False,
+            cache_results=False,
             repeats=2,
         ),
     )

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/inputs_1D/example_1D.py

@@ -38,13 +38,22 @@ def example_1D_float_repeats(
 
     # res = bench.get_result()
     bench.run_cfg = bch.BenchRunCfg(repeats=4)
+    # bench.plot_sweep(pass_repeat=True, plot_callbacks=False)
     bench.plot_sweep(pass_repeat=True)
 
     res = bench.get_result()
-    bench.report.append(res.to_auto())
-    bench.report.append(res.to_scatter())
+    bench.report.append(res.to_curve())
+    # bench.report.append(hv.Table(res.to_hv_dataset(bch.ReduceType.MINMAX)))
+    # bench.report.append(res.to_curve() + res.to_scatter_jitter(override=True))
+    # bench.report.append(res.to_line())
     bench.report.append(res.to_scatter_jitter(override=True))
+    # bench.report.append(res.to_error_bar())
+    # bench.report.append(res.to_explorer())
+    # bench.report.append(res.to_error_bar()
 
+    # bench.report.append(res.to_dataset())
+    # bench.report.append(res.to_xarray().hvplot.plot(kind="andrews_curves"))
+    # print(res.to_xarray())
     # bench.report.append()
     return bench
 

{holobench-1.35.0 → holobench-1.36.0}/bencher/example/shelved/example_float3D_cone.py

@@ -92,5 +92,5 @@
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_cache = True
+#     ex_run_cfg.cache_results = True
 #     example_cone(ex_run_cfg).report.show()

{holobench-1.35.0 → holobench-1.36.0}/bencher/job.py

@@ -59,7 +59,7 @@ class Executors(StrEnum):
     # THREADS=auto() #not that useful as most bench code is cpu bound
 
     @staticmethod
-    def factory(provider: Executors) -> Future():
+    def factory(provider: Executors) -> Future:
         providers = {
             Executors.SERIAL: None,
             Executors.MULTIPROCESSING: ProcessPoolExecutor(),
@@ -78,11 +78,11 @@ class FutureCache:
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(20e9),  # 20 GB
-        use_cache=True,
+        cache_results=True,
     ):
         self.executor_type = executor
         self.executor = None
-        if use_cache:
+        if cache_results:
             self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
             logging.info(f"cache dir: {self.cache.directory}")
         else:

{holobench-1.35.0 → holobench-1.36.0}/bencher/results/bench_result_base.py

@@ -33,7 +33,8 @@ from bencher.results.composable_container.composable_container_panel import (
 class ReduceType(Enum):
     AUTO = auto()  # automatically determine the best way to reduce the dataset
     SQUEEZE = auto()  # remove any dimensions of length 1
-    REDUCE = auto()  # get the mean and std dev of the the "repeat" dimension
+    REDUCE = auto()  # get the mean and std dev of the data along the "repeat" dimension
+    MINMAX = auto()  # get the minimum and maximum of data along the "repeat" dimension
     NONE = auto()  # don't reduce
 
 
@@ -93,16 +94,35 @@ class BenchResultBase(OptunaResult):
         if reduce == ReduceType.AUTO:
             reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
 
-        ds_out = self.ds if result_var is None else self.ds[result_var.name]
+        ds_out = self.ds.copy()
+
+        if result_var is not None:
+            ds_out = ds_out[result_var.name]
+
+        def rename_ds(dataset: xr.Dataset, suffix: str):
+            # var_name =
+            rename_dict = {var: f"{var}_{suffix}" for var in dataset.data_vars}
+            ds = dataset.rename_vars(rename_dict)
+            return ds
 
         match reduce:
             case ReduceType.REDUCE:
                 ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
-                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=True)
-
-                for v in ds_reduce_mean.data_vars:
-                    ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
-                ds_out = ds_reduce_mean
+                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=False)
+                ds_reduce_std = rename_ds(ds_reduce_std, "std")
+                ds_out = xr.merge([ds_reduce_mean, ds_reduce_std])
+                ds_out = xr.merge(
+                    [
+                        ds_reduce_mean,
+                        ds_reduce_std,
+                    ]
+                )
+            case ReduceType.MINMAX:  # TODO, need to pass mean, center of minmax, and minmax
+                ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
+                ds_reduce_min = ds_out.min(dim="repeat")
+                ds_reduce_max = ds_out.max(dim="repeat")
+                ds_reduce_range = rename_ds(ds_reduce_max - ds_reduce_min, "range")
+                ds_out = xr.merge([ds_reduce_mean, ds_reduce_range])
             case ReduceType.SQUEEZE:
                 ds_out = ds_out.squeeze(drop=True)
         if level is not None:

{holobench-1.35.0 → holobench-1.36.0}/bencher/results/holoview_result.py

@@ -190,6 +190,7 @@ class HoloviewResult(PanelResult):
             cat_range=VarRange(0, None),
             repeats_range=VarRange(2, None),
             reduce=ReduceType.REDUCE,
+            # reduce=ReduceType.MINMAX,
             target_dimension=2,
             result_var=result_var,
             result_types=(ResultVar),
@@ -200,7 +201,6 @@ class HoloviewResult(PanelResult):
         self, dataset: xr.Dataset, result_var: Parameter, **kwargs
     ) -> Optional[hv.Curve]:
         hvds = hv.Dataset(dataset)
-        # result_var = self.get_results_var_list(result_var)[0]
         title = self.title_from_ds(dataset, result_var, **kwargs)
         pt = hvds.to(hv.Curve).opts(title=title, **kwargs)
         pt *= hvds.to(hv.Spread).opts(alpha=0.2)

{holobench-1.35.0 → holobench-1.36.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.35.0"
+version = "1.36.0"
 
 authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
 description = "A package for benchmarking the performance of arbitrary functions"