holobench 1.8.0__tar.gz → 1.9.0__tar.gz

{holobench-1.8.0 → holobench-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: holobench
-Version: 1.8.0
+Version: 1.9.0
 Summary: A package for benchmarking the performance of arbitrary functions
 Author-email: Austin Gregg-Smith <blooop@gmail.com>
 Description-Content-Type: text/markdown

{holobench-1.8.0 → holobench-1.9.0}/bencher/bench_runner.py

@@ -103,7 +103,7 @@ class BenchRunner:
         for r in range(1, repeats + 1):
             for lvl in range(min_level, max_level + 1):
                 if grouped:
-                    report_level = BenchReport(f"{self.name}_{run_cfg.run_tag}")
+                    report_level = BenchReport(f"{run_cfg.run_tag}_{self.name}")
 
                 for bch_fn in self.bench_fns:
                     run_lvl = deepcopy(run_cfg)
@@ -114,6 +114,7 @@ class BenchRunner:
                         res = bch_fn(run_lvl, report_level)
                     else:
                         res = bch_fn(run_lvl, BenchReport())
+                        res.report.bench_name = f"{run_cfg.run_tag}_{res.report.bench_name}"
                         self.show_publish(res.report, show, publish, save, debug)
                     self.results.append(res)
                 if grouped:

{holobench-1.8.0 → holobench-1.9.0}/bencher/bencher.py

@@ -175,8 +175,13 @@ class Bench(BenchPlotServer):
         self.plot_callbacks = []
         self.plot = True
 
-    def add_plot_callback(self, callback: Callable[[BenchResult], pn.panel]) -> None:
-        self.plot_callbacks.append(callback)
+    def add_plot_callback(self, callback: Callable[[BenchResult], pn.panel], **kwargs) -> None:
+        """Add a plotting callback that will be called on any result produced when calling a sweep funciton.  You can pass additional arguments to the plotting function with kwargs.  e.g.  add_plot_callback(bch.BenchResult.to_video_grid,)
+
+        Args:
+            callback (Callable[[BenchResult], pn.panel]): _description_
+        """
+        self.plot_callbacks.append(partial(callback, **kwargs))
 
     def set_worker(self, worker: Callable, worker_input_cfg: ParametrizedSweep = None) -> None:
         """Set the benchmark worker function and optionally the type the worker expects

{holobench-1.8.0 → holobench-1.9.0}/bencher/example/example_image.py

@@ -18,6 +18,7 @@ class BenchPolygons(bch.ParametrizedSweep):
     linestyle = bch.StringSweep(["solid", "dashed", "dotted"])
     color = bch.StringSweep(["red", "green", "blue"])
     polygon = bch.ResultImage()
+    area = bch.ResultVar()
     # hmap = bch.ResultHmap()
 
     def __call__(self, **kwargs):
@@ -25,6 +26,7 @@ class BenchPolygons(bch.ParametrizedSweep):
         points = polygon_points(self.radius, self.sides)
         # self.hmap = hv.Curve(points)
         self.polygon = self.points_to_polygon_png(points, bch.gen_image_path("polygon"))
+        self.area = self.radius * self.sides
         return super().__call__()
 
     def points_to_polygon_png(self, points: list[float], filename: str):
@@ -52,17 +54,26 @@ class BenchPolygons(bch.ParametrizedSweep):
 def example_image(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
+    run_cfg.use_cache = False
     bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
 
+    bench.result_vars = ["polygon", "area"]
+
+    bench.add_plot_callback(bch.BenchResult.to_sweep_summary)
+    # bench.add_plot_callback(bch.BenchResult.to_auto, level=2)
+    bench.add_plot_callback(bch.BenchResult.to_panes, level=3)
+    # bench.add_plot_callback(bch.BenchResult.to_panes)
+
     sweep_vars = ["sides", "radius", "linewidth", "color"]
+
+    # sweep_vars = ["sides", "radius" ]
+
     for i in range(1, len(sweep_vars)):
         s = sweep_vars[:i]
         bench.plot_sweep(
             f"Polygons Sweeping {len(s)} Parameters",
             input_vars=s,
-            result_vars=[BenchPolygons.param.polygon],
         )
-
     return bench
 
 
@@ -97,6 +108,6 @@ if __name__ == "__main__":
     # ex_run_cfg.debug = True
     # ex_run_cfg.repeats = 2
     ex_run_cfg.level = 4
-    example_image_vid(ex_run_cfg).report.show()
+    # example_image_vid(ex_run_cfg).report.show()
     # example_image_vid_sequential(ex_run_cfg).report.show()
-    # example_image(ex_run_cfg).report.show()
+    example_image(ex_run_cfg).report.show()

{holobench-1.8.0 → holobench-1.9.0}/bencher/results/bench_result_base.py

@@ -10,6 +10,8 @@ import panel as pn
 from bencher.utils import int_to_col, color_tuple_to_css, callable_name
 
 from bencher.variables.parametrised_sweep import ParametrizedSweep
+from bencher.variables.inputs import with_level
+
 from bencher.variables.results import OptDir
 from copy import deepcopy
 from bencher.results.optuna_result import OptunaResult
@@ -21,6 +23,7 @@ from bencher.variables.results import (
 )
 
 from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
+from bencher.utils import listify
 
 # todo add plugins
 # https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
@@ -54,7 +57,7 @@ class BenchResultBase(OptunaResult):
         return self.ds.count()
 
     def to_hv_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
+        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
     ) -> hv.Dataset:
         """Generate a holoviews dataset from the xarray dataset.
 
@@ -67,11 +70,11 @@ class BenchResultBase(OptunaResult):
 
         if reduce == ReduceType.NONE:
             kdims = [i.name for i in self.bench_cfg.all_vars]
-            return hv.Dataset(self.to_dataset(reduce, result_var), kdims=kdims)
-        return hv.Dataset(self.to_dataset(reduce, result_var))
+            return hv.Dataset(self.to_dataset(reduce, result_var, level), kdims=kdims)
+        return hv.Dataset(self.to_dataset(reduce, result_var, level))
 
     def to_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
+        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
     ) -> xr.Dataset:
         """Generate a summarised xarray dataset.
 
@@ -84,20 +87,25 @@ class BenchResultBase(OptunaResult):
         if reduce == ReduceType.AUTO:
             reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
 
-        ds = self.ds if result_var is None else self.ds[result_var.name]
+        ds_out = self.ds if result_var is None else self.ds[result_var.name]
 
         match (reduce):
             case ReduceType.REDUCE:
-                ds_reduce_mean = ds.mean(dim="repeat", keep_attrs=True)
-                ds_reduce_std = ds.std(dim="repeat", keep_attrs=True)
+                ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
+                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=True)
 
                 for v in ds_reduce_mean.data_vars:
                     ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
-                return ds_reduce_mean
+                ds_out = ds_reduce_mean
             case ReduceType.SQUEEZE:
-                return ds.squeeze(drop=True)
-            case _:
-                return ds
+                ds_out = ds_out.squeeze(drop=True)
+        if level is not None:
+            coords_no_repeat = {}
+            for c, v in ds_out.coords.items():
+                if c != "repeat":
+                    coords_no_repeat[c] = with_level(v.to_numpy(), level)
+            return ds_out.sel(coords_no_repeat)
+        return ds_out
 
     def get_optimal_vec(
         self,
@@ -405,6 +413,45 @@ class BenchResultBase(OptunaResult):
             return container(val, styles={"background": "white"}, **kwargs)
         return val
 
+    @staticmethod
+    def select_level(
+        dataset: xr.Dataset,
+        level: int,
+        include_types: List[type] = None,
+        exclude_names: List[str] = None,
+    ) -> xr.Dataset:
+        """Given a dataset, return a reduced dataset that only contains data from a specified level.  By default all types of variables are filtered at the specified level.  If you only want to get a reduced level for some types of data you can pass in a list of types to get filtered, You can also pass a list of variables names to exclude from getting filtered
+        Args:
+            dataset (xr.Dataset): dataset to filter
+            level (int): desired data resolution level
+            include_types (List[type], optional): Only filter data of these types. Defaults to None.
+            exclude_names (List[str], optional): Only filter data with these variable names. Defaults to None.
+
+        Returns:
+            xr.Dataset: A reduced dataset at the specified level
+
+        Example:  a dataset with float_var: [1,2,3,4,5] cat_var: [a,b,c,d,e]
+
+        select_level(ds,2) -> [1,5] [a,e]
+        select_level(ds,2,(float)) -> [1,5] [a,b,c,d,e]
+        select_level(ds,2,exclude_names=["cat_var]) -> [1,5] [a,b,c,d,e]
+
+        see test_bench_result_base.py -> test_select_level()
+        """
+        coords_no_repeat = {}
+        for c, v in dataset.coords.items():
+            if c != "repeat":
+                vals = v.to_numpy()
+                print(vals.dtype)
+                include = True
+                if include_types is not None and vals.dtype not in listify(include_types):
+                    include = False
+                if exclude_names is not None and c in listify(exclude_names):
+                    include = False
+                if include:
+                    coords_no_repeat[c] = with_level(v.to_numpy(), level)
+        return dataset.sel(coords_no_repeat)
+
     # MAPPING TO LOWER LEVEL BENCHCFG functions so they are available at a top level.
     def to_sweep_summary(self, **kwargs):
         return self.bench_cfg.to_sweep_summary(**kwargs)

{holobench-1.8.0 → holobench-1.9.0}/bencher/results/composable_container/composable_container_base.py

@@ -36,9 +36,11 @@ class ComposableContainerBase:
             return f"{var_value}"
         return None
 
-    def __init__(self, horizontal: bool = True) -> None:
+    def __init__(
+        self, horizontal: bool = True, compose_method: ComposeType = ComposeType.right
+    ) -> None:
         self.horizontal: bool = horizontal
-        self.compose_method = ComposeType.right
+        self.compose_method = compose_method
         self.container = []
 
     def append(self, obj: Any) -> None:

{holobench-1.8.0 → holobench-1.9.0}/bencher/results/holoview_result.py

@@ -130,6 +130,7 @@ class HoloviewResult(PanelResult):
             float_range=VarRange(1, 1),
             cat_range=VarRange(0, None),
             repeats_range=VarRange(1, 1),
+            panel_range=VarRange(0, None),
             reduce=ReduceType.SQUEEZE,
             target_dimension=2,
             result_var=result_var,

{holobench-1.8.0 → holobench-1.9.0}/bencher/results/panel_result.py

@@ -2,6 +2,7 @@ from typing import Optional
 from functools import partial
 import panel as pn
 from param import Parameter
+import holoviews as hv
 from bencher.results.bench_result_base import BenchResultBase, ReduceType
 from bencher.results.video_result import VideoControls
 from bencher.variables.results import (
@@ -18,13 +19,23 @@ class PanelResult(BenchResultBase):
         )
 
     def to_panes(
-        self, result_var: Parameter = None, target_dimension: int = 0, container=None, **kwargs
+        self,
+        result_var: Parameter = None,
+        hv_dataset=None,
+        target_dimension: int = 0,
+        container=None,
+        level: int = None,
+        **kwargs
     ) -> Optional[pn.pane.panel]:
         if container is None:
             container = pn.pane.panel
+        if hv_dataset is None:
+            hv_dataset = self.to_hv_dataset(ReduceType.SQUEEZE, level=level)
+        elif not isinstance(hv_dataset, hv.Dataset):
+            hv_dataset = hv.Dataset(hv_dataset)
         return self.map_plot_panes(
             partial(self.ds_to_container, container=container),
-            hv_dataset=self.to_hv_dataset(ReduceType.SQUEEZE),
+            hv_dataset=hv_dataset,
             target_dimension=target_dimension,
             result_var=result_var,
             result_types=PANEL_TYPES,

{holobench-1.8.0 → holobench-1.9.0}/bencher/utils.py

@@ -150,7 +150,7 @@ def callable_name(any_callable: Callable[..., Any]) -> str:
 
 
 def listify(obj) -> list:
-    """Take an object and turn it into a list if its not already a list"""
+    """Take an object and turn it into a list if its not already a list.  However if the object is none, don't turn it into a list"""
     if obj is None:
         return None
     if isinstance(obj, list):

{holobench-1.8.0 → holobench-1.9.0}/bencher/variables/inputs.py

@@ -191,3 +191,8 @@ def box(name, center, width):
     var = FloatSweep(default=center, bounds=(center - width, center + width))
     var.name = name
     return var
+
+
+def with_level(arr: list, level) -> list:
+    return IntSweep(sample_values=arr).with_level(level).values()
+    # return tmp.with_sample_values(arr).with_level(level).values()

{holobench-1.8.0 → holobench-1.9.0}/bencher/variables/sweep_base.py

@@ -138,7 +138,7 @@ class SweepBase(param.Parameter):
             output.step = None  # pylint: disable = attribute-defined-outside-init
         return output
 
-    def with_sample_values(self, sample_values: int) -> SweepBase:
+    def with_sample_values(self, sample_values: list) -> SweepBase:
         output = deepcopy(self)
         # TODO set up class properly. Slightly complicated due to slots
         try:

{holobench-1.8.0 → holobench-1.9.0}/bencher/video_writer.py

@@ -32,7 +32,7 @@ class VideoWriter:
         return self.filename
 
     @staticmethod
-    def create_label(label, width=None, height=20):
+    def create_label(label, width=None, height=14):
         if width is None:
             width = len(label) * 8
         new_img = Image.new("RGB", (width, height), (255, 255, 255))

{holobench-1.8.0 → holobench-1.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.8.0"
+version = "1.9.0"
 
 authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
 description = "A package for benchmarking the performance of arbitrary functions"