holobench 1.25.2__py3-none-any.whl → 1.27.0__py3-none-any.whl

bencher/results/bench_result_base.py

@@ -0,0 +1,476 @@
+import logging
+from typing import List, Any, Tuple, Optional
+from enum import Enum, auto
+import xarray as xr
+from param import Parameter
+import holoviews as hv
+from functools import partial
+import panel as pn
+
+from bencher.utils import int_to_col, color_tuple_to_css, callable_name
+
+from bencher.variables.parametrised_sweep import ParametrizedSweep
+from bencher.variables.inputs import with_level
+
+from bencher.variables.results import OptDir
+from copy import deepcopy
+from bencher.results.optuna_result import OptunaResult
+from bencher.variables.results import ResultVar
+from bencher.plotting.plot_filter import VarRange, PlotFilter
+from bencher.utils import listify
+
+from bencher.variables.results import (
+    ResultReference,
+)
+
+from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
+
+# todo add plugins
+# https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
+# https://kaleidoescape.github.io/decorated-plugins/
+
+
+class ReduceType(Enum):
+    AUTO = auto()  # automatically determine the best way to reduce the dataset
+    SQUEEZE = auto()  # remove any dimensions of length 1
+    REDUCE = auto()  # get the mean and std dev of the the "repeat" dimension
+    NONE = auto()  # don't reduce
+
+
+class EmptyContainer:
+    """A wrapper for list like containers that only appends if the item is not None"""
+
+    def __init__(self, pane) -> None:
+        self.pane = pane
+
+    def append(self, child):
+        if child is not None:
+            self.pane.append(child)
+
+    def get(self):
+        return self.pane if len(self.pane) > 0 else None
+
+
+class BenchResultBase(OptunaResult):
+    def result_samples(self) -> int:
+        """The number of samples in the results dataframe"""
+        return self.ds.count()
+
+    def to_hv_dataset(
+        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+    ) -> hv.Dataset:
+        """Generate a holoviews dataset from the xarray dataset.
+
+        Args:
+            reduce (ReduceType, optional): Optionally perform reduce options on the dataset.  By default the returned dataset will calculate the mean and standard devation over the "repeat" dimension so that the dataset plays nicely with most of the holoviews plot types.  Reduce.Sqeeze is used if there is only 1 repeat and you want the "reduce" variable removed from the dataset. ReduceType.None returns an unaltered dataset. Defaults to ReduceType.AUTO.
+
+        Returns:
+            hv.Dataset: results in the form of a holoviews dataset
+        """
+
+        if reduce == ReduceType.NONE:
+            kdims = [i.name for i in self.bench_cfg.all_vars]
+            return hv.Dataset(self.to_dataset(reduce, result_var, level), kdims=kdims)
+        return hv.Dataset(self.to_dataset(reduce, result_var, level))
+
+    def to_dataset(
+        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+    ) -> xr.Dataset:
+        """Generate a summarised xarray dataset.
+
+        Args:
+            reduce (ReduceType, optional): Optionally perform reduce options on the dataset.  By default the returned dataset will calculate the mean and standard devation over the "repeat" dimension so that the dataset plays nicely with most of the holoviews plot types.  Reduce.Sqeeze is used if there is only 1 repeat and you want the "reduce" variable removed from the dataset. ReduceType.None returns an unaltered dataset. Defaults to ReduceType.AUTO.
+
+        Returns:
+            xr.Dataset: results in the form of an xarray dataset
+        """
+        if reduce == ReduceType.AUTO:
+            reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
+
+        ds_out = self.ds if result_var is None else self.ds[result_var.name]
+
+        match (reduce):
+            case ReduceType.REDUCE:
+                ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
+                ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=True)
+
+                for v in ds_reduce_mean.data_vars:
+                    ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
+                ds_out = ds_reduce_mean
+            case ReduceType.SQUEEZE:
+                ds_out = ds_out.squeeze(drop=True)
+        if level is not None:
+            coords_no_repeat = {}
+            for c, v in ds_out.coords.items():
+                if c != "repeat":
+                    coords_no_repeat[c] = with_level(v.to_numpy(), level)
+            return ds_out.sel(coords_no_repeat)
+        return ds_out
+
+    def get_optimal_vec(
+        self,
+        result_var: ParametrizedSweep,
+        input_vars: List[ParametrizedSweep],
+    ) -> List[Any]:
+        """Get the optimal values from the sweep as a vector.
+
+        Args:
+            result_var (bch.ParametrizedSweep): Optimal values of this result variable
+            input_vars (List[bch.ParametrizedSweep]): Define which input vars values are returned in the vector
+
+        Returns:
+            List[Any]: A vector of optimal values for the desired input vector
+        """
+
+        da = self.get_optimal_value_indices(result_var)
+        output = []
+        for iv in input_vars:
+            if da.coords[iv.name].values.size == 1:
+                # https://stackoverflow.com/questions/773030/why-are-0d-arrays-in-numpy-not-considered-scalar
+                # use [()] to convert from a 0d numpy array to a scalar
+                output.append(da.coords[iv.name].values[()])
+            else:
+                logging.warning(f"values size: {da.coords[iv.name].values.size}")
+                output.append(max(da.coords[iv.name].values[()]))
+            logging.info(f"Maximum value of {iv.name}: {output[-1]}")
+        return output
+
+    def get_optimal_value_indices(self, result_var: ParametrizedSweep) -> xr.DataArray:
+        """Get an xarray mask of the values with the best values found during a parameter sweep
+
+        Args:
+            result_var (bch.ParametrizedSweep): Optimal value of this result variable
+
+        Returns:
+            xr.DataArray: xarray mask of optimal values
+        """
+        result_da = self.ds[result_var.name]
+        if result_var.direction == OptDir.maximize:
+            opt_val = result_da.max()
+        else:
+            opt_val = result_da.min()
+        indicies = result_da.where(result_da == opt_val, drop=True).squeeze()
+        logging.info(f"optimal value of {result_var.name}: {opt_val.values}")
+        return indicies
+
+    def get_optimal_inputs(
+        self,
+        result_var: ParametrizedSweep,
+        keep_existing_consts: bool = True,
+        as_dict: bool = False,
+    ) -> Tuple[ParametrizedSweep, Any] | dict[ParametrizedSweep, Any]:
+        """Get a list of tuples of optimal variable names and value pairs, that can be fed in as constant values to subsequent parameter sweeps
+
+        Args:
+            result_var (bch.ParametrizedSweep): Optimal values of this result variable
+            keep_existing_consts (bool): Include any const values that were defined as part of the parameter sweep
+            as_dict (bool): return value as a dictionary
+
+        Returns:
+            Tuple[bch.ParametrizedSweep, Any]|[ParametrizedSweep, Any]: Tuples of variable name and optimal values
+        """
+        da = self.get_optimal_value_indices(result_var)
+        if keep_existing_consts:
+            output = deepcopy(self.bench_cfg.const_vars)
+        else:
+            output = []
+
+        for iv in self.bench_cfg.input_vars:
+            # assert da.coords[iv.name].values.size == (1,)
+            if da.coords[iv.name].values.size == 1:
+                # https://stackoverflow.com/questions/773030/why-are-0d-arrays-in-numpy-not-considered-scalar
+                # use [()] to convert from a 0d numpy array to a scalar
+                output.append((iv, da.coords[iv.name].values[()]))
+            else:
+                logging.warning(f"values size: {da.coords[iv.name].values.size}")
+                output.append((iv, max(da.coords[iv.name].values[()])))
+
+            logging.info(f"Maximum value of {iv.name}: {output[-1][1]}")
+        if as_dict:
+            return dict(output)
+        return output
+
+    def describe_sweep(self):
+        return self.bench_cfg.describe_sweep()
+
+    def get_best_holomap(self, name: str = None):
+        return self.get_hmap(name)[self.get_best_trial_params(True)]
+
+    def get_hmap(self, name: str = None):
+        try:
+            if name is None:
+                name = self.result_hmaps[0].name
+            if name in self.hmaps:
+                return self.hmaps[name]
+        except Exception as e:
+            raise RuntimeError(
+                "You are trying to plot a holomap result but it is not in the result_vars list.  Add the holomap to the result_vars list"
+            ) from e
+        return None
+
+    def to_plot_title(self) -> str:
+        if len(self.bench_cfg.input_vars) > 0 and len(self.bench_cfg.result_vars) > 0:
+            return f"{self.bench_cfg.result_vars[0].name} vs {self.bench_cfg.input_vars[0].name}"
+        return ""
+
+    def title_from_ds(self, dataset: xr.Dataset, result_var: Parameter, **kwargs):
+        if "title" in kwargs:
+            return kwargs["title"]
+
+        if isinstance(dataset, xr.DataArray):
+            tit = [dataset.name]
+            for d in dataset.dims:
+                tit.append(d)
+        else:
+            tit = [result_var.name]
+            tit.extend(list(dataset.sizes))
+
+        return " vs ".join(tit)
+
+    def get_results_var_list(self, result_var: ParametrizedSweep = None) -> List[ResultVar]:
+        return self.bench_cfg.result_vars if result_var is None else listify(result_var)
+
+    def map_plots(
+        self,
+        plot_callback: callable,
+        result_var: ParametrizedSweep = None,
+        row: EmptyContainer = None,
+    ) -> Optional[pn.Row]:
+        if row is None:
+            row = EmptyContainer(pn.Row(name=self.to_plot_title()))
+        for rv in self.get_results_var_list(result_var):
+            row.append(plot_callback(rv))
+        return row.get()
+
+    def map_plot_panes(
+        self,
+        plot_callback: callable,
+        hv_dataset: hv.Dataset = None,
+        target_dimension: int = 2,
+        result_var: ResultVar = None,
+        result_types=None,
+        pane_collection: pn.pane = None,
+        **kwargs,
+    ) -> Optional[pn.Row]:
+        if hv_dataset is None:
+            hv_dataset = self.to_hv_dataset()
+
+        if pane_collection is None:
+            pane_collection = pn.Row()
+
+        row = EmptyContainer(pane_collection)
+
+        # kwargs= self.set_plot_size(**kwargs)
+        for rv in self.get_results_var_list(result_var):
+            if result_types is None or isinstance(rv, result_types):
+                row.append(
+                    self.to_panes_multi_panel(
+                        hv_dataset,
+                        rv,
+                        plot_callback=partial(plot_callback, **kwargs),
+                        target_dimension=target_dimension,
+                    )
+                )
+        return row.get()
+
+    def filter(
+        self,
+        plot_callback: callable,
+        plot_filter=None,
+        float_range: VarRange = VarRange(0, None),
+        cat_range: VarRange = VarRange(0, None),
+        vector_len: VarRange = VarRange(1, 1),
+        result_vars: VarRange = VarRange(1, 1),
+        panel_range: VarRange = VarRange(0, 0),
+        repeats_range: VarRange = VarRange(1, None),
+        input_range: VarRange = VarRange(1, None),
+        reduce: ReduceType = ReduceType.AUTO,
+        target_dimension: int = 2,
+        result_var: ResultVar = None,
+        result_types=None,
+        pane_collection: pn.pane = None,
+        **kwargs,
+    ):
+        plot_filter = PlotFilter(
+            float_range=float_range,
+            cat_range=cat_range,
+            vector_len=vector_len,
+            result_vars=result_vars,
+            panel_range=panel_range,
+            repeats_range=repeats_range,
+            input_range=input_range,
+        )
+        matches_res = plot_filter.matches_result(self.plt_cnt_cfg, callable_name(plot_callback))
+        if matches_res.overall:
+            return self.map_plot_panes(
+                plot_callback=plot_callback,
+                hv_dataset=self.to_hv_dataset(reduce=reduce),
+                target_dimension=target_dimension,
+                result_var=result_var,
+                result_types=result_types,
+                pane_collection=pane_collection,
+                **kwargs,
+            )
+        return matches_res.to_panel()
+
+    def to_panes_multi_panel(
+        self,
+        hv_dataset: hv.Dataset,
+        result_var: ResultVar,
+        plot_callback: callable = None,
+        target_dimension: int = 1,
+        **kwargs,
+    ):
+        dims = len(hv_dataset.dimensions())
+        if target_dimension is None:
+            target_dimension = dims
+        return self._to_panes_da(
+            hv_dataset.data,
+            plot_callback=plot_callback,
+            target_dimension=target_dimension,
+            horizontal=dims <= target_dimension + 1,
+            result_var=result_var,
+            **kwargs,
+        )
+
+    def _to_panes_da(
+        self,
+        dataset: xr.Dataset,
+        plot_callback: callable = None,
+        target_dimension=1,
+        horizontal=False,
+        result_var=None,
+        **kwargs,
+    ) -> pn.panel:
+        # todo, when dealing with time and repeats, add feature to allow custom order of dimension recursion
+        ##todo remove recursion
+        num_dims = len(dataset.sizes)
+        # print(f"num_dims: {num_dims}, horizontal: {horizontal}, target: {target_dimension}")
+        dims = list(d for d in dataset.sizes)
+
+        time_dim_delta = 0
+        if self.bench_cfg.over_time:
+            time_dim_delta = 0
+
+        if num_dims > (target_dimension + time_dim_delta) and num_dims != 0:
+            selected_dim = dims[-1]
+            # print(f"selected dim {dim_sel}")
+            dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
+
+            outer_container = ComposableContainerPanel(
+                name=" vs ".join(dims), background_col=dim_color, horizontal=not horizontal
+            )
+            max_len = 0
+            for i in range(dataset.sizes[selected_dim]):
+                sliced = dataset.isel({selected_dim: i})
+                label_val = sliced.coords[selected_dim].values.item()
+                inner_container = ComposableContainerPanel(
+                    name=outer_container.name,
+                    width=num_dims - target_dimension,
+                    var_name=selected_dim,
+                    var_value=label_val,
+                    horizontal=horizontal,
+                )
+
+                panes = self._to_panes_da(
+                    sliced,
+                    plot_callback=plot_callback,
+                    target_dimension=target_dimension,
+                    horizontal=len(sliced.sizes) <= target_dimension + 1,
+                    result_var=result_var,
+                )
+                max_len = max(max_len, inner_container.label_len)
+                inner_container.append(panes)
+                outer_container.append(inner_container.container)
+            for c in outer_container.container:
+                c[0].width = max_len * 7
+        else:
+            return plot_callback(dataset=dataset, result_var=result_var, **kwargs)
+
+        return outer_container.container
+
+    def zero_dim_da_to_val(self, da_ds: xr.DataArray | xr.Dataset) -> Any:
+        # todo this is really horrible, need to improve
+        dim = None
+        if isinstance(da_ds, xr.Dataset):
+            dim = list(da_ds.keys())[0]
+            da = da_ds[dim]
+        else:
+            da = da_ds
+
+        for k in da.coords.keys():
+            dim = k
+            break
+        if dim is None:
+            return da_ds.values.squeeze().item()
+        return da.expand_dims(dim).values[0]
+
+    def ds_to_container(
+        self, dataset: xr.Dataset, result_var: Parameter, container, **kwargs
+    ) -> Any:
+        val = self.zero_dim_da_to_val(dataset[result_var.name])
+        if isinstance(result_var, ResultReference):
+            ref = self.object_index[val]
+            if ref is not None:
+                val = ref.obj
+                if ref.container is not None:
+                    return ref.container(val, **kwargs)
+        if container is not None:
+            return container(val, styles={"background": "white"}, **kwargs)
+        try:
+            container = result_var.to_container()
+            if container is not None:
+                return container(val)
+        except AttributeError as _:
+            # TODO make sure all vars have to_container method
+            pass
+        return val
+
+    @staticmethod
+    def select_level(
+        dataset: xr.Dataset,
+        level: int,
+        include_types: List[type] = None,
+        exclude_names: List[str] = None,
+    ) -> xr.Dataset:
+        """Given a dataset, return a reduced dataset that only contains data from a specified level.  By default all types of variables are filtered at the specified level.  If you only want to get a reduced level for some types of data you can pass in a list of types to get filtered, You can also pass a list of variables names to exclude from getting filtered
+        Args:
+            dataset (xr.Dataset): dataset to filter
+            level (int): desired data resolution level
+            include_types (List[type], optional): Only filter data of these types. Defaults to None.
+            exclude_names (List[str], optional): Only filter data with these variable names. Defaults to None.
+
+        Returns:
+            xr.Dataset: A reduced dataset at the specified level
+
+        Example:  a dataset with float_var: [1,2,3,4,5] cat_var: [a,b,c,d,e]
+
+        select_level(ds,2) -> [1,5] [a,e]
+        select_level(ds,2,(float)) -> [1,5] [a,b,c,d,e]
+        select_level(ds,2,exclude_names=["cat_var]) -> [1,5] [a,b,c,d,e]
+
+        see test_bench_result_base.py -> test_select_level()
+        """
+        coords_no_repeat = {}
+        for c, v in dataset.coords.items():
+            if c != "repeat":
+                vals = v.to_numpy()
+                print(vals.dtype)
+                include = True
+                if include_types is not None and vals.dtype not in listify(include_types):
+                    include = False
+                if exclude_names is not None and c in listify(exclude_names):
+                    include = False
+                if include:
+                    coords_no_repeat[c] = with_level(v.to_numpy(), level)
+        return dataset.sel(coords_no_repeat)
+
+    # MAPPING TO LOWER LEVEL BENCHCFG functions so they are available at a top level.
+    def to_sweep_summary(self, **kwargs):
+        return self.bench_cfg.to_sweep_summary(**kwargs)
+
+    def to_title(self, panel_name: str = None) -> pn.pane.Markdown:
+        return self.bench_cfg.to_title(panel_name)
+
+    def to_description(self, width: int = 800) -> pn.pane.Markdown:
+        return self.bench_cfg.to_description(width)

bencher/results/composable_container/composable_container_base.py

@@ -0,0 +1,73 @@
+from enum import auto
+from typing import Any, List
+from dataclasses import dataclass, field
+from strenum import StrEnum
+from bencher.results.float_formatter import FormatFloat
+
+
+# TODO enable these options
+class ComposeType(StrEnum):
+    right = auto()  # append the container to the right (creates a row)
+    down = auto()  # append the container below (creates a column)
+    sequence = auto()  # display the container after (in time)
+    # overlay = auto()  # overlay on top of the current container (alpha blending)
+
+    def flip(self):
+        match self:
+            case ComposeType.right:
+                return ComposeType.down
+            case ComposeType.down:
+                return ComposeType.right
+            case _:
+                raise RuntimeError("cannot flip this type")
+
+    @staticmethod
+    def from_horizontal(horizontal: bool):
+        return ComposeType.right if horizontal else ComposeType.down
+
+
+@dataclass(kw_only=True)
+class ComposableContainerBase:
+    """A base class for renderer backends.  A composable renderer"""
+
+    compose_method: ComposeType = ComposeType.right
+    container: List[Any] = field(default_factory=list)
+    label_len: int = 0
+
+    @staticmethod
+    def label_formatter(var_name: str, var_value: int | float | str) -> str:
+        """Take a variable name and values and return a pretty version with approximate fixed width
+
+        Args:
+            var_name (str): The name of the variable, usually a dimension
+            var_value (int | float | str): The value of the dimension
+
+        Returns:
+            str: Pretty string representation with fixed width
+        """
+
+        if isinstance(var_value, (int, float)):
+            var_value = FormatFloat()(var_value)
+        if var_name is not None and var_value is not None:
+            return f"{var_name}={var_value}"
+        if var_name is not None:
+            return f"{var_name}"
+        if var_value is not None:
+            return f"{var_value}"
+        return None
+
+    def append(self, obj: Any) -> None:
+        """Add an object to the container.  The relationship between the objects is defined by the ComposeType
+
+        Args:
+            obj (Any): Object to add to the container
+        """
+        self.container.append(obj)
+
+    def render(self):
+        """Return a representation of the container that can be composed with other render() results. This function can also be used to defer layout and rending options until all the information about the container content is known.  You may need to ovverride this method depending on the container. See composable_container_video as an example.
+
+        Returns:
+            Any: Visual representation of the container that can be combined with other containers
+        """
+        return self.container

bencher/results/composable_container/composable_container_panel.py

@@ -0,0 +1,39 @@
+import panel as pn
+from bencher.results.composable_container.composable_container_base import ComposableContainerBase
+from dataclasses import dataclass
+
+
+@dataclass(kw_only=True)
+class ComposableContainerPanel(ComposableContainerBase):
+    name: str = None
+    var_name: str = None
+    var_value: str = None
+    width: int = None
+    background_col: str = None
+    horizontal: bool = True
+
+    def __post_init__(
+        self,
+    ) -> None:
+        container_args = {
+            "name": self.name,
+            "styles": {},
+        }
+
+        if self.width is not None:
+            container_args["styles"]["border-bottom"] = f"{self.width}px solid grey"
+        if self.background_col is not None:
+            container_args["styles"]["background"] = self.background_col
+
+        if self.horizontal:
+            self.container = pn.Column(**container_args)
+            align = ("center", "center")
+        else:
+            self.container = pn.Row(**container_args)
+            align = ("end", "center")
+
+        label = self.label_formatter(self.var_name, self.var_value)
+        if label is not None:
+            self.label_len = len(label)
+            side = pn.pane.Markdown(label, align=align)
+            self.append(side)