holobench 1.3.6__py2.py3-none-any.whl

bencher/results/optuna_result.py

@@ -0,0 +1,354 @@
+from __future__ import annotations
+from typing import List
+from copy import deepcopy
+
+import numpy as np
+import optuna
+import panel as pn
+from collections import defaultdict
+from textwrap import wrap
+
+import pandas as pd
+import xarray as xr
+
+
+from optuna.visualization import (
+    plot_param_importances,
+    plot_pareto_front,
+)
+from bencher.utils import hmap_canonical_input
+from bencher.variables.time import TimeSnapshot, TimeEvent
+from bencher.bench_cfg import BenchCfg
+from bencher.plotting.plt_cnt_cfg import PltCntCfg
+
+
+# from bencher.results.bench_result_base import BenchResultBase
+from bencher.optuna_conversions import (
+    sweep_var_to_optuna_dist,
+    summarise_trial,
+    param_importance,
+    optuna_grid_search,
+    summarise_optuna_study,
+    sweep_var_to_suggest,
+)
+
+
+def convert_dataset_bool_dims_to_str(dataset: xr.Dataset) -> xr.Dataset:
+    """Given a dataarray that contains boolean coordinates, conver them to strings so that holoviews loads the data properly
+
+    Args:
+        dataarray (xr.DataArray): dataarray with boolean coordinates
+
+    Returns:
+        xr.DataArray: dataarray with boolean coordinates converted to strings
+    """
+    bool_coords = {}
+    for c in dataset.coords:
+        if dataset.coords[c].dtype == bool:
+            bool_coords[c] = [str(vals) for vals in dataset.coords[c].values]
+
+    if len(bool_coords) > 0:
+        return dataset.assign_coords(bool_coords)
+    return dataset
+
+
+class OptunaResult:
+    def __init__(self, bench_cfg: BenchCfg) -> None:
+        self.bench_cfg = bench_cfg
+        # self.wrap_long_time_labels(bench_cfg)  # todo remove
+        self.ds = xr.Dataset()
+        self.object_index = []
+        self.hmaps = defaultdict(dict)
+        self.result_hmaps = bench_cfg.result_hmaps
+        self.studies = []
+        self.plt_cnt_cfg = PltCntCfg()
+        self.plot_inputs = []
+
+        # self.width=600/
+        # self.height=600
+
+        #   bench_res.objects.append(rv)
+        # bench_res.reference_index = len(bench_res.objects)
+
+    def post_setup(self):
+        self.plt_cnt_cfg = PltCntCfg.generate_plt_cnt_cfg(self.bench_cfg)
+        self.bench_cfg = self.wrap_long_time_labels(self.bench_cfg)
+        self.ds = convert_dataset_bool_dims_to_str(self.ds)
+
+    def to_xarray(self) -> xr.Dataset:
+        return self.ds
+
+    def setup_object_index(self):
+        self.object_index = []
+
+    def to_pandas(self, reset_index=True) -> pd.DataFrame:
+        """Get the xarray results as a pandas dataframe
+
+        Returns:
+            pd.DataFrame: The xarray results array as a pandas dataframe
+        """
+        ds = self.to_xarray().to_dataframe()
+        if reset_index:
+            return ds.reset_index()
+        return ds
+
+    def wrap_long_time_labels(self, bench_cfg):
+        """Takes a benchCfg and wraps any index labels that are too long to be plotted easily
+
+        Args:
+            bench_cfg (BenchCfg):
+
+        Returns:
+            BenchCfg: updated config with wrapped labels
+        """
+        if bench_cfg.over_time:
+            if self.ds.coords["over_time"].dtype == np.datetime64:
+                # plotly catastrophically fails to plot anything with the default long string representation of time, so convert to a shorter time representation
+                self.ds.coords["over_time"] = [
+                    pd.to_datetime(t).strftime("%d-%m-%y %H-%M-%S")
+                    for t in self.ds.coords["over_time"].values
+                ]
+                # wrap very long time event labels because otherwise the graphs are unreadable
+            if bench_cfg.time_event is not None:
+                self.ds.coords["over_time"] = [
+                    "\n".join(wrap(t, 20)) for t in self.ds.coords["over_time"].values
+                ]
+        return bench_cfg
+
+    def to_optuna_plots(self) -> List[pn.pane.panel]:
+        """Create an optuna summary from the benchmark results
+
+        Returns:
+            List[pn.pane.panel]: A list of optuna plot summarising the benchmark process
+        """
+
+        return self.collect_optuna_plots()
+
+    def to_optuna_from_sweep(self, bench, n_trials=30):
+        optu = self.to_optuna_from_results(
+            bench.worker, n_trials=n_trials, extra_results=bench.results
+        )
+        return summarise_optuna_study(optu)
+
+    def to_optuna_from_results(
+        self,
+        worker,
+        n_trials=100,
+        extra_results: List[OptunaResult] = None,
+        sampler=optuna.samplers.TPESampler(),
+    ):
+        directions = []
+        for rv in self.bench_cfg.optuna_targets(True):
+            directions.append(rv.direction)
+
+        study = optuna.create_study(
+            sampler=sampler, directions=directions, study_name=self.bench_cfg.title
+        )
+
+        # add already calculated results
+        results_list = extra_results if extra_results is not None else [self]
+        for res in results_list:
+            if len(res.ds.sizes) > 0:
+                study.add_trials(res.bench_results_to_optuna_trials(True))
+
+        def wrapped(trial) -> tuple:
+            kwargs = {}
+            for iv in self.bench_cfg.input_vars:
+                kwargs[iv.name] = sweep_var_to_suggest(iv, trial)
+            result = worker(**kwargs)
+            output = []
+            for rv in self.bench_cfg.result_vars:
+                output.append(result[rv.name])
+            return tuple(output)
+
+        study.optimize(wrapped, n_trials=n_trials)
+        return study
+
+    def bench_results_to_optuna_trials(self, include_meta: bool = True) -> optuna.Study:
+        """Convert an xarray dataset to an optuna study so optuna can further optimise or plot the statespace
+
+        Args:
+            bench_cfg (BenchCfg): benchmark config to convert
+
+        Returns:
+            optuna.Study: optuna description of the study
+        """
+        if include_meta:
+            df = self.to_pandas()
+            all_vars = []
+            for v in self.bench_cfg.all_vars:
+                if type(v) != TimeEvent:
+                    all_vars.append(v)
+
+            print("All vars", all_vars)
+        else:
+            all_vars = self.bench_cfg.input_vars
+            # df = self.ds.
+            # if "repeat" in self.
+            # if self.bench_cfg.repeats>1:
+            # df = self.bench_cfg.ds.mean("repeat").to_dataframe().reset_index()
+            # else:
+            df = self.to_pandas().reset_index()
+        # df = self.bench_cfg.ds.mean("repeat").to_dataframe.reset_index()
+        # self.bench_cfg.all_vars
+        # del self.bench_cfg.meta_vars[1]
+
+        trials = []
+        distributions = {}
+        for i in all_vars:
+            distributions[i.name] = sweep_var_to_optuna_dist(i)
+
+        for row in df.iterrows():
+            params = {}
+            values = []
+            for i in all_vars:
+                if type(i) == TimeSnapshot:
+                    if type(row[1][i.name]) == np.datetime64:
+                        params[i.name] = row[1][i.name].timestamp()
+                else:
+                    params[i.name] = row[1][i.name]
+
+            for r in self.bench_cfg.optuna_targets():
+                values.append(row[1][r])
+
+            trials.append(
+                optuna.trial.create_trial(
+                    params=params,
+                    distributions=distributions,
+                    values=values,
+                )
+            )
+        return trials
+
+    def bench_result_to_study(self, include_meta: bool) -> optuna.Study:
+        trials = self.bench_results_to_optuna_trials(include_meta)
+        study = optuna_grid_search(self.bench_cfg)
+        optuna.logging.set_verbosity(optuna.logging.CRITICAL)
+        import warnings
+
+        # /usr/local/lib/python3.10/dist-packages/optuna/samplers/_grid.py:224: UserWarning: over_time contains a value with the type of <class 'pandas._libs.tslibs.timestamps.Timestamp'>, which is not supported by `GridSampler`. Please make sure a value is `str`, `int`, `float`, `bool` or `None` for persistent storage.
+
+        # this is not disabling the warning
+        warnings.filterwarnings(action="ignore", category=UserWarning)
+        # remove optuna gridsearch warning as we are not using their gridsearch because it has the most inexplicably terrible performance I have ever seen in my life. How can a for loop of 400 iterations start out with 100ms per loop and increase to greater than a 1000ms after 250ish iterations!?!?!??!!??!
+        study.add_trials(trials)
+        return study
+
+    def get_best_trial_params(self, canonical=False):
+        studies = self.bench_result_to_study(True)
+        out = studies.best_trials[0].params
+        if canonical:
+            return hmap_canonical_input(out)
+        return out
+
+    def get_pareto_front_params(self):
+        return [p.params for p in self.studies[0].trials]
+
+    def collect_optuna_plots(self) -> List[pn.pane.panel]:
+        """Use optuna to plot various summaries of the optimisation
+
+        Args:
+            study (optuna.Study): The study to plot
+            bench_cfg (BenchCfg): Benchmark config with options used to generate the study
+
+        Returns:
+            List[pn.pane.Pane]: A list of plots
+        """
+
+        self.studies = [self.bench_result_to_study(True)]
+        titles = ["# Analysis"]
+        if self.bench_cfg.repeats > 1:
+            self.studies.append(self.bench_result_to_study(False))
+            titles = [
+                "# Parameter Importance With Repeats",
+                "# Parameter Importance Without Repeats",
+            ]
+
+        study_repeats_pane = pn.Row()
+        for study, title in zip(self.studies, titles):
+            study_pane = pn.Column()
+            target_names = self.bench_cfg.optuna_targets()
+            param_str = []
+
+            study_pane.append(pn.pane.Markdown(title))
+
+            if len(target_names) > 1:
+                if len(target_names) <= 3:
+                    study_pane.append(
+                        plot_pareto_front(
+                            study, target_names=target_names, include_dominated_trials=False
+                        )
+                    )
+                else:
+                    print("plotting pareto front of first 3 result variables")
+                    study_pane.append(
+                        plot_pareto_front(
+                            study,
+                            targets=lambda t: (t.values[0], t.values[1], t.values[2]),
+                            target_names=target_names[:3],
+                            include_dominated_trials=False,
+                        )
+                    )
+
+                study_pane.append(param_importance(self.bench_cfg, study))
+                param_str.append(
+                    f"    Number of trials on the Pareto front: {len(study.best_trials)}"
+                )
+                for t in study.best_trials:
+                    param_str.extend(summarise_trial(t, self.bench_cfg))
+
+            else:
+                # cols.append(plot_optimization_history(study)) #TODO, maybe more clever when this is plotted?
+
+                # If there is only 1 parameter then there is no point is plotting relative importance.  Only worth plotting if there are multiple repeats of the same value so that you can compare the parameter vs to repeat to get a sense of the how much chance affects the results
+                # if bench_cfg.repeats > 1 and len(bench_cfg.input_vars) > 1:  #old code, not sure if its right
+                if len(self.bench_cfg.input_vars) > 1:
+                    study_pane.append(plot_param_importances(study, target_name=target_names[0]))
+
+                param_str.extend(summarise_trial(study.best_trial, self.bench_cfg))
+
+            kwargs = {"height": 500, "scroll": True} if len(param_str) > 30 else {}
+
+            param_str = "\n".join(param_str)
+            study_pane.append(
+                pn.Row(pn.pane.Markdown(f"## Best Parameters\n```text\n{param_str}"), **kwargs),
+            )
+
+            study_repeats_pane.append(study_pane)
+
+        return study_repeats_pane
+
+    # def extract_study_to_dataset(study: optuna.Study, bench_cfg: BenchCfg) -> BenchCfg:
+    #     """Extract an optuna study into an xarray dataset for easy plotting
+
+    #     Args:
+    #         study (optuna.Study): The result of a gridsearch
+    #         bench_cfg (BenchCfg): Options for the grid search
+
+    #     Returns:
+    #         BenchCfg: An updated config with the results included
+    #     """
+    #     for t in study.trials:
+    #         for it, rv in enumerate(bench_cfg.result_vars):
+    #             bench_cfg.ds[rv.name].loc[t.params] = t.values[it]
+    #     return bench_cfg
+
+    def deep(self) -> OptunaResult:  # pragma: no cover
+        """Return a deep copy of these results"""
+        return deepcopy(self)
+
+    def set_plot_size(self, **kwargs) -> dict:
+        if "width" not in kwargs:
+            if self.bench_cfg.plot_size is not None:
+                kwargs["width"] = self.bench_cfg.plot_size
+            # specific width overrrides general size
+            if self.bench_cfg.plot_width is not None:
+                kwargs["width"] = self.bench_cfg.plot_width
+
+        if "height" not in kwargs:
+            if self.bench_cfg.plot_size is not None:
+                kwargs["height"] = self.bench_cfg.plot_size
+            # specific height overrrides general size
+            if self.bench_cfg.plot_height is not None:
+                kwargs["height"] = self.bench_cfg.plot_height
+        return kwargs

bencher/results/panel_result.py

@@ -0,0 +1,113 @@
+from typing import Optional, Any
+from pathlib import Path
+from functools import partial
+import panel as pn
+import xarray as xr
+from param import Parameter
+from bencher.results.bench_result_base import BenchResultBase, ReduceType
+from bencher.variables.results import (
+    ResultVideo,
+    ResultContainer,
+    ResultReference,
+    PANEL_TYPES,
+)
+
+
+class PanelResult(BenchResultBase):
+    def to_video(self, **kwargs):
+        return self.map_plots(partial(self.to_video_multi, **kwargs))
+
+    def to_video_multi(self, result_var: Parameter, **kwargs) -> Optional[pn.Column]:
+        if isinstance(result_var, (ResultVideo, ResultContainer)):
+            vid_p = []
+
+            xr_dataset = self.to_hv_dataset(ReduceType.SQUEEZE)
+
+            def to_video_da(da, **kwargs):
+                if da is not None and Path(da).exists():
+                    vid = pn.pane.Video(da, autoplay=True, **kwargs)
+                    vid.loop = True
+                    vid_p.append(vid)
+                    return vid
+                return pn.pane.Markdown(f"video does not exist {da}")
+
+            plot_callback = partial(self.ds_to_container, container=partial(to_video_da, **kwargs))
+
+            panes = self.to_panes_multi_panel(
+                xr_dataset, result_var, plot_callback=plot_callback, target_dimension=0
+            )
+
+            def play_vid(_):  # pragma: no cover
+                for r in vid_p:
+                    r.paused = False
+                    r.loop = False
+
+            def pause_vid(_):  # pragma: no cover
+                for r in vid_p:
+                    r.paused = True
+
+            def reset_vid(_):  # pragma: no cover
+                for r in vid_p:
+                    r.paused = False
+                    r.time = 0
+
+            def loop_vid(_):  # pragma: no cover
+                for r in vid_p:
+                    r.paused = False
+                    r.time = 0
+                    r.loop = True
+
+            button_names = ["Play Videos", "Pause Videos", "Loop Videos", "Reset Videos"]
+            buttom_cb = [play_vid, pause_vid, reset_vid, loop_vid]
+            buttons = pn.Row()
+
+            for name, cb in zip(button_names, buttom_cb):
+                button = pn.widgets.Button(name=name)
+                pn.bind(cb, button, watch=True)
+                buttons.append(button)
+
+            return pn.Column(buttons, panes)
+        return None
+
+    def zero_dim_da_to_val(self, da_ds: xr.DataArray | xr.Dataset) -> Any:
+        # todo this is really horrible, need to improve
+        dim = None
+        if isinstance(da_ds, xr.Dataset):
+            dim = list(da_ds.keys())[0]
+            da = da_ds[dim]
+        else:
+            da = da_ds
+
+        for k in da.coords.keys():
+            dim = k
+            break
+        if dim is None:
+            return da_ds.values.squeeze().item()
+        return da.expand_dims(dim).values[0]
+
+    def ds_to_container(
+        self, dataset: xr.Dataset, result_var: Parameter, container, **kwargs
+    ) -> Any:
+        val = self.zero_dim_da_to_val(dataset[result_var.name])
+        if isinstance(result_var, ResultReference):
+            ref = self.object_index[val]
+            val = ref.obj
+            if ref.container is not None:
+                return ref.container(val, **kwargs)
+        if container is not None:
+            return container(val, styles={"background": "white"}, **kwargs)
+        return val
+
+    def to_panes(
+        self, result_var: Parameter = None, target_dimension: int = 0, container=None, **kwargs
+    ) -> Optional[pn.pane.panel]:
+        if container is None:
+            container = pn.pane.panel
+        return self.map_plot_panes(
+            partial(self.ds_to_container, container=container),
+            hv_dataset=self.to_hv_dataset(ReduceType.SQUEEZE),
+            target_dimension=target_dimension,
+            result_var=result_var,
+            result_types=PANEL_TYPES,
+            **kwargs,
+        )

bencher/results/plotly_result.py

@@ -0,0 +1,65 @@
+import panel as pn
+import plotly.graph_objs as go
+from typing import Optional
+import xarray as xr
+
+from param import Parameter
+
+from bencher.plotting.plot_filter import VarRange
+from bencher.results.bench_result_base import BenchResultBase, ReduceType
+from bencher.variables.results import ResultVar
+
+
+class PlotlyResult(BenchResultBase):
+    def to_volume(self, result_var: Parameter = None, **kwargs):
+        return self.filter(
+            self.to_volume_da,
+            float_range=VarRange(3, 3),
+            cat_range=VarRange(-1, 0),
+            reduce=ReduceType.REDUCE,
+            target_dimension=3,
+            result_var=result_var,
+            result_types=(ResultVar),
+            **kwargs,
+        )
+
+    def to_volume_da(
+        self, dataset: xr.Dataset, result_var: Parameter, width=600, height=600
+    ) -> Optional[pn.pane.Plotly]:
+        """Given a benchCfg generate a 3D surface plot
+        Returns:
+            pn.pane.Plotly: A 3d volume plot as a holoview in a pane
+        """
+        x = self.bench_cfg.input_vars[0]
+        y = self.bench_cfg.input_vars[1]
+        z = self.bench_cfg.input_vars[2]
+        opacity = 0.1
+        meandf = dataset[result_var.name].to_dataframe().reset_index()
+        data = [
+            go.Volume(
+                x=meandf[x.name],
+                y=meandf[y.name],
+                z=meandf[z.name],
+                value=meandf[result_var.name],
+                isomin=meandf[result_var.name].min(),
+                isomax=meandf[result_var.name].max(),
+                opacity=opacity,
+                surface_count=20,
+            )
+        ]
+
+        layout = go.Layout(
+            title=f"{result_var.name} vs ({x.name} vs {y.name} vs {z.name})",
+            width=width,
+            height=height,
+            margin=dict(t=50, b=50, r=50, l=50),
+            scene=dict(
+                xaxis_title=f"{x.name} [{x.units}]",
+                yaxis_title=f"{y.name} [{y.units}]",
+                zaxis_title=f"{z.name} [{z.units}]",
+            ),
+        )
+
+        fig = dict(data=data, layout=layout)
+
+        return pn.pane.Plotly(fig, name="volume_plotly")

bencher/utils.py

@@ -0,0 +1,148 @@
+from collections import namedtuple
+import xarray as xr
+from sortedcontainers import SortedDict
+import hashlib
+import re
+import math
+from colorsys import hsv_to_rgb
+from pathlib import Path
+from uuid import uuid4
+from functools import partial
+from typing import Callable, Any
+
+
+def hmap_canonical_input(dic: dict) -> tuple:
+    """From a dictionary of kwargs, return a hashable representation (tuple) that is always the same for the same inputs and retains the order of the input arguments.  e.g, {x=1,y=2} -> (1,2) and {y=2,x=1} -> (1,2).  This is used so that keywords arguments can be hashed and converted the the tuple keys that are used for holomaps
+
+    Args:
+        dic (dict): dictionary with keyword arguments and values in any order
+
+    Returns:
+        tuple: values of the dictionary always in the same order and hashable
+    """
+
+    function_input = SortedDict(dic)
+    return tuple(function_input.values())
+
+
+def make_namedtuple(class_name: str, **fields) -> namedtuple:
+    """Convenience method for making a named tuple
+
+    Args:
+        class_name (str): name of the named tuple
+
+    Returns:
+        namedtuple: a named tuple with the fields as values
+    """
+    return namedtuple(class_name, fields)(*fields.values())
+
+
+def get_nearest_coords(dataset: xr.Dataset, collapse_list=False, **kwargs) -> dict:
+    """Given an xarray dataset and kwargs of key value pairs of coordinate values, return a dictionary of the nearest coordinate name value pair that was found in the dataset
+
+    Args:
+        ds (xr.Dataset): dataset
+
+    Returns:
+        dict: nearest coordinate name value pair that matches the input coordinate name value pairs.
+    """
+
+    selection = dataset.sel(method="nearest", **kwargs)
+    cd = selection.coords.to_dataset().to_dict()["coords"]
+    cd2 = {}
+    for k, v in cd.items():
+        cd2[k] = v["data"]
+        if collapse_list and isinstance(cd2[k], list):
+            cd2[k] = cd2[k][0]  # select the first item in the list
+    return cd2
+
+
+def get_nearest_coords1D(val: Any, coords) -> Any:
+    if isinstance(val, (int, float)):
+        return min(coords, key=lambda x_: abs(x_ - val))
+    return val
+
+
+def hash_sha1(var: any) -> str:
+    """A hash function that avoids the PYTHONHASHSEED 'feature' which returns a different hash value each time the program is run"""
+    return hashlib.sha1(str(var).encode("ASCII")).hexdigest()
+
+
+def capitalise_words(message: str):
+    """Given a string of lowercase words, capitalise them
+
+    Args:
+        message (str): lower case string
+
+    Returns:
+        _type_: capitalised string
+    """
+    capitalized_message = " ".join([word.capitalize() for word in message.split(" ")])
+    return capitalized_message
+
+
+def un_camel(camel: str) -> str:
+    """Given a snake_case string return a CamelCase string
+
+    Args:
+        camel (str): camelcase string
+
+    Returns:
+        str: uncamelcased string
+    """
+
+    return capitalise_words(re.sub("([a-z])([A-Z])", r"\g<1> \g<2>", camel.replace("_", " ")))
+
+
+def int_to_col(int_val, sat=0.5, val=0.95, alpha=-1) -> tuple[float, float, float]:
+    """Uses the golden angle to generate colors programmatically with minimum overlap between colors.
+    https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
+
+    Args:
+        int_val (_type_): index of an object you want to color, this is mapped to hue in HSV
+        sat (float, optional): saturation in HSV. Defaults to 0.5.
+        val (float, optional): value in HSV. Defaults to 0.95.
+        alpha (int, optional): transparency.  If -1 then only RGB is returned, if 0 or greater, RGBA is returned. Defaults to -1.
+
+    Returns:
+        tuple[float, float, float] | tuple[float, float, float, float]: either RGB or RGBA vector
+    """
+    golden_ratio_conjugate = (1 + math.sqrt(5)) / 2
+    rgb = hsv_to_rgb(int_val * golden_ratio_conjugate, sat, val)
+    if alpha >= 0:
+        return (*rgb, alpha)
+    return rgb
+
+
+def lerp(value, input_low: float, input_high: float, output_low: float, output_high: float):
+    input_low = float(input_low)
+    return output_low + ((float(value) - input_low) / (float(input_high) - input_low)) * (
+        float(output_high) - output_low
+    )
+
+
+def color_tuple_to_css(color: tuple[float, float, float]) -> str:
+    return f"rgb{(color[0] * 255, color[1] * 255, color[2] * 255)}"
+
+
+def gen_path(filename, folder, suffix):
+    path = Path(f"cachedir/{folder}/{filename}/")
+    path.mkdir(parents=True, exist_ok=True)
+    return f"{path.absolute().as_posix()}/{filename}_{uuid4()}{suffix}"
+
+
+def gen_video_path(video_name: str, extension: str = ".webm") -> str:
+    return gen_path(video_name, "vid", extension)
+
+
+def gen_image_path(image_name: str, filetype=".png") -> str:
+    return gen_path(image_name, "img", filetype)
+
+
+def callable_name(any_callable: Callable[..., Any]) -> str:
+    if isinstance(any_callable, partial):
+        return any_callable.func.__name__
+    try:
+        return any_callable.__name__
+    except AttributeError:
+        return str(any_callable)