holobench 1.3.4__py3-none-any.whl → 1.22.2__py3-none-any.whl

bencher/results/optuna_result.py

@@ -1,332 +0,0 @@
-from __future__ import annotations
-from typing import List
-import numpy as np
-import optuna
-import panel as pn
-from collections import defaultdict
-from textwrap import wrap
-
-import pandas as pd
-import xarray as xr
-
-
-from optuna.visualization import (
-    plot_param_importances,
-    plot_pareto_front,
-)
-from bencher.utils import hmap_canonical_input
-from bencher.variables.time import TimeSnapshot, TimeEvent
-from bencher.bench_cfg import BenchCfg
-from bencher.plotting.plt_cnt_cfg import PltCntCfg
-
-
-# from bencher.results.bench_result_base import BenchResultBase
-from bencher.optuna_conversions import (
-    sweep_var_to_optuna_dist,
-    summarise_trial,
-    param_importance,
-    optuna_grid_search,
-    summarise_optuna_study,
-    sweep_var_to_suggest,
-)
-
-
-def convert_dataset_bool_dims_to_str(dataset: xr.Dataset) -> xr.Dataset:
-    """Given a dataarray that contains boolean coordinates, conver them to strings so that holoviews loads the data properly
-
-    Args:
-        dataarray (xr.DataArray): dataarray with boolean coordinates
-
-    Returns:
-        xr.DataArray: dataarray with boolean coordinates converted to strings
-    """
-    bool_coords = {}
-    for c in dataset.coords:
-        if dataset.coords[c].dtype == bool:
-            bool_coords[c] = [str(vals) for vals in dataset.coords[c].values]
-
-    if len(bool_coords) > 0:
-        return dataset.assign_coords(bool_coords)
-    return dataset
-
-
-class OptunaResult:
-    def __init__(self, bench_cfg: BenchCfg) -> None:
-        self.bench_cfg = bench_cfg
-        # self.wrap_long_time_labels(bench_cfg)  # todo remove
-        self.ds = xr.Dataset()
-        self.object_index = []
-        self.hmaps = defaultdict(dict)
-        self.result_hmaps = bench_cfg.result_hmaps
-        self.studies = []
-        self.plt_cnt_cfg = PltCntCfg()
-        self.plot_inputs = []
-
-        # self.width=600/
-        # self.height=600
-
-        #   bench_res.objects.append(rv)
-        # bench_res.reference_index = len(bench_res.objects)
-
-    def post_setup(self):
-        self.plt_cnt_cfg = PltCntCfg.generate_plt_cnt_cfg(self.bench_cfg)
-        self.bench_cfg = self.wrap_long_time_labels(self.bench_cfg)
-        self.ds = convert_dataset_bool_dims_to_str(self.ds)
-
-    def to_xarray(self) -> xr.Dataset:
-        return self.ds
-
-    def setup_object_index(self):
-        self.object_index = []
-
-    def to_pandas(self, reset_index=True) -> pd.DataFrame:
-        """Get the xarray results as a pandas dataframe
-
-        Returns:
-            pd.DataFrame: The xarray results array as a pandas dataframe
-        """
-        ds = self.to_xarray().to_dataframe()
-        if reset_index:
-            return ds.reset_index()
-        return ds
-
-    def wrap_long_time_labels(self, bench_cfg):
-        """Takes a benchCfg and wraps any index labels that are too long to be plotted easily
-
-        Args:
-            bench_cfg (BenchCfg):
-
-        Returns:
-            BenchCfg: updated config with wrapped labels
-        """
-        if bench_cfg.over_time:
-            if self.ds.coords["over_time"].dtype == np.datetime64:
-                # plotly catastrophically fails to plot anything with the default long string representation of time, so convert to a shorter time representation
-                self.ds.coords["over_time"] = [
-                    pd.to_datetime(t).strftime("%d-%m-%y %H-%M-%S")
-                    for t in self.ds.coords["over_time"].values
-                ]
-                # wrap very long time event labels because otherwise the graphs are unreadable
-            if bench_cfg.time_event is not None:
-                self.ds.coords["over_time"] = [
-                    "\n".join(wrap(t, 20)) for t in self.ds.coords["over_time"].values
-                ]
-        return bench_cfg
-
-    def to_optuna_plots(self) -> List[pn.pane.panel]:
-        """Create an optuna summary from the benchmark results
-
-        Returns:
-            List[pn.pane.panel]: A list of optuna plot summarising the benchmark process
-        """
-
-        return self.collect_optuna_plots()
-
-    def to_optuna_from_sweep(self, bench, n_trials=30):
-        optu = self.to_optuna_from_results(
-            bench.worker, n_trials=n_trials, extra_results=bench.results
-        )
-        return summarise_optuna_study(optu)
-
-    def to_optuna_from_results(
-        self,
-        worker,
-        n_trials=100,
-        extra_results: List[OptunaResult] = None,
-        sampler=optuna.samplers.TPESampler(),
-    ):
-        directions = []
-        for rv in self.bench_cfg.optuna_targets(True):
-            directions.append(rv.direction)
-
-        study = optuna.create_study(
-            sampler=sampler, directions=directions, study_name=self.bench_cfg.title
-        )
-
-        # add already calculated results
-        results_list = extra_results if extra_results is not None else [self]
-        for res in results_list:
-            if len(res.ds.sizes) > 0:
-                study.add_trials(res.bench_results_to_optuna_trials(True))
-
-        def wrapped(trial) -> tuple:
-            kwargs = {}
-            for iv in self.bench_cfg.input_vars:
-                kwargs[iv.name] = sweep_var_to_suggest(iv, trial)
-            result = worker(**kwargs)
-            output = []
-            for rv in self.bench_cfg.result_vars:
-                output.append(result[rv.name])
-            return tuple(output)
-
-        study.optimize(wrapped, n_trials=n_trials)
-        return study
-
-    def bench_results_to_optuna_trials(self, include_meta: bool = True) -> optuna.Study:
-        """Convert an xarray dataset to an optuna study so optuna can further optimise or plot the statespace
-
-        Args:
-            bench_cfg (BenchCfg): benchmark config to convert
-
-        Returns:
-            optuna.Study: optuna description of the study
-        """
-        if include_meta:
-            df = self.to_pandas()
-            all_vars = []
-            for v in self.bench_cfg.all_vars:
-                if type(v) != TimeEvent:
-                    all_vars.append(v)
-
-            print("All vars", all_vars)
-        else:
-            all_vars = self.bench_cfg.input_vars
-            # df = self.ds.
-            # if "repeat" in self.
-            # if self.bench_cfg.repeats>1:
-            # df = self.bench_cfg.ds.mean("repeat").to_dataframe().reset_index()
-            # else:
-            df = self.to_pandas().reset_index()
-        # df = self.bench_cfg.ds.mean("repeat").to_dataframe.reset_index()
-        # self.bench_cfg.all_vars
-        # del self.bench_cfg.meta_vars[1]
-
-        trials = []
-        distributions = {}
-        for i in all_vars:
-            distributions[i.name] = sweep_var_to_optuna_dist(i)
-
-        for row in df.iterrows():
-            params = {}
-            values = []
-            for i in all_vars:
-                if type(i) == TimeSnapshot:
-                    if type(row[1][i.name]) == np.datetime64:
-                        params[i.name] = row[1][i.name].timestamp()
-                else:
-                    params[i.name] = row[1][i.name]
-
-            for r in self.bench_cfg.optuna_targets():
-                values.append(row[1][r])
-
-            trials.append(
-                optuna.trial.create_trial(
-                    params=params,
-                    distributions=distributions,
-                    values=values,
-                )
-            )
-        return trials
-
-    def bench_result_to_study(self, include_meta: bool) -> optuna.Study:
-        trials = self.bench_results_to_optuna_trials(include_meta)
-        study = optuna_grid_search(self.bench_cfg)
-        optuna.logging.set_verbosity(optuna.logging.CRITICAL)
-        import warnings
-
-        # /usr/local/lib/python3.10/dist-packages/optuna/samplers/_grid.py:224: UserWarning: over_time contains a value with the type of <class 'pandas._libs.tslibs.timestamps.Timestamp'>, which is not supported by `GridSampler`. Please make sure a value is `str`, `int`, `float`, `bool` or `None` for persistent storage.
-
-        # this is not disabling the warning
-        warnings.filterwarnings(action="ignore", category=UserWarning)
-        # remove optuna gridsearch warning as we are not using their gridsearch because it has the most inexplicably terrible performance I have ever seen in my life. How can a for loop of 400 iterations start out with 100ms per loop and increase to greater than a 1000ms after 250ish iterations!?!?!??!!??!
-        study.add_trials(trials)
-        return study
-
-    def get_best_trial_params(self, canonical=False):
-        studies = self.bench_result_to_study(True)
-        out = studies.best_trials[0].params
-        if canonical:
-            return hmap_canonical_input(out)
-        return out
-
-    def get_pareto_front_params(self):
-        return [p.params for p in self.studies[0].trials]
-
-    def collect_optuna_plots(self) -> List[pn.pane.panel]:
-        """Use optuna to plot various summaries of the optimisation
-
-        Args:
-            study (optuna.Study): The study to plot
-            bench_cfg (BenchCfg): Benchmark config with options used to generate the study
-
-        Returns:
-            List[pn.pane.Pane]: A list of plots
-        """
-
-        self.studies = [self.bench_result_to_study(True)]
-        titles = ["# Analysis"]
-        if self.bench_cfg.repeats > 1:
-            self.studies.append(self.bench_result_to_study(False))
-            titles = [
-                "# Parameter Importance With Repeats",
-                "# Parameter Importance Without Repeats",
-            ]
-
-        study_repeats_pane = pn.Row()
-        for study, title in zip(self.studies, titles):
-            study_pane = pn.Column()
-            target_names = self.bench_cfg.optuna_targets()
-            param_str = []
-
-            study_pane.append(pn.pane.Markdown(title))
-
-            if len(target_names) > 1:
-                if len(target_names) <= 3:
-                    study_pane.append(
-                        plot_pareto_front(
-                            study, target_names=target_names, include_dominated_trials=False
-                        )
-                    )
-                else:
-                    print("plotting pareto front of first 3 result variables")
-                    study_pane.append(
-                        plot_pareto_front(
-                            study,
-                            targets=lambda t: (t.values[0], t.values[1], t.values[2]),
-                            target_names=target_names[:3],
-                            include_dominated_trials=False,
-                        )
-                    )
-
-                study_pane.append(param_importance(self.bench_cfg, study))
-                param_str.append(
-                    f"    Number of trials on the Pareto front: {len(study.best_trials)}"
-                )
-                for t in study.best_trials:
-                    param_str.extend(summarise_trial(t, self.bench_cfg))
-
-            else:
-                # cols.append(plot_optimization_history(study)) #TODO, maybe more clever when this is plotted?
-
-                # If there is only 1 parameter then there is no point is plotting relative importance.  Only worth plotting if there are multiple repeats of the same value so that you can compare the parameter vs to repeat to get a sense of the how much chance affects the results
-                # if bench_cfg.repeats > 1 and len(bench_cfg.input_vars) > 1:  #old code, not sure if its right
-                if len(self.bench_cfg.input_vars) > 1:
-                    study_pane.append(plot_param_importances(study, target_name=target_names[0]))
-
-                param_str.extend(summarise_trial(study.best_trial, self.bench_cfg))
-
-            kwargs = {"height": 500, "scroll": True} if len(param_str) > 30 else {}
-
-            param_str = "\n".join(param_str)
-            study_pane.append(
-                pn.Row(pn.pane.Markdown(f"## Best Parameters\n```text\n{param_str}"), **kwargs),
-            )
-
-            study_repeats_pane.append(study_pane)
-
-        return study_repeats_pane
-
-    # def extract_study_to_dataset(study: optuna.Study, bench_cfg: BenchCfg) -> BenchCfg:
-    #     """Extract an optuna study into an xarray dataset for easy plotting
-
-    #     Args:
-    #         study (optuna.Study): The result of a gridsearch
-    #         bench_cfg (BenchCfg): Options for the grid search
-
-    #     Returns:
-    #         BenchCfg: An updated config with the results included
-    #     """
-    #     for t in study.trials:
-    #         for it, rv in enumerate(bench_cfg.result_vars):
-    #             bench_cfg.ds[rv.name].loc[t.params] = t.values[it]
-    #     return bench_cfg

bencher/results/panel_result.py

@@ -1,113 +0,0 @@
-from typing import Optional, Any
-from pathlib import Path
-from functools import partial
-import panel as pn
-import xarray as xr
-from param import Parameter
-from bencher.results.bench_result_base import BenchResultBase, ReduceType
-from bencher.variables.results import (
-    ResultVideo,
-    ResultContainer,
-    ResultReference,
-    PANEL_TYPES,
-)
-
-
-class PanelResult(BenchResultBase):
-    def to_video(self, **kwargs):
-        return self.map_plots(partial(self.to_video_multi, **kwargs))
-
-    def to_video_multi(self, result_var: Parameter, **kwargs) -> Optional[pn.Column]:
-        if isinstance(result_var, (ResultVideo, ResultContainer)):
-            vid_p = []
-
-            xr_dataset = self.to_hv_dataset(ReduceType.SQUEEZE)
-
-            def to_video_da(da, **kwargs):
-                if da is not None and Path(da).exists():
-                    vid = pn.pane.Video(da, autoplay=True, **kwargs)
-                    vid.loop = True
-                    vid_p.append(vid)
-                    return vid
-                return pn.pane.Markdown(f"video does not exist {da}")
-
-            plot_callback = partial(self.ds_to_container, container=partial(to_video_da, **kwargs))
-
-            panes = self.to_panes_multi_panel(
-                xr_dataset, result_var, plot_callback=plot_callback, target_dimension=0
-            )
-
-            def play_vid(_):  # pragma: no cover
-                for r in vid_p:
-                    r.paused = False
-                    r.loop = False
-
-            def pause_vid(_):  # pragma: no cover
-                for r in vid_p:
-                    r.paused = True
-
-            def reset_vid(_):  # pragma: no cover
-                for r in vid_p:
-                    r.paused = False
-                    r.time = 0
-
-            def loop_vid(_):  # pragma: no cover
-                for r in vid_p:
-                    r.paused = False
-                    r.time = 0
-                    r.loop = True
-
-            button_names = ["Play Videos", "Pause Videos", "Loop Videos", "Reset Videos"]
-            buttom_cb = [play_vid, pause_vid, reset_vid, loop_vid]
-            buttons = pn.Row()
-
-            for name, cb in zip(button_names, buttom_cb):
-                button = pn.widgets.Button(name=name)
-                pn.bind(cb, button, watch=True)
-                buttons.append(button)
-
-            return pn.Column(buttons, panes)
-        return None
-
-    def zero_dim_da_to_val(self, da_ds: xr.DataArray | xr.Dataset) -> Any:
-        # todo this is really horrible, need to improve
-        dim = None
-        if isinstance(da_ds, xr.Dataset):
-            dim = list(da_ds.keys())[0]
-            da = da_ds[dim]
-        else:
-            da = da_ds
-
-        for k in da.coords.keys():
-            dim = k
-            break
-        if dim is None:
-            return da_ds.values.squeeze().item()
-        return da.expand_dims(dim).values[0]
-
-    def ds_to_container(
-        self, dataset: xr.Dataset, result_var: Parameter, container, **kwargs
-    ) -> Any:
-        val = self.zero_dim_da_to_val(dataset[result_var.name])
-        if isinstance(result_var, ResultReference):
-            ref = self.object_index[val]
-            val = ref.obj
-            if ref.container is not None:
-                return ref.container(val, **kwargs)
-        if container is not None:
-            return container(val, styles={"background": "white"}, **kwargs)
-        return val
-
-    def to_panes(
-        self, result_var: Parameter = None, target_dimension: int = 0, container=None, **kwargs
-    ) -> Optional[pn.pane.panel]:
-        if container is None:
-            container = pn.pane.panel
-        return self.map_plot_panes(
-            partial(self.ds_to_container, container=container),
-            hv_dataset=self.to_hv_dataset(ReduceType.SQUEEZE),
-            target_dimension=target_dimension,
-            result_var=result_var,
-            result_types=PANEL_TYPES,
-            **kwargs,
-        )

bencher/results/plotly_result.py

@@ -1,65 +0,0 @@
-import panel as pn
-import plotly.graph_objs as go
-from typing import Optional
-import xarray as xr
-
-from param import Parameter
-
-from bencher.plotting.plot_filter import VarRange
-from bencher.results.bench_result_base import BenchResultBase, ReduceType
-from bencher.variables.results import ResultVar
-
-
-class PlotlyResult(BenchResultBase):
-    def to_volume(self, result_var: Parameter = None, **kwargs):
-        return self.filter(
-            self.to_volume_da,
-            float_range=VarRange(3, 3),
-            cat_range=VarRange(-1, 0),
-            reduce=ReduceType.REDUCE,
-            target_dimension=3,
-            result_var=result_var,
-            result_types=(ResultVar),
-            **kwargs,
-        )
-
-    def to_volume_da(
-        self, dataset: xr.Dataset, result_var: Parameter, width=600, height=600
-    ) -> Optional[pn.pane.Plotly]:
-        """Given a benchCfg generate a 3D surface plot
-        Returns:
-            pn.pane.Plotly: A 3d volume plot as a holoview in a pane
-        """
-        x = self.bench_cfg.input_vars[0]
-        y = self.bench_cfg.input_vars[1]
-        z = self.bench_cfg.input_vars[2]
-        opacity = 0.1
-        meandf = dataset[result_var.name].to_dataframe().reset_index()
-        data = [
-            go.Volume(
-                x=meandf[x.name],
-                y=meandf[y.name],
-                z=meandf[z.name],
-                value=meandf[result_var.name],
-                isomin=meandf[result_var.name].min(),
-                isomax=meandf[result_var.name].max(),
-                opacity=opacity,
-                surface_count=20,
-            )
-        ]
-
-        layout = go.Layout(
-            title=f"{result_var.name} vs ({x.name} vs {y.name} vs {z.name})",
-            width=width,
-            height=height,
-            margin=dict(t=50, b=50, r=50, l=50),
-            scene=dict(
-                xaxis_title=f"{x.name} [{x.units}]",
-                yaxis_title=f"{y.name} [{y.units}]",
-                zaxis_title=f"{z.name} [{z.units}]",
-            ),
-        )
-
-        fig = dict(data=data, layout=layout)
-
-        return pn.pane.Plotly(fig, name="volume_plotly")

bencher/variables/inputs.py

@@ -1,193 +0,0 @@
-from enum import Enum
-from typing import List, Any
-
-import numpy as np
-from param import Integer, Number, Selector
-from bencher.variables.sweep_base import SweepBase, shared_slots
-
-
-class SweepSelector(Selector, SweepBase):
-    """A class to reprsent a parameter sweep of bools"""
-
-    __slots__ = shared_slots
-
-    def __init__(self, units: str = "ul", samples: int = None, samples_debug: int = 2, **params):
-        SweepBase.__init__(self)
-        Selector.__init__(self, **params)
-
-        self.units = units
-        if samples is None:
-            self.samples = len(self.objects)
-        else:
-            self.samples = samples
-        self.samples_debug = min(self.samples, samples_debug)
-
-    def values(self, debug=False) -> List[Any]:
-        """return all the values for a parameter sweep.  If debug is true return a reduced list"""
-        return self.indices_to_samples(self.samples_debug if debug else self.samples, self.objects)
-
-
-class BoolSweep(SweepSelector):
-    """A class to reprsent a parameter sweep of bools"""
-
-    def __init__(
-        self, units: str = "ul", samples: int = None, samples_debug: int = 2, default=True, **params
-    ):
-        SweepSelector.__init__(
-            self,
-            units=units,
-            samples=samples,
-            samples_debug=samples_debug,
-            default=default,
-            objects=[True, False] if default else [False, True],
-            **params,
-        )
-
-
-class StringSweep(SweepSelector):
-    """A class to reprsent a parameter sweep of strings"""
-
-    def __init__(
-        self,
-        string_list: List[str],
-        units: str = "",
-        samples: int = None,
-        samples_debug: int = 2,
-        **params,
-    ):
-        SweepSelector.__init__(
-            self,
-            objects=string_list,
-            instantiate=True,
-            units=units,
-            samples=samples,
-            samples_debug=samples_debug,
-            **params,
-        )
-
-
-class EnumSweep(SweepSelector):
-    """A class to reprsent a parameter sweep of enums"""
-
-    __slots__ = shared_slots
-
-    def __init__(
-        self, enum_type: Enum | List[Enum], units="", samples=None, samples_debug=2, **params
-    ):
-        # The enum can either be an Enum type or a list of enums
-        list_of_enums = isinstance(enum_type, list)
-        selector_list = enum_type if list_of_enums else list(enum_type)
-        SweepSelector.__init__(
-            self,
-            objects=selector_list,
-            instantiate=True,
-            units=units,
-            samples=samples,
-            samples_debug=samples_debug,
-            **params,
-        )
-        if not list_of_enums:  # Grab the docs from the enum type def
-            self.doc = enum_type.__doc__
-
-
-class IntSweep(Integer, SweepBase):
-    """A class to reprsent a parameter sweep of ints"""
-
-    __slots__ = shared_slots + ["sample_values"]
-
-    def __init__(self, units="ul", samples=None, samples_debug=2, sample_values=None, **params):
-        SweepBase.__init__(self)
-        Integer.__init__(self, **params)
-
-        self.units = units
-        self.samples_debug = samples_debug
-
-        if sample_values is None:
-            if samples is None:
-                if self.bounds is None:
-                    raise RuntimeError("You must define bounds for integer types")
-                self.samples = 1 + self.bounds[1] - self.bounds[0]
-            else:
-                self.samples = samples
-            self.sample_values = None
-        else:
-            self.sample_values = sample_values
-            self.samples = len(self.sample_values)
-            if "default" not in params:
-                self.default = sample_values[0]
-
-    def values(self, debug=False) -> List[int]:
-        """return all the values for a parameter sweep.  If debug is true return the  list"""
-        sample_values = (
-            self.sample_values
-            if self.sample_values is not None
-            else list(range(int(self.bounds[0]), int(self.bounds[1] + 1)))
-        )
-
-        return self.indices_to_samples(self.samples_debug if debug else self.samples, sample_values)
-
-    ###THESE ARE COPIES OF INTEGER VALIDATION BUT ALSO ALLOW NUMPY INT TYPES
-    def _validate_value(self, val, allow_None):
-        if callable(val):
-            return
-
-        if allow_None and val is None:
-            return
-
-        if not isinstance(val, (int, np.integer)):
-            raise ValueError(
-                "Integer parameter %r must be an integer, " "not type %r." % (self.name, type(val))
-            )
-
-    ###THESE ARE COPIES OF INTEGER VALIDATION BUT ALSO ALLOW NUMPY INT TYPES
-    def _validate_step(self, val, step):
-        if step is not None and not isinstance(step, (int, np.integer)):
-            raise ValueError(
-                "Step can only be None or an " "integer value, not type %r" % type(step)
-            )
-
-
-class FloatSweep(Number, SweepBase):
-    """A class to represent a parameter sweep of floats"""
-
-    __slots__ = shared_slots + ["sample_values"]
-
-    def __init__(
-        self, units="ul", samples=10, samples_debug=2, sample_values=None, step=None, **params
-    ):
-        SweepBase.__init__(self)
-        Number.__init__(self, step=step, **params)
-
-        self.units = units
-        self.samples_debug = samples_debug
-
-        self.sample_values = sample_values
-
-        if sample_values is None:
-            self.samples = samples
-        else:
-            self.samples = len(self.sample_values)
-            if "default" not in params:
-                self.default = sample_values[0]
-
-    def values(self, debug=False) -> List[float]:
-        """return all the values for a parameter sweep.  If debug is true return a reduced list"""
-        samps = self.samples_debug if debug else self.samples
-        if self.sample_values is None:
-            if self.step is None:
-                return np.linspace(self.bounds[0], self.bounds[1], samps)
-
-            return np.arange(self.bounds[0], self.bounds[1], self.step)
-        if debug:
-            indices = [
-                int(i)
-                for i in np.linspace(0, len(self.sample_values) - 1, self.samples_debug, dtype=int)
-            ]
-            return [self.sample_values[i] for i in indices]
-        return self.sample_values
-
-
-def box(name, center, width):
-    var = FloatSweep(default=center, bounds=(center - width, center + width))
-    var.name = name
-    return var