holobench 1.25.2__py3-none-any.whl → 1.27.0__py3-none-any.whl

bencher/example/optuna/example_optuna.py

@@ -0,0 +1,78 @@
+import bencher as bch
+import numpy as np
+import optuna
+from optuna.samplers import TPESampler
+
+
+def objective(trial):
+    x = trial.suggest_float("x", -10, 10)
+    return x**2
+
+
+study = optuna.create_study(sampler=TPESampler())
+study.optimize(objective, n_trials=10)
+
+rast_range = 1.5
+
+optimal_value = 0.1234
+
+
+class ToyOptimisationProblem(bch.ParametrizedSweep):
+    input1 = bch.FloatSweep(default=0, bounds=[-rast_range, rast_range], samples=10)
+    input2 = bch.FloatSweep(default=0, bounds=[-rast_range, rast_range], samples=10)
+    offset = bch.FloatSweep(default=0, bounds=[-rast_range, rast_range])
+
+    bump_scale = bch.FloatSweep(default=1.5, bounds=[1, 10])
+
+    # RESULTS
+    output = bch.ResultVar("ul", bch.OptDir.minimize)
+
+    def rastrigin(self, **kwargs) -> dict:
+        """A modified version of the rastrigin function which is very difficult to find the global optimum
+        https://en.wikipedia.org/wiki/Rastrigin_function
+
+        Returns:
+            dict: dictionary of return values
+        """
+        self.update_params_from_kwargs(**kwargs)
+        x = np.array([self.input1 + optimal_value, self.input2 + optimal_value])
+
+        self.output = (
+            np.sum(x * x - self.bump_scale * np.cos(self.bump_scale * np.pi * x))
+            + self.bump_scale * np.size(x)
+            + self.offset
+        )
+        return self.get_results_values_as_dict()
+
+
+def optuna_rastrigin(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+):
+    explorer = ToyOptimisationProblem()
+
+    bench = bch.Bench("Rastrigin", explorer.rastrigin, run_cfg=run_cfg, report=report)
+
+    # res = bench.to_optuna(
+    #     input_vars=[explorer.param.input1, explorer.param.input2],
+    #     result_vars=[explorer.param.output],
+    #     n_trials=30
+    # )
+
+    # run_cfg.use_optuna = True
+    res = bench.plot_sweep(
+        "Rastrigin",
+        input_vars=[explorer.param.input1, explorer.param.input2],
+        result_vars=[explorer.param.output],
+        run_cfg=run_cfg,
+    )
+
+    bench.report.append(res.to_optuna_plots())
+    bench.report.append(res.to_optuna_from_sweep(bench))
+    bench.report.append_markdown(
+        f"The optimal value should be input1:{-optimal_value},input2:{-optimal_value} with a value of 0"
+    )
+    return bench
+
+
+if __name__ == "__main__":
+    optuna_rastrigin().report.show()

bencher/example/shelved/example_float2D_scatter.py

@@ -0,0 +1,109 @@
+# import random
+
+# import bencher as bch
+
+
+# class GaussianDist(bch.ParametrizedSweep):
+#     """A class to represent a gaussian distribution."""
+
+#     mean = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], doc="mean of the gaussian distribution", samples=3
+#     )
+#     sigma = bch.FloatSweep(
+#         default=1, bounds=[0, 1.0], doc="standard deviation of gaussian distribution", samples=4
+#     )
+
+
+# class Example2DGaussianResult(bch.ParametrizedSweep):
+#     """A class to represent the properties of a volume sample."""
+
+#     gauss_x = bch.ResultVar("m", doc="x value of the 2D gaussian")
+#     gauss_y = bch.ResultVar("m", doc="y value of the 2D gaussian")
+
+#     point2D = bch.ResultVec(2, "m", doc="2D vector of the point")
+
+
+# def bench_fn(dist: GaussianDist) -> Example2DGaussianResult:
+#     """This function samples a point from a gaussian distribution.
+
+#     Args:
+#         dist (GaussianDist): Sample point
+
+#     Returns:
+#         Example2DGaussianResult: Value at that point
+#     """
+#     output = Example2DGaussianResult()
+
+#     output.gauss_x = random.gauss(dist.mean, dist.sigma)
+#     output.gauss_y = random.gauss(dist.mean, dist.sigma)
+#     output.point2D = [output.gauss_x, output.gauss_y]
+
+#     return output
+
+
+# def example_floats2D_scatter(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+#     """Example of how to perform a 3D floating point parameter sweep
+
+#     Args:
+#         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+#     Returns:
+#         Bench: results of the parameter sweep
+#     """
+#     bench = bch.Bench(
+#         "Bencher_Example_Floats_Scatter", bench_fn, GaussianDist, plot_lib=bch.PlotLibrary.default()
+#     )
+
+#     bench.plot_sweep(
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter Example",
+#         run_cfg=run_cfg,
+#     )
+
+#     bench.plot_sweep(
+#         input_vars=[GaussianDist.param.mean],
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter With Changing Mean",
+#         run_cfg=run_cfg,
+#     )
+
+#     bench.plot_sweep(
+#         input_vars=[GaussianDist.param.sigma],
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter With Changing Sigma",
+#         run_cfg=run_cfg,
+#     )
+
+#     # future work
+#     # bench.plot_sweep(
+#     #     input_vars=[GaussianDist.param.mean, GaussianDist.param.sigma],
+#     #     result_vars=[
+#     #         GaussianResult.param.point2D,
+#     #         GaussianResult.param.gauss_x,
+#     #         GaussianResult.param.gauss_y,
+#     #     ],
+#     #     title="Float 2D Scatter With Changing Sigma",
+#     #     run_cfg=run_cfg,
+#     # )
+
+#     return bench
+
+
+# if __name__ == "__main__":
+#     ex_run_cfg = bch.BenchRunCfg()
+#     ex_run_cfg.repeats = 50
+#     ex_run_cfg.over_time = True
+#     # ex_run_cfg.clear_history = True
+#     example_floats2D_scatter(ex_run_cfg).report.show()

bencher/example/shelved/example_float3D_cone.py

@@ -0,0 +1,96 @@
+# import numpy as np
+
+# import bencher as bch
+
+
+# class VolumeSample(bch.ParametrizedSweep):
+#     """A class to represent a 3D point in space."""
+
+#     pos_samples = 1
+
+#     x = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="x coordinate of the sample volume"
+#     )
+#     y = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="y coordinate of the sample volume"
+#     )
+#     z = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="z coordinate of the sample volume"
+#     )
+
+#     vec_samples = 5
+
+#     vec_x = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="x coordinate of the sample volume"
+#     )
+#     vec_y = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="y coordinate of the sample volume"
+#     )
+#     vec_z = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="z coordinate of the sample volume"
+#     )
+
+
+# class VolumeResult(bch.ParametrizedSweep):
+#     """A class to represent the properties of a volume sample."""
+
+#     vec_dir = bch.ResultVec(3, "vec", doc="A vector field with an interesting shape")
+
+
+# def bench_fn(point: VolumeSample, normalise=False) -> VolumeResult:
+#     """This function takes a 3D point as input and returns distance of that point to the origin.
+
+#     Args:
+#         point (VolumeSample): Sample point
+
+#     Returns:
+#         VolumeResult: Value at that point
+#     """
+#     output = VolumeResult()
+
+#     vec = np.array([point.vec_x, point.vec_y, point.vec_z])
+
+#     if normalise:
+#         norm = np.linalg.norm(vec)
+#         if norm > 0:
+#             vec /= norm
+
+#     output.vec_dir = list(vec)
+#     return output
+
+
+# def example_cone(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+#     """Example of how to perform a 3D floating point parameter sweep
+
+#     Args:
+#         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+#     Returns:
+#         Bench: results of the parameter sweep
+#     """
+#     bench = bch.Bench("Bencher_Example_Cone", bench_fn, VolumeSample)
+
+#     bench.plot_sweep(
+#         input_vars=[
+#             VolumeSample.param.x,
+#             VolumeSample.param.y,
+#             VolumeSample.param.z,
+#             VolumeSample.param.vec_x,
+#             VolumeSample.param.vec_y,
+#             VolumeSample.param.vec_z,
+#         ],
+#         result_vars=[
+#             VolumeResult.param.vec_dir,
+#         ],
+#         title="Float 3D cone Example",
+#         description="""This example shows how to sample 3 floating point variables and plot a vector field representation of the results.  The benchmark function returns the distance to the origin""",
+#         run_cfg=run_cfg,
+#     )
+
+#     return bench
+
+
+# if __name__ == "__main__":
+#     ex_run_cfg = bch.BenchRunCfg()
+#     ex_run_cfg.use_cache = True
+#     example_cone(ex_run_cfg).report.show()

bencher/example/shelved/example_kwargs.py

@@ -0,0 +1,63 @@
+# import math
+
+# import bencher as bch
+
+
+# def bench_function(
+#     theta: float = 0,
+#     offset: float = 0,
+#     scale: float = 1.0,
+#     trig_func: str = "sin",
+#     **kwargs,  # pylint: disable=unused-argument
+# ) -> dict:
+#     """All the other examples use classes and parameters to define the inputs and outputs to the function. However it makes the code less flexible when integrating with other systems, so this example shows a more basic interface that accepts and returns dictionaries.  The classes still need to be defined however because that is how the sweep and plotting settings are calcuated"""
+#     output = {}
+
+#     if trig_func == "sin":
+#         output["voltage"] = offset + math.sin(theta) * scale
+#     elif trig_func == "cos":
+#         output["voltage"] = offset + math.cos(theta) * scale
+
+#     return output
+
+
+# class InputCfg(bch.ParametrizedSweep):
+#     """This class is used to define the default values and bounds of the variables to benchmark."""
+
+#     theta = bch.FloatSweep(
+#         default=0.0,
+#         bounds=[0.0, 6.0],
+#         doc="Input angle to the trig function",
+#         units="rad",
+#         samples=10,
+#     )
+
+#     offset = bch.FloatSweep(
+#         default=0.0,
+#         bounds=[0.0, 3.0],
+#         doc="Add an offset voltage to the result of the trig function",
+#         units="v",
+#         samples=5,
+#     )
+
+#     trig_func = bch.StringSweep(["sin", "cos"], doc="Select what trigonometric function use")
+
+
+# class OutputVoltage(bch.ParametrizedSweep):
+#     voltage = bch.ResultVar(units="v", doc="Output voltage")
+
+
+# if __name__ == "__main__":
+#     # pass the objective function you have defined to bencher.  The other examples pass the InputCfg type, but this benchmark function accepts a kwargs dictionary so you don't need to pass the inputCfg type.
+#     bench = bch.Bench("Bencher_Example_Categorical", bench_function)
+
+#     # Bencher needs to know the metadata of the variable in order to automatically sweep and plot it, so it is passed by using param's metadata syntax.  InputCfg.param.* is how to access the metadata defined in the class description.
+#     bench.plot_sweep(
+#         input_vars=[InputCfg.param.theta, InputCfg.param.offset],
+#         result_vars=[OutputVoltage.param.voltage],
+#         title="Example with kwarg inputs and dict output",
+#         description=bench_function.__doc__,
+#     )
+
+#     # launch web server and view
+#     bench.report.show()

bencher/plotting/plot_filter.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+from typing import Optional
+from dataclasses import dataclass
+from bencher.plotting.plt_cnt_cfg import PltCntCfg
+import panel as pn
+
+
+class VarRange:
+    """A VarRange represents the bounded and unbounded ranges of integers.  This class is used to define filters for various variable types.  For example by defining cat_var = VarRange(0,0), calling matches(0) will return true, but any other integer will not match.  You can also have unbounded ranges for example VarRange(2,None) will match to 2,3,4... up to infinity. for By default the lower and upper bounds are set to -1 so so that no matter what value is passsed to matches() will return false.  Matches only takes 0 and positive integers."""
+
+    def __init__(self, lower_bound: int = 0, upper_bound: int = -1) -> None:
+        """
+        Args:
+            lower_bound (int, optional): The smallest acceptable value to matches(). Passing None will result in a lower bound of 0 (as matches only accepts positive integers). Defaults to 0.
+            upper_bound (int, optional): The largest acceptable value to matches().  Passing None will result in no upper bound. Defaults to -1 which results in a range with no matches.
+        """
+        self.lower_bound = lower_bound
+        self.upper_bound = upper_bound
+
+    def matches(self, val: int) -> bool:
+        """Checks that a value is within the variable range.  lower_bound and upper_bound are inclusive (lower_bound<=val<=upper_bound )
+
+        Args:
+            val (int): A positive integer representing a number of items
+
+        Returns:
+            bool: True if the items is within the range, False otherwise.
+
+        Raises:
+            ValueError: If val < 0
+        """
+        if val < 0:
+            raise ValueError("val must be >= 0")
+        if self.lower_bound is not None:
+            lower_match = val >= self.lower_bound
+        else:
+            lower_match = True
+
+        if self.upper_bound is not None:
+            upper_match = val <= self.upper_bound
+        else:
+            upper_match = True
+
+        return lower_match and upper_match
+
+    def matches_info(self, val, name):
+        match = self.matches(val)
+        info = f"{name}\t{match}\t{self.lower_bound}>= {val} <={self.upper_bound}"
+        return match, info
+
+    def __str__(self) -> str:
+        return f"VarRange(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})"
+
+
+@dataclass
+class PlotFilter:
+    """A class for representing the types of results a plot is able to represent."""
+
+    float_range: VarRange = VarRange()
+    cat_range: VarRange = VarRange()
+    vector_len: VarRange = VarRange(1, 1)
+    result_vars: VarRange = VarRange(1, 1)
+    panel_range: VarRange = VarRange(0, 0)
+    repeats_range: VarRange = VarRange(1, None)
+    input_range: VarRange = VarRange(1, None)
+
+    def matches_result(self, plt_cnt_cfg: PltCntCfg, plot_name: str) -> PlotMatchesResult:
+        """Checks if the result data signature matches the type of data the plot is able to display."""
+        return PlotMatchesResult(self, plt_cnt_cfg, plot_name)
+
+
+# @dataclass
+class PlotMatchesResult:
+    """Stores information about which properites match the requirements of a particular plotter"""
+
+    def __init__(self, plot_filter: PlotFilter, plt_cnt_cfg: PltCntCfg, plot_name: str):
+        match_info = []
+        matches = []
+
+        match_candidates = [
+            (plot_filter.float_range, plt_cnt_cfg.float_cnt, "float"),
+            (plot_filter.cat_range, plt_cnt_cfg.cat_cnt, "cat"),
+            (plot_filter.vector_len, plt_cnt_cfg.vector_len, "vec"),
+            (plot_filter.result_vars, plt_cnt_cfg.result_vars, "results"),
+            (plot_filter.panel_range, plt_cnt_cfg.panel_cnt, "panels"),
+            (plot_filter.repeats_range, plt_cnt_cfg.repeats, "repeats"),
+            (plot_filter.input_range, plt_cnt_cfg.inputs_cnt, "inputs"),
+        ]
+
+        for m, cnt, name in match_candidates:
+            match, info = m.matches_info(cnt, name)
+            matches.append(match)
+            if not match:
+                match_info.append(info)
+
+        self.overall = all(matches)
+
+        match_info.insert(0, f"plot {plot_name} matches: {self.overall}")
+        self.matches_info = "\n".join(match_info).strip()
+        self.plt_cnt_cfg = plt_cnt_cfg
+
+        if self.plt_cnt_cfg.print_debug:
+            print(f"checking {plot_name} result: {self.overall}")
+            if not self.overall:
+                print(self.matches_info)
+
+    def to_panel(self, **kwargs) -> Optional[pn.pane.Markdown]:
+        if self.plt_cnt_cfg.print_debug:
+            return pn.pane.Markdown(self.matches_info, **kwargs)
+        return None

bencher/plotting/plt_cnt_cfg.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+import param
+from bencher.bench_cfg import BenchCfg
+from bencher.variables.results import PANEL_TYPES
+
+from bencher.variables.inputs import IntSweep, FloatSweep, BoolSweep, EnumSweep, StringSweep
+from bencher.variables.time import TimeSnapshot
+
+
+class PltCntCfg(param.Parameterized):
+    """Plot Count Config"""
+
+    float_vars = param.List(doc="A list of float vars in order of plotting, x then y")
+    float_cnt = param.Integer(0, doc="The number of float variables to plot")
+    cat_vars = param.List(doc="A list of categorical values to plot in order hue,row,col")
+    cat_cnt = param.Integer(0, doc="The number of cat variables")
+    vector_len = param.Integer(1, doc="The vector length of the return variable , scalars = len 1")
+    result_vars = param.Integer(1, doc="The number result variables to plot")  # todo remove
+    panel_vars = param.List(doc="A list of panel results")
+    panel_cnt = param.Integer(0, doc="Number of results reprented as panel panes")
+    repeats = param.Integer(0, doc="The number of repeat samples")
+    inputs_cnt = param.Integer(0, doc="The number of repeat samples")
+
+    print_debug = param.Boolean(
+        True, doc="Print debug information about why a filter matches this config or not"
+    )
+
+    @staticmethod
+    def generate_plt_cnt_cfg(
+        bench_cfg: BenchCfg,
+    ) -> PltCntCfg:
+        """Given a BenchCfg work out how many float and cat variables there are and store in a PltCntCfg class
+
+        Args:
+            bench_cfg (BenchCfg): See BenchCfg definition
+
+        Raises:
+            ValueError: If no plotting procedure could be automatically detected
+
+        Returns:
+            PltCntCfg: see PltCntCfg definition
+        """
+        plt_cnt_cfg = PltCntCfg()
+        # plt_cnt_cfg.float_vars = deepcopy(bench_cfg.iv_time)
+
+        plt_cnt_cfg.cat_vars = []
+        plt_cnt_cfg.float_vars = []
+
+        for iv in bench_cfg.input_vars:
+            type_allocated = False
+            if isinstance(iv, (IntSweep, FloatSweep, TimeSnapshot)):
+                # if "IntSweep" in typestr or "FloatSweep" in typestr:
+                plt_cnt_cfg.float_vars.append(iv)
+                type_allocated = True
+            if isinstance(iv, (EnumSweep, BoolSweep, StringSweep)):
+                # if "EnumSweep" in typestr or "BoolSweep" in typestr or "StringSweep" in typestr:
+                plt_cnt_cfg.cat_vars.append(iv)
+                type_allocated = True
+
+            if not type_allocated:
+                raise ValueError(f"No rule for type {type(iv)}")
+
+        for rv in bench_cfg.result_vars:
+            if isinstance(rv, PANEL_TYPES):
+                plt_cnt_cfg.panel_vars.append(rv)
+
+        plt_cnt_cfg.float_cnt = len(plt_cnt_cfg.float_vars)
+        plt_cnt_cfg.cat_cnt = len(plt_cnt_cfg.cat_vars)
+        plt_cnt_cfg.panel_cnt = len(plt_cnt_cfg.panel_vars)
+        plt_cnt_cfg.repeats = bench_cfg.repeats
+        plt_cnt_cfg.inputs_cnt = len(bench_cfg.input_vars)
+        return plt_cnt_cfg
+
+    def __str__(self):
+        return f"float_cnt: {self.float_cnt}\ncat_cnt: {self.cat_cnt} \npanel_cnt: {self.panel_cnt}\nvector_len: {self.vector_len}"

bencher/results/bench_result.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+from typing import List
+import panel as pn
+
+from bencher.results.bench_result_base import EmptyContainer
+from bencher.results.video_summary import VideoSummaryResult
+from bencher.results.panel_result import PanelResult
+from bencher.results.plotly_result import PlotlyResult
+from bencher.results.holoview_result import HoloviewResult
+from bencher.utils import listify
+
+
+class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult):
+    """Contains the results of the benchmark and has methods to cast the results to various datatypes and graphical representations"""
+
+    def __init__(self, bench_cfg) -> None:
+        PlotlyResult.__init__(self, bench_cfg)
+        HoloviewResult.__init__(self, bench_cfg)
+
+    @staticmethod
+    def default_plot_callbacks():
+        return [
+            # VideoSummaryResult.to_video_summary, #quite expensive so not turned on by default
+            HoloviewResult.to_bar,
+            HoloviewResult.to_scatter_jitter,
+            HoloviewResult.to_curve,
+            HoloviewResult.to_line,
+            HoloviewResult.to_heatmap,
+            PlotlyResult.to_volume,
+            # PanelResult.to_video,
+            PanelResult.to_panes,
+        ]
+
+    @staticmethod
+    def plotly_callbacks():
+        return [HoloviewResult.to_surface, PlotlyResult.to_volume]
+
+    def plot(self) -> pn.panel:
+        """Plots the benchresult using the plot callbacks defined by the bench run
+
+        Returns:
+             pn.panel: A panel representation of the results
+        """
+        if self.bench_cfg.plot_callbacks is not None:
+            return pn.Column(*[cb(self) for cb in self.bench_cfg.plot_callbacks])
+        return None
+
+    def to_auto(
+        self,
+        plot_list: List[callable] = None,
+        remove_plots: List[callable] = None,
+        **kwargs,
+    ) -> List[pn.panel]:
+        self.plt_cnt_cfg.print_debug = False
+        plot_list = listify(plot_list)
+        remove_plots = listify(remove_plots)
+
+        if plot_list is None:
+            plot_list = BenchResult.default_plot_callbacks()
+        if remove_plots is not None:
+            for p in remove_plots:
+                plot_list.remove(p)
+
+        kwargs = self.set_plot_size(**kwargs)
+
+        row = EmptyContainer(pn.Row())
+        for plot_callback in plot_list:
+            if self.plt_cnt_cfg.print_debug:
+                print(f"checking: {plot_callback.__name__}")
+            # the callbacks are passed from the static class definition, so self needs to be passed before the plotting callback can be called
+            row.append(plot_callback(self, **kwargs))
+
+        self.plt_cnt_cfg.print_debug = True
+        if len(row.pane) == 0:
+            row.append(
+                pn.pane.Markdown("No Plotters are able to represent these results", **kwargs)
+            )
+        return row.pane
+
+    def to_auto_plots(self, **kwargs) -> pn.panel:
+        """Given the dataset result of a benchmark run, automatically dedeuce how to plot the data based on the types of variables that were sampled
+
+        Args:
+            bench_cfg (BenchCfg): Information on how the benchmark was sampled and the resulting data
+
+        Returns:
+            pn.pane: A panel containing plot results
+        """
+
+        plot_cols = pn.Column()
+        plot_cols.append(self.to_sweep_summary(name="Plots View"))
+        plot_cols.append(self.to_auto(**kwargs))
+        plot_cols.append(self.bench_cfg.to_post_description())
+        return plot_cols