holobench 1.25.2__py3-none-any.whl → 1.27.0__py3-none-any.whl

bencher/example/example_levels.py

@@ -0,0 +1,181 @@
+import bencher as bch
+from bencher.utils import int_to_col
+
+import math
+import holoviews as hv
+from typing import Any, List
+import panel as pn
+from holoviews import opts
+
+
+class LevelsExample(bch.ParametrizedSweep):
+    xval = bch.FloatSweep(bounds=[0, 3.14])
+    yval = bch.FloatSweep(bounds=[0, 3.14])
+    level = bch.IntSweep(default=1, bounds=[1, 6])
+
+    output = bch.ResultVar(units="v")
+    hmap = bch.ResultHmap()
+
+    def __call__(self, **kwargs: Any) -> Any:
+        self.update_params_from_kwargs(**kwargs)
+        self.output = math.sin(self.xval) + math.cos(self.yval)
+        self.hmap = hv.Points((self.xval, self.yval)).opts(
+            marker="o", size=110 - self.level * 20, color=int_to_col(self.level - 1)
+        )
+
+        return self.get_results_values_as_dict()
+
+
+class RunWithLevel(bch.ParametrizedSweep):
+    level = bch.IntSweep(default=1, bounds=[1, 8])
+    dimensions = bch.IntSweep(default=1, bounds=[1, 2])
+
+    level_samples = bch.ResultVar()
+
+    def __call__(self, **kwargs) -> dict():
+        self.update_params_from_kwargs(**kwargs)
+
+        self.level_samples = int(
+            pow(
+                len(bch.FloatSweep(bounds=[0, 1]).with_level(self.level).values()),
+                self.dimensions,
+            )
+        )
+        return self.get_results_values_as_dict()
+
+
+def run_with_dim(bench: bch.Bench, dims: List[bch.SweepBase]) -> List[bch.BenchResult]:
+    results = []
+    for level in range(1, 6):
+        print(level)
+        res = bench.plot_sweep(
+            f"Level:{level}",
+            input_vars=dims,
+            const_vars=LevelsExample.get_input_defaults(
+                [LevelsExample.param.level.with_const(level)]
+            ),
+            result_vars=[LevelsExample.param.output, LevelsExample.param.hmap],
+            run_cfg=bch.BenchRunCfg(level=level, auto_plot=False),
+        )
+
+        results.append(res)
+    return results
+
+
+def run_levels_1D(bench: bch.Bench) -> bch.Bench:
+    results = run_with_dim(bench, [LevelsExample.param.xval])
+    bench.report.append_title("Using Levels to define sample density")
+
+    bench1 = bch.Bench("levels", RunWithLevel(), run_cfg=bch.BenchRunCfg(auto_plot=False))
+    res1 = bench1.plot_sweep("Levels", input_vars=[RunWithLevel.param.level])
+
+    bench.report.append_markdown(
+        "Sample levels let you perform parameter sweeps without having to decide how many samples to take when defining the class.  If you perform a sweep at level 2, then all the points are reused when sampling at level 3.  The higher levels reuse the points from lower levels to avoid having to recompute potentially expensive samples. The other advantage is that it enables a workflow where you can quickly see the results of the sweep at a low resolution to sense check the code, and then run it at a high level to get the fidelity you want.  When calling a sweep at a high level, you can publish the intermediate lower level results as the computiation continues so that you can track the progress of the computation and end the sweep early when you have sufficient resolution",
+        width=600,
+    )
+    row = pn.Row()
+    row.append(res1.to_table())
+    # row.append(res1.to_curve().opts(shared_axes=False))
+    row.append(res1.to_curve())
+
+    bench.report.append(row)
+
+    bench.report.append_markdown(
+        "Level 1 returns a single point at the lower bound of the parameter. Level 2 uses the uppper and lower bounds of the parameter. All subsequent levels are created by adding a sample between each previously calculated sample to ensure that all previous values can be reused while retaining an equal sample spacing.  The following plots show the sample points as circles and the corresponding plot of a sin function sampled at that level.",
+        width=600,
+    )
+
+    combined_pts = hv.Overlay()
+    combined_curve = hv.Overlay()
+    for it, r in enumerate(results):
+        lvl = it + 1
+        row = pn.Row()
+        pts = r.to_holomap().overlay().opts(title=f"Sample Points for level: {lvl}", height=300)
+        ds = r.to_hv_dataset()
+        crv = r.to_curve_ds(ds.data).opts(shared_axes=False, height=300) * r.to_hv_dataset(
+            bch.ReduceType.NONE
+        ).to(hv.Scatter).opts(
+            title=f"Function Values for level: {lvl}", size=5, height=300, shared_axes=False
+        )
+
+        combined_pts *= pts
+        combined_curve *= crv
+        row.append(pts)
+        row.append(crv)
+        bench.report.append_markdown(f"## {r.bench_cfg.title}")
+        bench.report.append(row)
+
+    bench.report.append_markdown(
+        "This plot overlays the previous plots into a single image. It shows how each level overlaps the previous level"
+    )
+
+    bench.report.append(pn.Row(combined_pts, combined_curve))
+    return bench
+
+
+def run_levels_2D(bench: bch.Bench) -> bch.Bench:
+    results = run_with_dim(bench, [LevelsExample.param.xval, LevelsExample.param.yval])
+    bench.report.append_markdown("# Using Levels to define 2D sample density", "Levels 2D")
+
+    bench1 = bch.Bench("lol", RunWithLevel(), run_cfg=bch.BenchRunCfg(auto_plot=False))
+    res1 = bench1.plot_sweep(
+        "Levels",
+        input_vars=[RunWithLevel.param.level],
+        const_vars=[RunWithLevel.param.dimensions.with_const(2)],
+    )
+    row = pn.Row()
+    row.append(res1.to_table())
+    # row.append(res1.to_curve().opts(shared_axes=False))
+    row.append(res1.to_curve())
+
+    bench.report.append(row)
+
+    for it, r in enumerate(results):
+        lvl = it + 1
+        row = pn.Row()
+        bench.report.append_markdown(f"## {r.bench_cfg.title}")
+        row.append(
+            r.to_holomap()
+            .overlay()
+            .opts(title=f"Sample Points for level: {lvl}", shared_axes=False)
+        )
+        row.append(
+            r.to_heatmap_single(r.bench_cfg.result_vars[0], bch.ReduceType.NONE).opts(
+                title=f"Function Value Heatmap for level: {lvl}", shared_axes=False
+            )
+        )
+        bench.report.append(row)
+
+    bench.report.append_markdown(
+        "This plot overlays the previous plots into a single image. It shows how each level overlaps the previous level"
+    )
+    overlay = hv.Overlay()
+    for lvl, r in enumerate(results):
+        overlay *= (
+            r.to_holomap()
+            .overlay()
+            .opts(width=1000, height=1000, show_legend=False, shared_axes=False)
+        )
+
+    bench.report.append(overlay)
+    return bench
+
+
+def run_levels(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    hv.extension("bokeh")
+    opts.defaults(
+        opts.Curve(show_legend=False),
+        opts.Points(show_legend=False),
+    )
+
+    bench = bch.Bench("Levels", LevelsExample(), run_cfg=run_cfg, report=report)
+    bench = run_levels_1D(bench)
+    bench = run_levels_2D(bench)
+
+    return bench
+
+
+if __name__ == "__main__":
+    run_levels().report.show()

bencher/example/example_levels2.py

@@ -0,0 +1,37 @@
+import bencher as bch
+
+
+class Square(bch.ParametrizedSweep):
+    """An example of a datatype with an integer and float parameter"""
+
+    x = bch.FloatSweep(default=0, bounds=[0, 6])
+
+    result = bch.ResultVar("ul", doc="Square of x")
+
+    def __call__(self, **kwargs) -> dict:
+        self.update_params_from_kwargs(**kwargs)
+        self.result = self.x * self.x
+        return self.get_results_values_as_dict()
+
+
+def example_levels2(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
+    """This example shows how to define a custom set of value to sample from intead of a uniform sweep
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+
+    bench = Square().to_bench(run_cfg=run_cfg, report=report)
+
+    # These are all equivalent
+    bench.plot_sweep(input_vars=[Square.param.x.with_level(run_cfg.level, 3)])
+    bench.plot_sweep(input_vars=[bch.p("x", max_level=3)])
+
+    return bench
+
+
+if __name__ == "__main__":
+    example_levels2(bch.BenchRunCfg(level=4)).report.show()

bencher/example/example_pareto.py

@@ -0,0 +1,53 @@
+# pylint: disable=duplicate-code
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_pareto(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    """Example of how to calculate the pareto front of a parameter sweep
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+    run_cfg.use_optuna = True
+
+    bench = bch.Bench(
+        "Multi-objective optimisation",
+        bench_function,
+        ExampleBenchCfgIn,
+        run_cfg=run_cfg,
+        report=report,
+    )
+
+    res = bench.plot_sweep(
+        title="Pareto Optimisation with Optuna",
+        description="This example shows how to plot the pareto front of the tradeoff between multiple criteria.  When multiple result variable are defined, and use_optuna=True a pareto plot and the relative importance of each input variable on the output criteria is plotted. A summary of the points on the pareto front is printed as well.  You can use the pareto plot to decide the how to trade off one objective for another.  Pareto plots are suppored for 2D and 3D.  If you have more than 3 result variables the first 3 are selected for the pareto plot.  Plotting 4D surfaces is left as an exercise to the reader",
+        input_vars=[
+            ExampleBenchCfgIn.param.theta,
+            ExampleBenchCfgIn.param.offset,
+        ],
+        result_vars=[ExampleBenchCfgOut.param.out_sin, ExampleBenchCfgOut.param.out_cos],
+        const_vars=ExampleBenchCfgIn.get_input_defaults(
+            [ExampleBenchCfgIn.param.noisy.with_const(True)]
+        ),
+        post_description="""# Post Description 
+This is a slightly unusual way of doing pareto optimisation as we are not using a typical multi-objective optimisation algorithm [TODO, add example].  Instead we are performing a grid search and looking at the resulting pareto plot.  The reason for doing a grid search instead of standard pareto optimisation is that we can produce more isolated plots of how an input affects an output which can help understanding of the parameter space.  Future examples will show how to use grid search to bootstrap further optimisation with a multi objective optimiser""",
+    )
+
+    bench.report.append(res.to_optuna_plots())
+    return bench
+
+
+if __name__ == "__main__":
+    run_cfg_ex = bch.BenchRunCfg()
+    run_cfg_ex.repeats = 2
+    run_cfg_ex.level = 2
+    example_pareto(run_cfg_ex).report.show()

bencher/example/example_sample_cache.py

@@ -0,0 +1,85 @@
+import bencher as bch
+
+
+class UnreliableClass(bch.ParametrizedSweep):
+    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
+
+    input_val = bch.IntSweep(
+        default=0,
+        bounds=[0, 3],
+        doc="If check limit=True the crashy_fn will crash if this value is >1",
+    )
+    return_value = bch.ResultVar(
+        units="ul",
+        doc="This is a dummy result variable. In this example, it is the same as the value passed in.",
+    )
+    trigger_crash = bch.ResultVar(
+        units="True/False",
+        doc="if true crashy_fn will crash when input_val >1",
+    )
+
+    def crashy_fn(self, input_val: int = 0, **kwargs) -> float:  # pylint: disable=unused-argument
+        if self.trigger_crash:
+            if input_val > 1:
+                raise RuntimeError("I crashed for no good reason ;P")
+
+        return {"return_value": input_val, "trigger_crash": self.trigger_crash}
+
+
+def example_sample_cache(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(),
+    report: bch.BenchReport = bch.BenchReport(),
+    trigger_crash: bool = False,
+) -> bch.Bench:
+    """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+        trigger_crash: (bool): Turn on/off code to artificially trigger a crash
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+
+    instance = UnreliableClass()
+    instance.trigger_crash = trigger_crash
+
+    bencher = bch.Bench("example_sample_cache", instance.crashy_fn, run_cfg=run_cfg, report=report)
+
+    bencher.plot_sweep(
+        title="Example Crashy Function with the sample_cache",
+        input_vars=[UnreliableClass.param.input_val],
+        result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
+        description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
+        run_cfg=run_cfg,
+        post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
+    )
+    return bencher
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.repeats = 1
+    ex_run_cfg.executor = bch.Executors.SCOOP
+
+    # this will store the result of of every call to crashy_fn
+    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.clear_sample_cache = True
+
+    try:
+        # this will crash after iteration 2 because we are checking the crash_threshold >1.  We don't want to lose those (potentially expensive to calculate) datapoints so they are stored in the sample_cache
+        example_sample_cache(ex_run_cfg, trigger_crash=True)
+    except RuntimeError as e:
+        print(f"caught the exception {e}")
+
+    print(
+        "Running the same benchmark but without checking the limit.  The benchmarking should load the previously calculated values and continue to finish calculating the values that were missed due to the crash"
+    )
+    ex_run_cfg.clear_sample_cache = False
+    example_sample_cache(ex_run_cfg, trigger_crash=False)
+
+    ex_run_cfg.repeats = 2
+
+    example_sample_cache(ex_run_cfg, trigger_crash=False).report.show()
+
+    # see the test_sample_cache for a more detailed explanation of the mechanisms of the cache

bencher/example/example_sample_cache_context.py

@@ -0,0 +1,116 @@
+from enum import auto
+
+from strenum import StrEnum
+
+import bencher as bch
+
+
+class ExampleEnum(StrEnum):
+    value_1 = auto()
+    value_2 = auto()
+    # value3 = auto()
+    # value4 = auto()
+
+
+class Cfg(bch.ParametrizedSweep):
+    enum1 = bch.EnumSweep(ExampleEnum)
+    result = bch.ResultVar()
+
+    # def __call__(self,**kwargs) -> Any:
+    #     self.update_params_from_kwargs(**kwargs)
+    #     self.result = float(str(self.enum1)[-1])
+    #     return self.get_results_values_as_dict()
+
+
+def bench_function(cfg: Cfg):
+    return {"result": float(str(cfg.enum1)[-1])}
+
+
+def print_assert_equal(msg, first, second):
+    print(f"{msg} {first}=={second}")
+    assert first == second
+
+
+def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_calls=-1):
+    print_assert_equal(
+        "worker wrapper call count",
+        bencher.sample_cache.worker_wrapper_call_count,
+        wrapper_calls * run_cfg.repeats,
+    )
+    print_assert_equal(
+        "worker fn call count",
+        bencher.sample_cache.worker_fn_call_count,
+        fn_calls * run_cfg.repeats,
+    )
+    print_assert_equal(
+        "worker cache call count",
+        bencher.sample_cache.worker_cache_call_count,
+        cache_calls * run_cfg.repeats,
+    )
+
+
+def example_cache_context() -> bch.Bench:
+    run_cfg = bch.BenchRunCfg()
+    run_cfg.use_sample_cache = True
+    run_cfg.only_hash_tag = True
+    run_cfg.repeats = 2
+    run_cfg.parallel = False
+
+    bencher = bch.Bench("bench_context", bench_function, Cfg, run_cfg=run_cfg)
+
+    # clear all tags from the cache at the beginning so that the example works the same not matter how many times the example is run.  When using this for you own code you probably don't want to clear the cache at the beginning because you will lose all the data you collected.
+    bencher.clear_tag_from_sample_cache("example_tag1", run_cfg)
+    bencher.clear_tag_from_sample_cache("example_tag2", run_cfg)
+
+    # run a benchmark with a constant value and save results with example_tag1
+    bencher.plot_sweep(
+        title="Benchmark enum=value_1",
+        const_vars=[Cfg.param.enum1.with_const(ExampleEnum.value_1)],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # there are not values in the cache, so we expect 1 fn call and 0 cache calls
+    assert_call_counts(bencher, run_cfg, wrapper_calls=1, fn_calls=1, cache_calls=0)
+
+    # now run another benchmark with the same tag but a different value
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=value_2",
+        const_vars=[Cfg.param.enum1.with_const(ExampleEnum.value_2)],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # these values have not been calcuated before so there should be 1 fn call
+    assert_call_counts(bencher, run_cfg, wrapper_calls=1, fn_calls=1, cache_calls=0)
+
+    # now create a new benchmark that calculates the values of the previous two benchmarks. The tag is the same so those values will be loaded from the cache instead of getting calculated again
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=[value_1,value_2] combined",
+        input_vars=[Cfg.param.enum1],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # both calls hit the cache.
+    assert_call_counts(bencher, run_cfg, wrapper_calls=2, fn_calls=0, cache_calls=2)
+
+    # run the same benchmark as before but use a different tag.  The previously cached values will not be used and fresh values will be calculated instead
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=[value_1,value_2] with different tag",
+        input_vars=[Cfg.param.enum1],
+        result_vars=[Cfg.param.result],
+        tag="example_tag2",
+    )
+
+    # Both calls are calcuated becuase the tag is different so they don't hit the cache
+    assert_call_counts(bencher, run_cfg, wrapper_calls=2, fn_calls=2, cache_calls=0)
+
+    return bencher
+
+
+if __name__ == "__main__":
+    example_cache_context().report.show()

bencher/example/example_simple.py

@@ -0,0 +1,134 @@
+# you need this import to be able to reference a class from a static method in that class
+from __future__ import annotations
+
+import math
+import random
+import time
+from datetime import datetime
+from enum import auto
+
+from strenum import StrEnum
+
+import bencher as bch
+
+
+# define a class with the output variables you want to benchmark. It must inherit from ParametrizedSweep (which inherits from param.Parametrized). Param is a python library that allows you to track metadata about parameters.  I would recommend reading at least the intro: https://param.holoviz.org/.  I have extended param with some extra metadata such is the units of the variable so that it can automaticaly be plotted.
+class OutputCfg(bch.ParametrizedSweep):
+    """A class for defining what variables the benchmark function returns and metadata on those variables"""
+
+    # Documenting the variable here with enables automatic summaries of what has been benchmarked.
+    # This made up example uses accuracy as an example, but the variable defined here can be any metric that is important for the performance of a system.  You can also define the direction of the optimisation i.e. to minimise or maximise the metric.
+    accuracy = bch.ResultVar(
+        units="%", direction=bch.OptDir.maximize, doc="The accuracy of the algorithm."
+    )
+
+
+# Define categorical variables with enums that inherit from StrEnum.  In this example, its just an arbitrary set of categories that have an unknown influence on the metric you want to understand. In a real world case these would be a set of conditions or settings you are benchmarking
+class AlgoSetting(StrEnum):
+    """Use enums to describe categorical input.  In this example they are given names that describe how they affect the function output, but in a real world example these will be some settings to an algorithm that you want to understand how they affect the metric you are trying to optimise."""
+
+    # add some random noise to the output.  When your algorithm has noisy output it often is an indication that something is not quite right.  The graphs should show that you want to avoid the "noisy" setting in your algorithm
+    noisy = auto()
+
+    # This is the setting with the best performance, and characterising that that is is the goal of the benchmarking
+    optimum = auto()
+
+    poor = auto()  # this setting results in poor performance
+
+
+# define a class with the input variables you want to benchmark.  It must inherit from ParametrizeSweep. This class defines a struct that is passed to the benchmark function.  The function must be pure and so we define it as a staticmethod that takes an InputCfg class and returns an OutputCfg class. By accepting and returning parametrized classes the metadata about what the relationship between the input and output are easy to track.
+class InputCfg(bch.ParametrizedSweep):
+    # The variables must be defined as one of the Sweep types, i.e, FloatSweep, IntSweep, EnumSweep from bencher.bench_vars
+    # theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
+
+    # Define sweep variables by passing in an enum class name. The first element of the enum is the default by convention, but you can overrride the default in the constructor
+    algo_setting_enum = bch.EnumSweep(AlgoSetting, default=AlgoSetting.poor)
+
+    # In this case there are no units so its marked as unitless or ul. You can define how many evenly distributed samples to sample the parameter with
+    algo_setting_float = bch.FloatSweep(
+        default=0.0,
+        bounds=[0.0, 6.0],
+        doc="This represents a continuous input value to your function that affects the desired output in a way you want to characterise.",
+        units="ul",
+        samples=10,
+    )
+
+    # define the objective function you want to benchmark. It must be static and have no side effects. It should accept 1 input of type InputCfg (or whatever your input config class is called) and return the OutputCfg class you have defined
+    @staticmethod
+    def bench_function(cfg: InputCfg) -> OutputCfg:
+        """Takes an ExampleBenchCfgIn and returns a ExampleBenchCfgOut output.  This is just a dummy example so the behavior of the function is rather transparent, but in a real use case the function would be a black box you want to characterise."""
+        output = OutputCfg()
+
+        output.accuracy = 50 + math.sin(cfg.algo_setting_float) * 5
+
+        # this simulates random long term change in the function
+        output.accuracy += time.localtime(datetime.now().second).tm_sec / 30
+
+        match cfg.algo_setting_enum:
+            case AlgoSetting.noisy:
+                # add some random noise to the output.  When your algorith has noisy output it often is an indication that something is not quite right.  The graphs should show that you want to avoid the "noisy" setting in your algorithm
+                output.accuracy += random.uniform(-10, 10)
+            case AlgoSetting.optimum:
+                output.accuracy += 30  # This is the setting with the best performance, and characterising that is is the goal of the benchmarking
+            case AlgoSetting.poor:
+                output.accuracy -= 20  # this setting results in poor performance
+        return output
+
+
+if __name__ == "__main__":
+    # pass the objective function you have defined to bencher.  This benchmark function can be reused for multiple sweeps.  You also need to pass the inputCfg to the bencher so that it can process the metadata about the input configuration.
+    bench = bch.Bench("Bencher_Example_Categorical", InputCfg.bench_function, InputCfg)
+
+    # Bencher needs to know the metadata of the variable in order to automatically sweep and plot it, so it is passed by using param's metadata syntax.  InputCfg.param.* is how to access the metadata defined in the class description.  Unfortunately vscode autocomplete doesn't work with params metaclass machinery so you will need to look at the class definition to get a list of possible settings. Define what parameter you want to sweep over and the result variable you want to plot.  If you pass 1 input, it will perform a 1D sweep over that dimension and plot a line or a bar graph of the result (depending on if that variable on continuous or discrete).  In this example we are going to sweep the enum variable and record the accuracy.
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_enum],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 1D enum sweep",
+        description="""Sample all the values in enum setting and record the resulting accuracy.  The algo_setting_float is not mentioned in the inputs and so it takes the default value that was set in the InputCfg class.  Repeats=10 so the benchmark function is called 10 times serially.  This is why the function must be pure, if a past call to the function affects the future call to the function (through global side effects) any statistics you calculate will not be correct. 
+        """,
+        post_description="Here you can see the affect of each setting on the output and the optimum is clearly the best.",
+        run_cfg=bch.BenchRunCfg(repeats=10),
+    )
+
+    # There is also a floating point input setting that affects the performance of the algorithm.  By passing only the float setting, the InputCfg class will use the default setting of the categorical value so you can understand the float setting in isolation
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_float],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 1D float sweep",
+        description="""Perform a 1D sweep over the continuous variable algo_setting_float taking sweep the bounds and number of samples from the InputCfg class definition.  The algo_setting_enum is not mentioned in the inputs and so it takes the default value that was set in the InputCfg class.  Repeats=10 so the benchmark function is called 10 times serially.  
+        """,
+        post_description="The plot shows the output is affected by the float input in a continuous way with a peak around 1.5",
+        run_cfg=bch.BenchRunCfg(repeats=10),
+    )
+
+    # This sweep is a combination of the previous two sweeps
+    bench.plot_sweep(
+        input_vars=[
+            InputCfg.param.algo_setting_float,
+            InputCfg.param.algo_setting_enum,
+        ],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 2D sweep",
+        description="""Perform a 2D sweep over the enum and continuous variable to see how they act together.  Here the setting use_optuna=True so additional graphs a plotted at the end. 
+        """,
+        post_description="In this example function the two input settings combine in a linear and predictable way, so the best combination of settings is enum = AlgoSetting.optimum and float = 1.33.  Setting use_optuna=True adds a plot of how much each input parameter affects the metric and a printout of the best parameter values found during the sweep.  If the value for repeat is high it is an indication there is something wrong with your benchmark function. The repeat should have no affect on the value of the function if calls to the function are independent.  This can be useful to detect undesired side effects in your code",
+        run_cfg=bch.BenchRunCfg(repeats=10, use_optuna=True, serve_xarray=True, serve_pandas=True),
+    )
+
+    # In the last example we track the value of the categorical values over time.
+    # run this code in a loop twice to simulate calling the benchmarking function at different times.  The most common use case for tracking over time would be run once a day during nightly benchmarking
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_enum],
+        result_vars=[OutputCfg.param.accuracy],
+        const_vars=[(InputCfg.param.algo_setting_float, 1.33)],
+        title="Simple example 1D sweep over time",
+        description="""Once you have found the optimal settings for your algorithm you want to make sure that the performance is not lost over time.  You can set variables to a constant value and in this case the float value is set to its optimum value.  The first time this function is run only the results from sweeping the categorical value is plotted (the same as example 1), but the second time it is run a graph the values over time is shown. [Run the code again if you don't see a graph over time]. If the graphs over time shows long term changes (not just noise), it indicate there is another external factor that is affecting your performace over time, i.e. dependencies changing, physical degradation of equipment, an unnoticed bug from a pull request etc...
+
+        This shows the basic features of bencher.  These examples are purposefully simplified to demonstrate its features in isolation and don't reeally show the real advantages of bencher.  If you only have a few inputs and outputs its not that complicated to throw together some plots of performance.  The power of bencher is that when you have a system with many moving parts that all interact with eachother, teasing apart those influences becomes much harder because the parameter spaces combine quite quickly into a high dimensional mess. Bencher makes it easier to experiment with different combination of inputs to gain an intuition of the system performance. Bencher can plot up to 6D input natively and you can add custom plots if you have exotic data types or state spaces [WIP]. 
+        """,
+        post_description="",
+        run_cfg=bch.BenchRunCfg(repeats=10, over_time=True, clear_history=False),
+    )
+
+    # launch web server and view
+    bench.report.show()

bencher/example/example_simple_bool.py

@@ -0,0 +1,35 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_1D_bool(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional categorical variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = bch.Bench(
+        "benchmarking_example_categorical1D",
+        bench_function,
+        ExampleBenchCfgIn,
+    )
+
+    # here we sample the input variable theta and plot the value of output1. The (noisy) function is sampled 20 times so you can see the distribution
+    res = bench.plot_sweep(
+        title="Example 1D Bool",
+        input_vars=[ExampleBenchCfgIn.param.noisy],
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        description=example_1D_bool.__doc__,
+        run_cfg=run_cfg,
+    )
+    bench.report.append(res.to_bar())
+
+    return bench
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.repeats = 20
+
+    example_1D_bool(ex_run_cfg).report.show()