holobench 1.25.1__py3-none-any.whl → 1.26.3__py3-none-any.whl

bencher/example/example_sample_cache.py

@@ -0,0 +1,85 @@
+import bencher as bch
+
+
+class UnreliableClass(bch.ParametrizedSweep):
+    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
+
+    input_val = bch.IntSweep(
+        default=0,
+        bounds=[0, 3],
+        doc="If check limit=True the crashy_fn will crash if this value is >1",
+    )
+    return_value = bch.ResultVar(
+        units="ul",
+        doc="This is a dummy result variable. In this example, it is the same as the value passed in.",
+    )
+    trigger_crash = bch.ResultVar(
+        units="True/False",
+        doc="if true crashy_fn will crash when input_val >1",
+    )
+
+    def crashy_fn(self, input_val: int = 0, **kwargs) -> float:  # pylint: disable=unused-argument
+        if self.trigger_crash:
+            if input_val > 1:
+                raise RuntimeError("I crashed for no good reason ;P")
+
+        return {"return_value": input_val, "trigger_crash": self.trigger_crash}
+
+
+def example_sample_cache(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(),
+    report: bch.BenchReport = bch.BenchReport(),
+    trigger_crash: bool = False,
+) -> bch.Bench:
+    """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+        trigger_crash: (bool): Turn on/off code to artificially trigger a crash
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+
+    instance = UnreliableClass()
+    instance.trigger_crash = trigger_crash
+
+    bencher = bch.Bench("example_sample_cache", instance.crashy_fn, run_cfg=run_cfg, report=report)
+
+    bencher.plot_sweep(
+        title="Example Crashy Function with the sample_cache",
+        input_vars=[UnreliableClass.param.input_val],
+        result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
+        description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
+        run_cfg=run_cfg,
+        post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
+    )
+    return bencher
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.repeats = 1
+    ex_run_cfg.executor = bch.Executors.SCOOP
+
+    # this will store the result of of every call to crashy_fn
+    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.clear_sample_cache = True
+
+    try:
+        # this will crash after iteration 2 because we are checking the crash_threshold >1.  We don't want to lose those (potentially expensive to calculate) datapoints so they are stored in the sample_cache
+        example_sample_cache(ex_run_cfg, trigger_crash=True)
+    except RuntimeError as e:
+        print(f"caught the exception {e}")
+
+    print(
+        "Running the same benchmark but without checking the limit.  The benchmarking should load the previously calculated values and continue to finish calculating the values that were missed due to the crash"
+    )
+    ex_run_cfg.clear_sample_cache = False
+    example_sample_cache(ex_run_cfg, trigger_crash=False)
+
+    ex_run_cfg.repeats = 2
+
+    example_sample_cache(ex_run_cfg, trigger_crash=False).report.show()
+
+    # see the test_sample_cache for a more detailed explanation of the mechanisms of the cache

bencher/example/example_sample_cache_context.py

@@ -0,0 +1,116 @@
+from enum import auto
+
+from strenum import StrEnum
+
+import bencher as bch
+
+
+class ExampleEnum(StrEnum):
+    value_1 = auto()
+    value_2 = auto()
+    # value3 = auto()
+    # value4 = auto()
+
+
+class Cfg(bch.ParametrizedSweep):
+    enum1 = bch.EnumSweep(ExampleEnum)
+    result = bch.ResultVar()
+
+    # def __call__(self,**kwargs) -> Any:
+    #     self.update_params_from_kwargs(**kwargs)
+    #     self.result = float(str(self.enum1)[-1])
+    #     return self.get_results_values_as_dict()
+
+
+def bench_function(cfg: Cfg):
+    return {"result": float(str(cfg.enum1)[-1])}
+
+
+def print_assert_equal(msg, first, second):
+    print(f"{msg} {first}=={second}")
+    assert first == second
+
+
+def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_calls=-1):
+    print_assert_equal(
+        "worker wrapper call count",
+        bencher.sample_cache.worker_wrapper_call_count,
+        wrapper_calls * run_cfg.repeats,
+    )
+    print_assert_equal(
+        "worker fn call count",
+        bencher.sample_cache.worker_fn_call_count,
+        fn_calls * run_cfg.repeats,
+    )
+    print_assert_equal(
+        "worker cache call count",
+        bencher.sample_cache.worker_cache_call_count,
+        cache_calls * run_cfg.repeats,
+    )
+
+
+def example_cache_context() -> bch.Bench:
+    run_cfg = bch.BenchRunCfg()
+    run_cfg.use_sample_cache = True
+    run_cfg.only_hash_tag = True
+    run_cfg.repeats = 2
+    run_cfg.parallel = False
+
+    bencher = bch.Bench("bench_context", bench_function, Cfg, run_cfg=run_cfg)
+
+    # clear all tags from the cache at the beginning so that the example works the same not matter how many times the example is run.  When using this for you own code you probably don't want to clear the cache at the beginning because you will lose all the data you collected.
+    bencher.clear_tag_from_sample_cache("example_tag1", run_cfg)
+    bencher.clear_tag_from_sample_cache("example_tag2", run_cfg)
+
+    # run a benchmark with a constant value and save results with example_tag1
+    bencher.plot_sweep(
+        title="Benchmark enum=value_1",
+        const_vars=[Cfg.param.enum1.with_const(ExampleEnum.value_1)],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # there are not values in the cache, so we expect 1 fn call and 0 cache calls
+    assert_call_counts(bencher, run_cfg, wrapper_calls=1, fn_calls=1, cache_calls=0)
+
+    # now run another benchmark with the same tag but a different value
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=value_2",
+        const_vars=[Cfg.param.enum1.with_const(ExampleEnum.value_2)],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # these values have not been calcuated before so there should be 1 fn call
+    assert_call_counts(bencher, run_cfg, wrapper_calls=1, fn_calls=1, cache_calls=0)
+
+    # now create a new benchmark that calculates the values of the previous two benchmarks. The tag is the same so those values will be loaded from the cache instead of getting calculated again
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=[value_1,value_2] combined",
+        input_vars=[Cfg.param.enum1],
+        result_vars=[Cfg.param.result],
+        tag="example_tag1",
+    )
+
+    # both calls hit the cache.
+    assert_call_counts(bencher, run_cfg, wrapper_calls=2, fn_calls=0, cache_calls=2)
+
+    # run the same benchmark as before but use a different tag.  The previously cached values will not be used and fresh values will be calculated instead
+    bencher.clear_call_counts()
+    bencher.plot_sweep(
+        title="Benchmark enum=[value_1,value_2] with different tag",
+        input_vars=[Cfg.param.enum1],
+        result_vars=[Cfg.param.result],
+        tag="example_tag2",
+    )
+
+    # Both calls are calcuated becuase the tag is different so they don't hit the cache
+    assert_call_counts(bencher, run_cfg, wrapper_calls=2, fn_calls=2, cache_calls=0)
+
+    return bencher
+
+
+if __name__ == "__main__":
+    example_cache_context().report.show()

bencher/example/example_simple.py

@@ -0,0 +1,134 @@
+# you need this import to be able to reference a class from a static method in that class
+from __future__ import annotations
+
+import math
+import random
+import time
+from datetime import datetime
+from enum import auto
+
+from strenum import StrEnum
+
+import bencher as bch
+
+
+# define a class with the output variables you want to benchmark. It must inherit from ParametrizedSweep (which inherits from param.Parametrized). Param is a python library that allows you to track metadata about parameters.  I would recommend reading at least the intro: https://param.holoviz.org/.  I have extended param with some extra metadata such is the units of the variable so that it can automaticaly be plotted.
+class OutputCfg(bch.ParametrizedSweep):
+    """A class for defining what variables the benchmark function returns and metadata on those variables"""
+
+    # Documenting the variable here with enables automatic summaries of what has been benchmarked.
+    # This made up example uses accuracy as an example, but the variable defined here can be any metric that is important for the performance of a system.  You can also define the direction of the optimisation i.e. to minimise or maximise the metric.
+    accuracy = bch.ResultVar(
+        units="%", direction=bch.OptDir.maximize, doc="The accuracy of the algorithm."
+    )
+
+
+# Define categorical variables with enums that inherit from StrEnum.  In this example, its just an arbitrary set of categories that have an unknown influence on the metric you want to understand. In a real world case these would be a set of conditions or settings you are benchmarking
+class AlgoSetting(StrEnum):
+    """Use enums to describe categorical input.  In this example they are given names that describe how they affect the function output, but in a real world example these will be some settings to an algorithm that you want to understand how they affect the metric you are trying to optimise."""
+
+    # add some random noise to the output.  When your algorithm has noisy output it often is an indication that something is not quite right.  The graphs should show that you want to avoid the "noisy" setting in your algorithm
+    noisy = auto()
+
+    # This is the setting with the best performance, and characterising that that is is the goal of the benchmarking
+    optimum = auto()
+
+    poor = auto()  # this setting results in poor performance
+
+
+# define a class with the input variables you want to benchmark.  It must inherit from ParametrizeSweep. This class defines a struct that is passed to the benchmark function.  The function must be pure and so we define it as a staticmethod that takes an InputCfg class and returns an OutputCfg class. By accepting and returning parametrized classes the metadata about what the relationship between the input and output are easy to track.
+class InputCfg(bch.ParametrizedSweep):
+    # The variables must be defined as one of the Sweep types, i.e, FloatSweep, IntSweep, EnumSweep from bencher.bench_vars
+    # theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
+
+    # Define sweep variables by passing in an enum class name. The first element of the enum is the default by convention, but you can overrride the default in the constructor
+    algo_setting_enum = bch.EnumSweep(AlgoSetting, default=AlgoSetting.poor)
+
+    # In this case there are no units so its marked as unitless or ul. You can define how many evenly distributed samples to sample the parameter with
+    algo_setting_float = bch.FloatSweep(
+        default=0.0,
+        bounds=[0.0, 6.0],
+        doc="This represents a continuous input value to your function that affects the desired output in a way you want to characterise.",
+        units="ul",
+        samples=10,
+    )
+
+    # define the objective function you want to benchmark. It must be static and have no side effects. It should accept 1 input of type InputCfg (or whatever your input config class is called) and return the OutputCfg class you have defined
+    @staticmethod
+    def bench_function(cfg: InputCfg) -> OutputCfg:
+        """Takes an ExampleBenchCfgIn and returns a ExampleBenchCfgOut output.  This is just a dummy example so the behavior of the function is rather transparent, but in a real use case the function would be a black box you want to characterise."""
+        output = OutputCfg()
+
+        output.accuracy = 50 + math.sin(cfg.algo_setting_float) * 5
+
+        # this simulates random long term change in the function
+        output.accuracy += time.localtime(datetime.now().second).tm_sec / 30
+
+        match cfg.algo_setting_enum:
+            case AlgoSetting.noisy:
+                # add some random noise to the output.  When your algorith has noisy output it often is an indication that something is not quite right.  The graphs should show that you want to avoid the "noisy" setting in your algorithm
+                output.accuracy += random.uniform(-10, 10)
+            case AlgoSetting.optimum:
+                output.accuracy += 30  # This is the setting with the best performance, and characterising that is is the goal of the benchmarking
+            case AlgoSetting.poor:
+                output.accuracy -= 20  # this setting results in poor performance
+        return output
+
+
+if __name__ == "__main__":
+    # pass the objective function you have defined to bencher.  This benchmark function can be reused for multiple sweeps.  You also need to pass the inputCfg to the bencher so that it can process the metadata about the input configuration.
+    bench = bch.Bench("Bencher_Example_Categorical", InputCfg.bench_function, InputCfg)
+
+    # Bencher needs to know the metadata of the variable in order to automatically sweep and plot it, so it is passed by using param's metadata syntax.  InputCfg.param.* is how to access the metadata defined in the class description.  Unfortunately vscode autocomplete doesn't work with params metaclass machinery so you will need to look at the class definition to get a list of possible settings. Define what parameter you want to sweep over and the result variable you want to plot.  If you pass 1 input, it will perform a 1D sweep over that dimension and plot a line or a bar graph of the result (depending on if that variable on continuous or discrete).  In this example we are going to sweep the enum variable and record the accuracy.
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_enum],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 1D enum sweep",
+        description="""Sample all the values in enum setting and record the resulting accuracy.  The algo_setting_float is not mentioned in the inputs and so it takes the default value that was set in the InputCfg class.  Repeats=10 so the benchmark function is called 10 times serially.  This is why the function must be pure, if a past call to the function affects the future call to the function (through global side effects) any statistics you calculate will not be correct. 
+        """,
+        post_description="Here you can see the affect of each setting on the output and the optimum is clearly the best.",
+        run_cfg=bch.BenchRunCfg(repeats=10),
+    )
+
+    # There is also a floating point input setting that affects the performance of the algorithm.  By passing only the float setting, the InputCfg class will use the default setting of the categorical value so you can understand the float setting in isolation
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_float],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 1D float sweep",
+        description="""Perform a 1D sweep over the continuous variable algo_setting_float taking sweep the bounds and number of samples from the InputCfg class definition.  The algo_setting_enum is not mentioned in the inputs and so it takes the default value that was set in the InputCfg class.  Repeats=10 so the benchmark function is called 10 times serially.  
+        """,
+        post_description="The plot shows the output is affected by the float input in a continuous way with a peak around 1.5",
+        run_cfg=bch.BenchRunCfg(repeats=10),
+    )
+
+    # This sweep is a combination of the previous two sweeps
+    bench.plot_sweep(
+        input_vars=[
+            InputCfg.param.algo_setting_float,
+            InputCfg.param.algo_setting_enum,
+        ],
+        result_vars=[OutputCfg.param.accuracy],
+        title="Simple example 2D sweep",
+        description="""Perform a 2D sweep over the enum and continuous variable to see how they act together.  Here the setting use_optuna=True so additional graphs a plotted at the end. 
+        """,
+        post_description="In this example function the two input settings combine in a linear and predictable way, so the best combination of settings is enum = AlgoSetting.optimum and float = 1.33.  Setting use_optuna=True adds a plot of how much each input parameter affects the metric and a printout of the best parameter values found during the sweep.  If the value for repeat is high it is an indication there is something wrong with your benchmark function. The repeat should have no affect on the value of the function if calls to the function are independent.  This can be useful to detect undesired side effects in your code",
+        run_cfg=bch.BenchRunCfg(repeats=10, use_optuna=True, serve_xarray=True, serve_pandas=True),
+    )
+
+    # In the last example we track the value of the categorical values over time.
+    # run this code in a loop twice to simulate calling the benchmarking function at different times.  The most common use case for tracking over time would be run once a day during nightly benchmarking
+    bench.plot_sweep(
+        input_vars=[InputCfg.param.algo_setting_enum],
+        result_vars=[OutputCfg.param.accuracy],
+        const_vars=[(InputCfg.param.algo_setting_float, 1.33)],
+        title="Simple example 1D sweep over time",
+        description="""Once you have found the optimal settings for your algorithm you want to make sure that the performance is not lost over time.  You can set variables to a constant value and in this case the float value is set to its optimum value.  The first time this function is run only the results from sweeping the categorical value is plotted (the same as example 1), but the second time it is run a graph the values over time is shown. [Run the code again if you don't see a graph over time]. If the graphs over time shows long term changes (not just noise), it indicate there is another external factor that is affecting your performace over time, i.e. dependencies changing, physical degradation of equipment, an unnoticed bug from a pull request etc...
+
+        This shows the basic features of bencher.  These examples are purposefully simplified to demonstrate its features in isolation and don't reeally show the real advantages of bencher.  If you only have a few inputs and outputs its not that complicated to throw together some plots of performance.  The power of bencher is that when you have a system with many moving parts that all interact with eachother, teasing apart those influences becomes much harder because the parameter spaces combine quite quickly into a high dimensional mess. Bencher makes it easier to experiment with different combination of inputs to gain an intuition of the system performance. Bencher can plot up to 6D input natively and you can add custom plots if you have exotic data types or state spaces [WIP]. 
+        """,
+        post_description="",
+        run_cfg=bch.BenchRunCfg(repeats=10, over_time=True, clear_history=False),
+    )
+
+    # launch web server and view
+    bench.report.show()

bencher/example/example_simple_bool.py

@@ -0,0 +1,35 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_1D_bool(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional categorical variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = bch.Bench(
+        "benchmarking_example_categorical1D",
+        bench_function,
+        ExampleBenchCfgIn,
+    )
+
+    # here we sample the input variable theta and plot the value of output1. The (noisy) function is sampled 20 times so you can see the distribution
+    res = bench.plot_sweep(
+        title="Example 1D Bool",
+        input_vars=[ExampleBenchCfgIn.param.noisy],
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        description=example_1D_bool.__doc__,
+        run_cfg=run_cfg,
+    )
+    bench.report.append(res.to_bar())
+
+    return bench
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.repeats = 20
+
+    example_1D_bool(ex_run_cfg).report.show()

bencher/example/example_simple_cat.py

@@ -0,0 +1,48 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+# pylint: disable=duplicate-code
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_1D_cat(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional categorical variable and plot the result of passing that parameter sweep to the benchmarking function
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+
+    explorer = ExampleBenchCfgIn()
+    bench = bch.Bench(
+        "benchmarking_example_categorical1D",
+        bench_function,
+        ExampleBenchCfgIn,
+        run_cfg=run_cfg,
+        report=report,
+    )
+
+    # here we sample the input variable theta and plot the value of output1. The (noisy) function is sampled 20 times so you can see the distribution
+    bench.plot_sweep(
+        title="Example 1D Categorical",
+        description="""This example shows how to sample a 1 dimensional categorical variable and plot the result of passing that parameter sweep to the benchmarking function""",
+        input_vars=[ExampleBenchCfgIn.param.postprocess_fn],
+        result_vars=[ExampleBenchCfgOut.param.out_cos, ExampleBenchCfgOut.param.out_sin],
+        const_vars=explorer.get_input_defaults(),
+    )
+
+    return bench
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.repeats = 10
+
+    srv1 = example_1D_cat(ex_run_cfg).report.show()

bencher/example/example_simple_float.py

@@ -0,0 +1,28 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import math
+import bencher as bch
+
+
+class SimpleFloat(bch.ParametrizedSweep):
+    theta = bch.FloatSweep(
+        default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30
+    )
+    out_sin = bch.ResultVar(units="v", doc="sin of theta")
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        self.out_sin = math.sin(self.theta)
+        return super().__call__(**kwargs)
+
+
+def example_1D_float(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = SimpleFloat().to_bench(run_cfg, report)
+    bench.plot_sweep()
+    return bench
+
+
+if __name__ == "__main__":
+    example_1D_float().report.show()

bencher/example/example_simple_float2d.py

@@ -0,0 +1,29 @@
+"""This file contains examples for how to perform basic 2D benchmarking parameter sweeps"""
+
+import math
+import bencher as bch
+
+
+class SimpleFloat(bch.ParametrizedSweep):
+    theta = bch.FloatSweep(
+        default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30
+    )
+    offset = bch.FloatSweep(default=0, bounds=[0, 1], doc="Input angle", units="rad")
+    out_sin = bch.ResultVar(units="v", doc="sin of theta")
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        self.out_sin = math.sin(self.theta) + self.offset
+        return super().__call__(**kwargs)
+
+
+def example_2D_float(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = SimpleFloat().to_bench(run_cfg, report)
+    bench.plot_sweep()
+    return bench
+
+
+if __name__ == "__main__":
+    example_2D_float().report.show()

bencher/example/example_strings.py

@@ -0,0 +1,47 @@
+import bencher as bch
+
+
+class TestPrinting(bch.ParametrizedSweep):
+    # INPUTS
+    a = bch.StringSweep(default=None, string_list=["a1", "a2"], allow_None=True)
+    b = bch.StringSweep(default=None, string_list=["b1", "b2"], allow_None=True)
+    c = bch.StringSweep(default=None, string_list=["c1", "c2"], allow_None=True)
+    d = bch.StringSweep(default=None, string_list=["d1", "d2"], allow_None=True)
+
+    # RESULTS
+    result = bch.ResultString()
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+
+        self.result = self.a
+        if self.b is not None:
+            self.result += f",{self.b}"
+        if self.c is not None:
+            self.result += f",{self.c}"
+        if self.d is not None:
+            self.result += f",{self.d}"
+        self.result += "\n\ttab\n\t\ttab2"
+
+        self.result = bch.tabs_in_markdown(self.result)
+        return super().__call__()
+
+
+def example_strings(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    bench = bch.Bench("strings", TestPrinting(), run_cfg=run_cfg, report=report)
+
+    for s in [
+        [TestPrinting.param.a],
+        [TestPrinting.param.a, TestPrinting.param.b],
+        [TestPrinting.param.a, TestPrinting.param.b, TestPrinting.param.c],
+        [TestPrinting.param.a, TestPrinting.param.b, TestPrinting.param.c, TestPrinting.param.d],
+    ]:
+        bench.plot_sweep(f"String Panes {[v.name for v in s]}", input_vars=s)
+
+    return bench
+
+
+if __name__ == "__main__":
+    example_strings().report.show()

bencher/example/example_time_event.py

@@ -0,0 +1,63 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+# pylint: disable=duplicate-code
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_time_event(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    """This example shows how to manually set time events as a string so that progress can be monitored over time"""
+
+    bencher = bch.Bench(
+        "benchmarking_example_categorical1D",
+        bench_function,
+        ExampleBenchCfgIn,
+        run_cfg=run_cfg,
+        report=report,
+    )
+
+    ExampleBenchCfgIn.param.offset.bounds = [0, 100]
+
+    # manually override the default value based on the time event string so that the graphs are not all just straight lines
+    ExampleBenchCfgIn.param.offset.default = int(str(hash(run_cfg.time_event))[-1])
+
+    # here we sample the input variable theta and plot the value of output1. The (noisy) function is sampled 20 times so you can see the distribution
+    bencher.plot_sweep(
+        title="Example 1D Categorical",
+        input_vars=[ExampleBenchCfgIn.param.postprocess_fn],
+        result_vars=[ExampleBenchCfgOut.param.out_cos],
+        description=example_time_event.__doc__,
+        run_cfg=run_cfg,
+    )
+    return bencher
+
+
+def run_example_time_event(ex_run_cfg):
+    ex_run_cfg.repeats = 1
+    ex_run_cfg.print_pandas = True
+    ex_run_cfg.over_time = True
+
+    ex_run_cfg.clear_cache = True
+    ex_run_cfg.clear_history = True
+
+    ex_run_cfg.time_event = "-first_event"
+    example_time_event(ex_run_cfg)
+
+    ex_run_cfg.clear_cache = False
+    ex_run_cfg.clear_history = False
+    ex_run_cfg.time_event = "_second_event"
+    example_time_event(ex_run_cfg)
+
+    ex_run_cfg.time_event = (
+        "*third_event has a very very long label to demonstrate automatic text wrapping"
+    )
+    return example_time_event(ex_run_cfg)
+
+
+if __name__ == "__main__":
+    run_example_time_event(bch.BenchRunCfg()).report.show()