holobench 1.25.2__py3-none-any.whl → 1.27.0__py3-none-any.whl

bencher/bench_report.py

@@ -157,112 +157,9 @@ class BenchReport(BenchPlotServer):
 
         return publish_url
 
-
-# def append(self,pane):
-# self.report.append(pane)
-
-# def __getstate__(self):
-#     state = self.__dict__.copy()
-#     # Don't pickle baz
-#     del state["pane"]
-#     return state
-
-# def __setstate__(self, state):
-#     self.__dict__.update(state)
-#     # Add baz back since it doesn't exist in the pickle
-#     self.report = []
-
-# def publish_old(
-#     self,
-#     directory: str = "bench_results",
-#     branch_name: str = "bench_results",
-#     url_postprocess: Callable = None,
-#     **kwargs,
-# ) -> str:
-#     """Publish the results as an html file by committing it to the bench_results branch in the current repo. If you have set up your repo with github pages or equivalent then the html file will be served as a viewable webpage.
-
-#     Args:
-#         directory (str, optional): Directory to save the results. Defaults to "bench_results".
-#         branch_name (str, optional): Branch to publish on. Defaults to "bench_results".
-#         url_postprocess (Callable, optional): A function that maps the origin url to a github pages url. Pass your own function if you are using another git providers. Defaults to None.
-
-#     Returns:
-#         str: _description_
-#     """
-
-#     def get_output(cmd: str) -> str:
-#         return (
-#             subprocess.run(cmd.split(" "), stdout=subprocess.PIPE, check=False)
-#             .stdout.decode("utf=8")
-#             .strip()
-#         )
-
-#     def postprocess_url(publish_url: str, branch_name: str, report_path: str, **kwargs) -> str:
-#         # import re
-
-#         # return re.sub(
-#         #     """((git|ssh|http(s)?)|(git@[\w\.-]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?""",
-#         #     """https://$7/""",
-#         #     publish_url,
-#         # )
-#         # git@github.com:user/project.git
-#         # https://github.com/user/project.git
-#         # http://github.com/user/project.git
-#         # git@192.168.101.127:user/project.git
-#         # https://192.168.101.127/user/project.git
-#         # http://192.168.101.127/user/project.git
-#         # ssh://user@host.xz:port/path/to/repo.git/
-#         # ssh://user@host.xz/path/to/repo.git/
-#         # ssh://host.xz:port/path/to/repo.git/
-#         # ssh://host.xz/path/to/repo.git/
-#         # ssh://user@host.xz/path/to/repo.git/
-#         # ssh://host.xz/path/to/repo.git/
-#         # ssh://user@host.xz/~user/path/to/repo.git/
-#         # ssh://host.xz/~user/path/to/repo.git/
-#         # ssh://user@host.xz/~/path/to/repo.git
-#         # ssh://host.xz/~/path/to/repo.git
-#         # git://host.xz/path/to/repo.git/
-#         # git://host.xz/~user/path/to/repo.git/
-#         # http://host.xz/path/to/repo.git/
-#         # https://host.xz/path/to/repo.git/
-#         # https://regex101.com/r/qT7NP0/3
-
-#         return publish_url.replace(".git", f"/blob/{directory}/{report_path}")
-
-#     if url_postprocess is None:
-#         url_postprocess = postprocess_url
-#     current_branch = get_output("git symbolic-ref --short HEAD")
-#     logging.info(f"on branch: {current_branch}")
-#     stash_msg = get_output("git stash")
-#     logging.info(f"stashing current work :{stash_msg}")
-#     checkout_msg = get_output(f"git checkout -b {branch_name}")
-#     checkout_msg = get_output(f"git checkout {branch_name}")
-#     get_output("git pull")
-
-#     logging.info(f"checking out branch: {checkout_msg}")
-#     report_path = self.save(directory, in_html_folder=False)
-#     logging.info(f"created report at: {report_path.absolute()}")
-#     # commit_msg = f""
-#     logging.info("adding report to git")
-#     get_output(f"git add {report_path.absolute()}")
-#     get_output("git status")
-#     logging.info("committing report")
-#     cmd = f'git commit -m "generate_report:{self.bench_name}"'
-#     logging.info(cmd)
-#     get_output(cmd)
-#     logging.info("pushing report to origin")
-#     get_output(f"git push --set-upstream origin {branch_name}")
-#     logging.info("checking out original branch")
-#     get_output(f"git checkout {current_branch}")
-#     if "No local changes" not in stash_msg:
-#         logging.info("restoring work with git stash pop")
-#         get_output("git stash pop")
-
-#     publish_url = get_output("git remote get-url --push origin")
-#     logging.info(f"raw url:{publish_url}")
-#     publish_url = url_postprocess(
-#         publish_url, branch_name=branch_name, report_path=report_path, **kwargs
-#     )
-#     logging.info("Published report @")
-#     logging.info(publish_url)
-#     return publish_url
+    # @staticmethod
+    # def publish_github(github_user: str, repo_name: str, branch_name: str) -> Tuple[str, str]:
+    #     return (
+    #         f"https://github.com/{github_user}/{repo_name}.git",
+    #         f"https://github.com/{github_user}/{repo_name}/blob/{branch_name}",
+    #     )

bencher/example/benchmark_data.py

@@ -0,0 +1,196 @@
+"""This file contains an example of how to define benchmarking parameters sweeps. Categorical values are defined as enums and passed to EnumSweep classes, other types of sweeps are defined by their respective classes.
+
+You can define a subclass which contains an input configuration which can be passed to a function in a type safe way. You can combine the subclass with a higher level class which contains more configuation parameters.  This is to help manage the complexity of large configuration/parameter spaces. 
+"""
+
+import math
+import random
+from enum import auto
+from strenum import StrEnum
+from bencher.variables.inputs import IntSweep, FloatSweep, StringSweep, EnumSweep, BoolSweep
+from bencher.variables.results import ResultVar, OptDir
+from bencher.variables.parametrised_sweep import ParametrizedSweep
+
+
+class PostprocessFn(StrEnum):
+    """Apply a postprocessing step to the data"""
+
+    absolute = auto()  # return the abs of the output data
+    negate = auto()  # return the negative of the output data
+
+
+class NoiseDistribution(StrEnum):
+    """A categorical variable describing the types of random noise"""
+
+    uniform = auto()  # uniform random noiase
+    gaussian = auto()  # gaussian noise
+    lognorm = auto()  # lognorm noise
+
+
+class NoiseCfg(ParametrizedSweep):
+    """A class for storing the parameters for generating various types of noise"""
+
+    noisy = BoolSweep(
+        default=False, doc="Optionally add random noise to the output of the function"
+    )
+
+    noise_distribution = EnumSweep(NoiseDistribution, doc=NoiseDistribution.__doc__)
+
+    sigma = FloatSweep(
+        default=1,
+        bounds=[0, 10],
+        doc="The standard deviation of the noise",
+        units="v",
+    )
+
+
+def calculate_noise(config: NoiseCfg) -> float:
+    """Generate a float value based on a noise distribution and scale
+
+    Args:
+        config (NoiseCfg): see NoiseCfg type
+
+    Returns:
+        float: a noisy float value
+    """
+
+    noise = 0.0
+    if config.noisy:
+        match config.noise_distribution:
+            case NoiseDistribution.uniform:
+                noise = random.uniform(0, config.sigma)
+            case NoiseDistribution.gaussian:
+                noise = random.gauss(0, config.sigma)
+            case NoiseDistribution.lognorm:
+                noise = random.lognormvariate(0, config.sigma)
+
+    return noise
+
+
+class ExampleBenchCfgIn(NoiseCfg):
+    """A class for representing a set of configuration options for the example worker. This class inherits from NoiseCfg so it also stores all its values as well."""
+
+    theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
+    offset = FloatSweep(default=0, bounds=[0, 0.3], doc="dc offset", units="v", samples=30)
+    postprocess_fn = EnumSweep(PostprocessFn)
+
+
+class ExampleBenchCfgOut(ParametrizedSweep):
+    out_sin = ResultVar(units="v", direction=OptDir.minimize, doc="sin of theta with some noise")
+    out_cos = ResultVar(units="v", direction=OptDir.minimize, doc="cos of theta with some noise")
+    out_bool = ResultVar(units="%", doc="sin > 0.5")
+
+
+def negate_fn(fn_input: float):
+    """returns the negative of the input
+
+    Args:
+        fn_input (float): any float value
+
+    Returns:
+        float: negative of the input
+    """
+    return -fn_input
+
+
+def bench_function(cfg: ExampleBenchCfgIn) -> ExampleBenchCfgOut:
+    """Takes an ExampleBenchCfgIn and returns a ExampleBenchCfgOut output"""
+    out = ExampleBenchCfgOut()
+    noise = calculate_noise(cfg)
+
+    postprocess_fn = abs if cfg.postprocess_fn == PostprocessFn.absolute else negate_fn
+
+    out.out_sin = postprocess_fn(cfg.offset + math.sin(cfg.theta) + noise)
+    out.out_cos = postprocess_fn(cfg.offset + math.cos(cfg.theta) + noise)
+
+    out.out_bool = out.out_sin > 0.5
+    return out
+
+
+class ExampleBenchCfg(ParametrizedSweep):
+    theta = FloatSweep(default=0, bounds=[0, math.pi], doc="Input angle", units="rad", samples=30)
+    offset = FloatSweep(default=0, bounds=[0, 0.3], doc="dc offset", units="v", samples=30)
+    postprocess_fn = EnumSweep(PostprocessFn)
+
+    noisy = BoolSweep(
+        default=False, doc="Optionally add random noise to the output of the function"
+    )
+    noise_distribution = EnumSweep(NoiseDistribution, doc=NoiseDistribution.__doc__)
+    sigma = FloatSweep(
+        default=1,
+        bounds=[0, 10],
+        doc="The standard deviation of the noise",
+        units="v",
+    )
+
+    out_sin = ResultVar(units="v", direction=OptDir.minimize, doc="sin of theta with some noise")
+    out_cos = ResultVar(units="v", direction=OptDir.minimize, doc="cos of theta with some noise")
+    out_bool = ResultVar(units="%", doc="sin > 0.5")
+
+    def __call__(self, **kwwargs) -> dict:
+        self.update_params_from_kwargs(**kwwargs)
+
+        noise = self.calculate_noise()
+        postprocess_fn = abs if self.postprocess_fn == PostprocessFn.absolute else negate_fn
+
+        self.out_sin = postprocess_fn(self.offset + math.sin(self.theta) + noise)
+        self.out_cos = postprocess_fn(self.offset + math.cos(self.theta) + noise)
+        self.out_bool = self.out_sin > 0.5
+        return self.get_results_values_as_dict()
+
+    def calculate_noise(self):
+        noise = 0.0
+        if self.noisy:
+            match self.noise_distribution:
+                case NoiseDistribution.uniform:
+                    noise = random.uniform(0, self.sigma)
+                case NoiseDistribution.gaussian:
+                    noise = random.gauss(0, self.sigma)
+                case NoiseDistribution.lognorm:
+                    noise = random.lognormvariate(0, self.sigma)
+
+        return noise
+
+
+def call(**kwargs) -> dict:
+    return ExampleBenchCfg().__call__(**kwargs)
+
+
+class AllSweepVars(ParametrizedSweep):
+    """A class containing all the sweep types, This class is used for unit testing how the configuration classes are serialised and hashed"""
+
+    var_float = FloatSweep(default=5, bounds=(0, 10), units="m/s")
+    var_int = IntSweep(default=3, bounds=[0, 4])
+    var_int_big = IntSweep(default=0, bounds=[0, 100], samples=3)
+    var_bool = BoolSweep()
+    var_string = StringSweep(["string1", "string2"])
+    var_enum = EnumSweep(PostprocessFn)
+
+    result = ResultVar()
+
+    def __call__(self, **kwargs) -> dict:
+        self.update_params_from_kwargs(**kwargs)
+        self.result = self.var_float + self.var_int
+        return self.get_results_values_as_dict()
+
+
+class SimpleBenchClass(ParametrizedSweep):
+    var1 = IntSweep(default=0, bounds=[0, 2])
+
+    result = ResultVar()
+
+    def __call__(self, **kwargs) -> dict:
+        self.update_params_from_kwargs(**kwargs)
+        self.result = self.var1
+        return self.get_results_values_as_dict()
+
+
+class SimpleBenchClassFloat(ParametrizedSweep):
+    var1 = FloatSweep(bounds=[0, 100])
+
+    result = ResultVar()
+
+    def __call__(self, **kwargs) -> dict:
+        self.update_params_from_kwargs(**kwargs)
+        self.result = self.var1
+        return self.get_results_values_as_dict()

bencher/example/example_all.py

@@ -0,0 +1,45 @@
+import bencher as bch
+
+from bencher.example.example_categorical import example_categorical
+from bencher.example.example_floats import example_floats
+from bencher.example.example_floats2D import example_floats2D
+from bencher.example.example_pareto import example_pareto
+from bencher.example.example_simple_cat import example_1D_cat
+from bencher.example.example_simple_float import example_1D_float
+from bencher.example.example_float_cat import example_float_cat
+from bencher.example.example_time_event import example_time_event
+from bencher.example.example_float3D import example_floats3D
+
+from bencher.example.example_custom_sweep import example_custom_sweep
+from bencher.example.example_holosweep import example_holosweep
+from bencher.example.example_holosweep_tap import example_holosweep_tap
+
+from bencher.example.optuna.example_optuna import optuna_rastrigin
+from bencher.example.example_sample_cache import example_sample_cache
+
+# from bencher.example.example_workflow import example_floats2D_workflow, example_floats3D_workflow
+
+
+if __name__ == "__main__":
+    run_cfg = bch.BenchRunCfg()
+    run_cfg.overwrite_sample_cache = True
+    bench_runner = bch.BenchRunner("bencher_examples", run_cfg=run_cfg)
+
+    bench_runner.add_run(example_categorical)
+    bench_runner.add_run(example_floats)
+    bench_runner.add_run(example_floats2D)
+    bench_runner.add_run(example_floats3D)
+    bench_runner.add_run(example_1D_cat)
+    bench_runner.add_run(example_1D_float)
+    bench_runner.add_run(example_pareto)
+    bench_runner.add_run(example_float_cat)
+    bench_runner.add_run(example_time_event)
+    bench_runner.add_run(example_custom_sweep)
+    bench_runner.add_run(example_holosweep)
+    bench_runner.add_run(example_holosweep_tap)
+    bench_runner.add_run(optuna_rastrigin)
+    bench_runner.add_run(example_sample_cache)
+
+    # bench_runner.run(level=2, show=True, grouped=True)
+
+    bench_runner.run(level=4, show=True, grouped=True, save=False)

bencher/example/example_categorical.py

@@ -0,0 +1,99 @@
+# pylint: disable=duplicate-code
+
+import bencher as bch
+
+# All the examples will be using the data structures and benchmark function defined in this file
+from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+
+
+def example_categorical(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    """Example of how to perform a categorical parameter sweep
+
+    Args:
+        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+    Returns:
+        Bench: results of the parameter sweep
+    """
+
+    with open("README.md", "r", encoding="utf-8") as file:
+        readme = file.read()
+
+    # run_cfg.over_time = True
+
+    bench = bch.Bench(
+        "Bencher_Example_Categorical",
+        bench_function,
+        ExampleBenchCfgIn,
+        run_cfg=run_cfg,
+        report=report,
+    )
+
+    bench.report.append(readme, "Intro")
+
+    bench.plot_sweep(
+        input_vars=[ExampleBenchCfgIn.param.noisy],
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        title="Categorical 1D Example",
+        description="""This example shows how to sample categorical values. The same objective from the float examples is used but theta is kept constant with a value of 0 (as described in the ExampleBenchCfgIn class definition).
+
+        
+```
+    def bench_function(cfg: ExampleBenchCfgIn) -> ExampleBenchCfgOut:
+        "Takes an ExampleBenchCfgIn and returns a ExampleBenchCfgOut output"
+        out = ExampleBenchCfgOut()
+        noise = calculate_noise(cfg)
+        offset = 0.0
+
+        postprocess_fn = abs if cfg.postprocess_fn == PostprocessFn.absolute else negate_fn
+
+        out.out_sin = postprocess_fn(offset + math.sin(cfg.theta) + noise)          
+        return out
+```
+        
+        """,
+        post_description="The plot shows when noise=True the output has uniform random noise.",
+    )
+
+    bench.plot_sweep(
+        input_vars=[ExampleBenchCfgIn.param.noisy, ExampleBenchCfgIn.param.noise_distribution],
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        title="Categorical 2D Example",
+        description="""Adding another categorical value creates a facet plot over that dimension""",
+        post_description="The output shows swarm plots of different noise distributions",
+    )
+
+    bench.plot_sweep(
+        input_vars=[
+            ExampleBenchCfgIn.param.noisy,
+            ExampleBenchCfgIn.param.noise_distribution,
+            ExampleBenchCfgIn.param.postprocess_fn,
+        ],
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        title="Categorical 3D Example",
+        description="""Adding another categorical value extends the facets to the right""",
+        post_description="The output shows swarm plots of different noise distributions",
+    )
+
+    run_cfg.over_time = True
+    bench.plot_sweep(
+        input_vars=[
+            ExampleBenchCfgIn.param.noisy,
+            ExampleBenchCfgIn.param.noise_distribution,
+            ExampleBenchCfgIn.param.postprocess_fn,
+        ],
+        title="Categorical 3D Example Over Time",
+        result_vars=[ExampleBenchCfgOut.param.out_sin],
+        description="""Lastly, what if you want to track these distributions over time? Set over_time=True and bencher will cache and display historical resuts alongside the latest result.  Use clear_history=True to clear that cache.""",
+        post_description="The output shows faceted line plot with confidence intervals for the mean value over time.",
+        run_cfg=run_cfg,
+    )
+
+    return bench
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    example_categorical(ex_run_cfg).report.show()

bencher/example/example_composable_container.py

@@ -0,0 +1,106 @@
+import bencher as bch
+
+from bencher.example.example_image import BenchPolygons
+
+
+class BenchComposableContainerImage(BenchPolygons):
+    compose_method = bch.EnumSweep(bch.ComposeType)
+    labels = bch.BoolSweep()
+    num_frames = bch.IntSweep(default=5, bounds=[1, 100])
+    polygon_vid = bch.ResultVideo()
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        var_name = None
+        var_value = None
+
+        if self.labels:
+            var_name = "sides"
+            var_value = self.sides
+        vr = bch.ComposableContainerVideo()
+        for i in range(self.num_frames):
+            res = super().__call__(start_angle=i)
+            print(res)
+            vr.append(res["polygon"])
+        self.polygon_vid = vr.to_video(
+            bch.RenderCfg(
+                compose_method=self.compose_method,
+                var_name=var_name,
+                var_value=var_value,
+                max_frame_duration=1.0 / 20.0,
+            )
+        )
+        return self.get_results_values_as_dict()
+
+
+class BenchComposableContainerVideo(bch.ParametrizedSweep):
+    unequal_length = bch.BoolSweep()
+    compose_method = bch.EnumSweep(bch.ComposeType)
+    labels = bch.BoolSweep()
+    polygon_vid = bch.ResultVideo()
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        vr = bch.ComposableContainerVideo()
+        for i in range(3, 5):
+            num_frames = i * 10 if self.unequal_length else 5
+            res = BenchComposableContainerImage().__call__(
+                compose_method=bch.ComposeType.sequence, sides=i, num_frames=num_frames
+            )
+            vr.append(res["polygon_vid"])
+
+        self.polygon_vid = vr.to_video(bch.RenderCfg(compose_method=kwargs.get("compose_method")))
+        return self.get_results_values_as_dict()
+
+
+def example_composable_container_image(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    bench = BenchComposableContainerImage().to_bench(run_cfg, report)
+    bench.result_vars = ["polygon_vid"]
+    # bench.add_plot_callback(bch.BenchResult.to_panes)
+    # bench.add_plot_callback(bch.BenchResult.to_video_grid, result_types=(bch.ResultVideo))
+    # bench.add_plot_callback(bch.BenchResult.to_video_summary, result_types=(bch.ResultVideo))
+    # bench.plot_sweep(input_vars=["compose_method", "labels"])
+
+    bench.plot_sweep(input_vars=["compose_method"])
+
+    # bench.compose_
+    # bench.plot_sweep(
+    # input_vars=[bch.p("num_frames", [2, 8, 20])],
+    # const_vars=dict(compose_method=bch.ComposeType.sequence),
+    # )
+
+    return bench
+
+
+def example_composable_container_video(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    bench = BenchComposableContainerVideo().to_bench(run_cfg, report)
+
+    bench.result_vars = ["polygon_vid"]
+    bench.add_plot_callback(bch.BenchResult.to_panes)
+    bench.add_plot_callback(bch.BenchResult.to_video_grid, result_types=(bch.ResultVideo))
+    bench.add_plot_callback(bch.BenchResult.to_video_summary, result_types=(bch.ResultVideo))
+    bench.plot_sweep(input_vars=["compose_method", "labels"], const_vars=dict(unequal_length=True))
+
+    res = bench.plot_sweep(
+        input_vars=[],
+        const_vars=dict(unequal_length=False, compose_method=bch.ComposeType.sequence),
+        plot_callbacks=False,
+    )
+
+    bench.report.append(res.to_video_grid())
+
+    return bench
+
+
+if __name__ == "__main__":
+    ex_run_cfg = bch.BenchRunCfg()
+    ex_run_cfg.use_sample_cache = False
+    # ex_run_cfg.level = 2
+    ex_report = bch.BenchReport()
+    example_composable_container_image(ex_run_cfg, report=ex_report)
+    # example_composable_container_video(ex_run_cfg, report=ex_report)
+    ex_report.show()