holobench 1.33.2__tar.gz → 1.35.0__tar.gz

{holobench-1.33.2 → holobench-1.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: holobench
-Version: 1.33.2
+Version: 1.35.0
 Summary: A package for benchmarking the performance of arbitrary functions
 Project-URL: Repository, https://github.com/dyson-ai/bencher
 Project-URL: Home, https://github.com/dyson-ai/bencher
@@ -38,7 +38,7 @@ Requires-Dist: pre-commit<=4.0.1; extra == 'test'
 Requires-Dist: pylint<=3.3.3,>=3.2.5; extra == 'test'
 Requires-Dist: pytest-cov<=6.0.0,>=4.1; extra == 'test'
 Requires-Dist: pytest<=8.3.4,>=7.4; extra == 'test'
-Requires-Dist: ruff<=0.8.4,>=0.5.0; extra == 'test'
+Requires-Dist: ruff<=0.8.5,>=0.5.0; extra == 'test'
 Description-Content-Type: text/markdown
 
 # Bencher

{holobench-1.33.2 → holobench-1.35.0}/bencher/__init__.py

@@ -56,7 +56,13 @@ from .utils import (
     publish_file,
     github_content,
 )
-from .utils_rerun import publish_and_view_rrd, rrd_to_pane, capture_rerun_window
+
+try:
+    from .utils_rerun import publish_and_view_rrd, rrd_to_pane, capture_rerun_window
+    from .flask_server import run_flask_in_thread
+except ModuleNotFoundError as e:
+    pass
+
 
 from .plotting.plot_filter import VarRange, PlotFilter
 from .variables.parametrised_sweep import ParametrizedSweep
@@ -68,5 +74,3 @@ from .bench_report import BenchReport, GithubPagesCfg
 from .job import Executors
 from .video_writer import VideoWriter, add_image
 from .class_enum import ClassEnum, ExampleEnum
-
-from .flask_server import run_flask_in_thread

{holobench-1.33.2 → holobench-1.35.0}/bencher/bencher.py

@@ -371,7 +371,7 @@ class Bench(BenchPlotServer):
                     title += "s"
                 title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in const_vars_in])
             else:
-                raise RuntimeError("you must pass a title, or define inputs or consts")
+                title = " ".join([i.name for i in result_vars_in])
 
         if run_cfg.level > 0:
             inputs = []
@@ -871,6 +871,9 @@ class Bench(BenchPlotServer):
     def get_result(self, index: int = -1) -> BenchResult:
         return self.results[index]
 
+    def get_ds(self, index: int = -1) -> xr.Dataset:
+        return self.get_result(index).to_xarray()
+
     def publish(self, remote_callback: Callable) -> str:
         branch_name = f"{self.bench_name}_{self.run_cfg.run_tag}"
         return self.report.publish(remote_callback, branch_name=branch_name)

{holobench-1.33.2 → holobench-1.35.0}/bencher/example/example_simple_float.py

@@ -21,6 +21,7 @@ def example_1D_float(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport =
 
     bench = SimpleFloat().to_bench(run_cfg, report)
     bench.plot_sweep()
+    bench.report.append(bench.get_result().to_explorer())
     return bench
 
 

holobench-1.35.0/bencher/example/inputs_0D/example_0D.py

@@ -0,0 +1,34 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+import random
+
+
+class SimpleFloat0D(bch.ParametrizedSweep):
+    """This class has 0 input dimensions and 1 output dimensions.  It samples from a gaussian distribution"""
+
+    # This defines a variable that we want to plot
+    output = bch.ResultVar(units="ul", doc="a sample from a gaussian distribution")
+
+    def __call__(self, **kwargs) -> dict:
+        """Generate a sample from a uniform distribution
+
+        Returns:
+            dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
+        """
+
+        self.output = random.gauss(mu=0.0, sigma=1.0)
+        return super().__call__(**kwargs)
+
+
+def example_0D(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = SimpleFloat0D().to_bench(run_cfg, report)
+    bench.plot_sweep()
+    return bench
+
+
+if __name__ == "__main__":
+    run_config = bch.BenchRunCfg(repeats=100)
+    example_0D(run_config).report.show()

holobench-1.35.0/bencher/example/inputs_1D/example_1D.py

@@ -0,0 +1,53 @@
+"""This file has some examples for how to perform basic benchmarking parameter sweeps"""
+
+import bencher as bch
+
+
+class DataSource:
+    def __init__(self):
+        self.data = [
+            [0, 0, 0, 0],
+            [1, 1, 1, 1],
+            [1, 1, 1, 1],
+            [2, 1, 1, 0],
+            [2, 2, 0, 0],
+            [2, 2, 1, 1],
+        ]
+
+    def call(self, index, repeat):
+        return self.data[index][repeat - 1]
+
+
+class Example1D(bch.ParametrizedSweep):
+    index = bch.IntSweep(default=0, bounds=[0, 5], doc="Input angle", units="rad", samples=30)
+    output = bch.ResultVar(units="v", doc="sin of theta")
+
+    def __call__(self, **kwargs):
+        self.update_params_from_kwargs(**kwargs)
+        self.output = DataSource().call(self.index, kwargs["repeat"])
+        return super().__call__(**kwargs)
+
+
+def example_1D_float_repeats(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+
+    bench = Example1D().to_bench(run_cfg, report)
+    # bench.plot_sweep(pass_repeat=True,plot_callbacks=False)
+
+    # res = bench.get_result()
+    bench.run_cfg = bch.BenchRunCfg(repeats=4)
+    bench.plot_sweep(pass_repeat=True)
+
+    res = bench.get_result()
+    bench.report.append(res.to_auto())
+    bench.report.append(res.to_scatter())
+    bench.report.append(res.to_scatter_jitter(override=True))
+
+    # bench.report.append()
+    return bench
+
+
+if __name__ == "__main__":
+    example_1D_float_repeats().report.show()

{holobench-1.33.2 → holobench-1.35.0}/bencher/plotting/plot_filter.py

@@ -64,16 +64,24 @@ class PlotFilter:
     repeats_range: VarRange = VarRange(1, None)
     input_range: VarRange = VarRange(1, None)
 
-    def matches_result(self, plt_cnt_cfg: PltCntCfg, plot_name: str) -> PlotMatchesResult:
+    def matches_result(
+        self, plt_cnt_cfg: PltCntCfg, plot_name: str, override: bool = False
+    ) -> PlotMatchesResult:
         """Checks if the result data signature matches the type of data the plot is able to display."""
-        return PlotMatchesResult(self, plt_cnt_cfg, plot_name)
+        return PlotMatchesResult(self, plt_cnt_cfg, plot_name, override)
 
 
 # @dataclass
 class PlotMatchesResult:
     """Stores information about which properties match the requirements of a particular plotter"""
 
-    def __init__(self, plot_filter: PlotFilter, plt_cnt_cfg: PltCntCfg, plot_name: str):
+    def __init__(
+        self,
+        plot_filter: PlotFilter,
+        plt_cnt_cfg: PltCntCfg,
+        plot_name: str,
+        override: bool = False,
+    ):
         match_info = []
         matches = []
 
@@ -92,8 +100,11 @@ class PlotMatchesResult:
             matches.append(match)
             if not match:
                 match_info.append(info)
-
-        self.overall = all(matches)
+        if override:
+            match_info.append(f"override: {override}")
+            self.overall = True
+        else:
+            self.overall = all(matches)
 
         match_info.insert(0, f"plot {plot_name} matches: {self.overall}")
         self.matches_info = "\n".join(match_info).strip()

{holobench-1.33.2 → holobench-1.35.0}/bencher/results/bench_result.py

@@ -7,11 +7,12 @@ from bencher.results.video_summary import VideoSummaryResult
 from bencher.results.panel_result import PanelResult
 from bencher.results.plotly_result import PlotlyResult
 from bencher.results.holoview_result import HoloviewResult
+from bencher.results.hvplot_result import HvplotResult
 from bencher.results.dataset_result import DataSetResult
 from bencher.utils import listify
 
 
-class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult, DataSetResult):
+class BenchResult(PlotlyResult, HoloviewResult, HvplotResult, VideoSummaryResult, DataSetResult):  # noqa pylint: disable=too-many-ancestors
     """Contains the results of the benchmark and has methods to cast the results to various datatypes and graphical representations"""
 
     def __init__(self, bench_cfg) -> None:
@@ -28,6 +29,7 @@ class BenchResult(PlotlyResult, HoloviewResult, VideoSummaryResult, DataSetResul
             HoloviewResult.to_curve,
             HoloviewResult.to_line,
             HoloviewResult.to_heatmap,
+            HvplotResult.to_histogram,
             PlotlyResult.to_volume,
             # PanelResult.to_video,
             PanelResult.to_panes,

{holobench-1.33.2 → holobench-1.35.0}/bencher/results/bench_result_base.py

@@ -21,7 +21,9 @@ from bencher.utils import listify
 
 from bencher.variables.results import ResultReference, ResultDataSet
 
-from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
+from bencher.results.composable_container.composable_container_panel import (
+    ComposableContainerPanel,
+)
 
 # todo add plugins
 # https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
@@ -55,7 +57,10 @@ class BenchResultBase(OptunaResult):
         return self.ds.count()
 
     def to_hv_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+        self,
+        reduce: ReduceType = ReduceType.AUTO,
+        result_var: ResultVar = None,
+        level: int = None,
     ) -> hv.Dataset:
         """Generate a holoviews dataset from the xarray dataset.
 
@@ -72,7 +77,10 @@ class BenchResultBase(OptunaResult):
         return hv.Dataset(self.to_dataset(reduce, result_var, level))
 
     def to_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None, level: int = None
+        self,
+        reduce: ReduceType = ReduceType.AUTO,
+        result_var: ResultVar = None,
+        level: int = None,
     ) -> xr.Dataset:
         """Generate a summarised xarray dataset.
 
@@ -331,6 +339,7 @@ class BenchResultBase(OptunaResult):
         result_var: ResultVar = None,
         result_types=None,
         pane_collection: pn.pane = None,
+        override=False,
         **kwargs,
     ):
         plot_filter = PlotFilter(
@@ -342,7 +351,9 @@ class BenchResultBase(OptunaResult):
             repeats_range=repeats_range,
             input_range=input_range,
         )
-        matches_res = plot_filter.matches_result(self.plt_cnt_cfg, callable_name(plot_callback))
+        matches_res = plot_filter.matches_result(
+            self.plt_cnt_cfg, callable_name(plot_callback), override
+        )
         if matches_res.overall:
             return self.map_plot_panes(
                 plot_callback=plot_callback,
@@ -400,7 +411,9 @@ class BenchResultBase(OptunaResult):
             dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
 
             outer_container = ComposableContainerPanel(
-                name=" vs ".join(dims), background_col=dim_color, horizontal=not horizontal
+                name=" vs ".join(dims),
+                background_col=dim_color,
+                horizontal=not horizontal,
             )
             max_len = 0
             for i in range(dataset.sizes[selected_dim]):

{holobench-1.33.2 → holobench-1.35.0}/bencher/results/holoview_result.py

@@ -6,9 +6,15 @@ import holoviews as hv
 from param import Parameter
 from functools import partial
 import hvplot.xarray  # noqa pylint: disable=duplicate-code,unused-import
+import hvplot.pandas  # noqa pylint: disable=duplicate-code,unused-import
 import xarray as xr
 
-from bencher.utils import hmap_canonical_input, get_nearest_coords, get_nearest_coords1D, listify
+from bencher.utils import (
+    hmap_canonical_input,
+    get_nearest_coords,
+    get_nearest_coords1D,
+    listify,
+)
 from bencher.results.panel_result import PanelResult
 from bencher.results.bench_result_base import ReduceType
 
@@ -81,7 +87,9 @@ class HoloviewResult(PanelResult):
 
         # return time_widget_args
 
-    def to_bar(self, result_var: Parameter = None, **kwargs) -> Optional[pn.panel]:
+    def to_bar(
+        self, result_var: Parameter = None, override: bool = False, **kwargs
+    ) -> Optional[pn.panel]:
         return self.filter(
             self.to_bar_ds,
             float_range=VarRange(0, 0),
@@ -92,6 +100,7 @@ class HoloviewResult(PanelResult):
             target_dimension=2,
             result_var=result_var,
             result_types=(ResultVar),
+            override=override,
             **kwargs,
         )
 
@@ -105,7 +114,11 @@ class HoloviewResult(PanelResult):
         return da_plot.hvplot.bar(by=by, **time_widget_args, **kwargs)
 
     def hv_container_ds(
-        self, dataset: xr.Dataset, result_var: Parameter, container: hv.Chart = None, **kwargs
+        self,
+        dataset: xr.Dataset,
+        result_var: Parameter,
+        container: hv.Chart = None,
+        **kwargs,
     ):
         return hv.Dataset(dataset[result_var.name]).to(container).opts(**kwargs)
 
@@ -428,7 +441,9 @@ class HoloviewResult(PanelResult):
 
     def to_scatter(self, **kwargs) -> Optional[pn.panel]:
         match_res = PlotFilter(
-            float_range=VarRange(0, 0), cat_range=VarRange(0, None), repeats_range=VarRange(1, 1)
+            float_range=VarRange(0, 0),
+            cat_range=VarRange(0, None),
+            repeats_range=VarRange(1, 1),
         ).matches_result(self.plt_cnt_cfg, "to_hvplot_scatter")
         if match_res.overall:
             hv_ds = self.to_hv_dataset(ReduceType.SQUEEZE)
@@ -530,17 +545,22 @@ class HoloviewResult(PanelResult):
     def to_scatter_jitter(
         self,
         result_var: Parameter = None,
+        override: bool = False,
         **kwargs,  # pylint: disable=unused-argument
     ) -> List[hv.Scatter]:
-        return self.overlay_plots(partial(self.to_scatter_jitter_single, **kwargs))
+        return self.overlay_plots(
+            partial(self.to_scatter_jitter_single, override=override, **kwargs)
+        )
 
-    def to_scatter_jitter_single(self, result_var: Parameter, **kwargs) -> Optional[hv.Scatter]:
+    def to_scatter_jitter_single(
+        self, result_var: Parameter, override: bool = True, **kwargs
+    ) -> Optional[hv.Scatter]:
         matches = PlotFilter(
             float_range=VarRange(0, 0),
             cat_range=VarRange(0, None),
             repeats_range=VarRange(2, None),
             input_range=VarRange(1, None),
-        ).matches_result(self.plt_cnt_cfg, "to_scatter_jitter")
+        ).matches_result(self.plt_cnt_cfg, "to_scatter_jitter", override)
         if matches.overall:
             ds = self.to_hv_dataset(ReduceType.NONE)
             pt = (
@@ -639,7 +659,13 @@ class HoloviewResult(PanelResult):
     def to_table(self):
         return self.to(hv.Table, ReduceType.SQUEEZE)
 
-    def to_surface(self, result_var: Parameter = None, **kwargs) -> Optional[pn.panel]:
+    def to_tabulator(self, **kwargs):
+        """Passes the data to the panel Tabulator type to display an interactive table
+        see https://panel.holoviz.org/reference/widgets/Tabulator.html for extra options
+        """
+        return pn.widgets.Tabulator(self.to_pandas(), **kwargs)
+
+    def to_surface(self, result_var: Parameter = None, **kwargs) -> Optional[pn.pane.Pane]:
         return self.filter(
             self.to_surface_ds,
             float_range=VarRange(2, None),

holobench-1.35.0/bencher/results/hvplot_result.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+from typing import Optional
+import panel as pn
+from param import Parameter
+import hvplot.xarray  # noqa pylint: disable=duplicate-code,unused-import
+import hvplot.pandas  # noqa pylint: disable=duplicate-code,unused-import
+import xarray as xr
+
+from bencher.results.panel_result import PanelResult
+from bencher.results.bench_result_base import ReduceType
+
+from bencher.plotting.plot_filter import VarRange
+from bencher.variables.results import ResultVar
+
+
+class HvplotResult(PanelResult):
+    def to_explorer(self) -> pn.pane.Pane:
+        """Produces a hvplot explorer instance to explore the generated dataset
+        see: https://hvplot.holoviz.org/getting_started/explorer.html
+
+        Returns:
+            pn.pane.Pane: A dynamic pane for exploring a dataset
+        """
+
+        if len(self.bench_cfg.input_vars) > 0:
+            return self.to_xarray().hvplot.explorer()
+
+        # For some reason hvplot doesn't like 1D datasets in xarray, so convert to pandas which it has no problem with
+        # TODO look into why this is, its probably due to how I am setting up the indexing in xarray.
+        return self.to_pandas().hvplot.explorer()
+
+    def to_histogram(self, result_var: Parameter = None, **kwargs) -> Optional[pn.pane.Pane]:
+        return self.filter(
+            self.to_histogram_ds,
+            float_range=VarRange(0, 0),
+            cat_range=VarRange(0, None),
+            input_range=VarRange(0, 0),
+            reduce=ReduceType.NONE,
+            target_dimension=2,
+            result_var=result_var,
+            result_types=(ResultVar),
+            **kwargs,
+        )
+
+    def to_histogram_ds(self, dataset: xr.Dataset, result_var: Parameter, **kwargs):
+        return dataset.hvplot(
+            kind="hist",
+            y=[result_var.name],
+            ylabel="count",
+            legend="bottom_right",
+            widget_location="bottom",
+            title=f"{result_var.name} vs Count",
+            **kwargs,
+        )

{holobench-1.33.2 → holobench-1.35.0}/bencher/results/optuna_result.py

@@ -277,7 +277,9 @@ class OptunaResult:
                 if len(target_names) <= 3:
                     study_pane.append(
                         plot_pareto_front(
-                            study, target_names=target_names, include_dominated_trials=False
+                            study,
+                            target_names=target_names,
+                            include_dominated_trials=False,
                         )
                     )
                 else:
@@ -312,7 +314,10 @@ class OptunaResult:
 
             param_str = "\n".join(param_str)
             study_pane.append(
-                pn.Row(pn.pane.Markdown(f"## Best Parameters\n```text\n{param_str}"), **kwargs),
+                pn.Row(
+                    pn.pane.Markdown(f"## Best Parameters\n```text\n{param_str}"),
+                    **kwargs,
+                ),
             )
 
             study_repeats_pane.append(study_pane)

{holobench-1.33.2 → holobench-1.35.0}/bencher/variables/parametrised_sweep.py

@@ -192,7 +192,12 @@ class ParametrizedSweep(Parameterized):
             )
         )
 
-    def __call__(self, **kwargs):
+    def __call__(self, **kwargs) -> dict:
+        """This is the function that is called to record data samples in the benchmarking function.  It should be overridden with your custom logic and then call the parent method  "return super().__call__(**kwargs)"
+
+        Returns:
+            dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
+        """
         return self.get_results_values_as_dict()
 
     def plot_hmap(self, **kwargs):

{holobench-1.33.2 → holobench-1.35.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.33.2"
+version = "1.35.0"
 
 authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
 description = "A package for benchmarking the performance of arbitrary functions"
@@ -58,7 +58,7 @@ test = [
     "pytest-cov>=4.1,<=6.0.0",
     "pytest>=7.4,<=8.3.4",
     "hypothesis>=6.104.2,<=6.123.2",
-    "ruff>=0.5.0,<=0.8.4",
+    "ruff>=0.5.0,<=0.8.5",
     "coverage>=7.5.4,<=7.6.10",
     "pre-commit<=4.0.1",
 ]
@@ -88,26 +88,26 @@ format = "ruff format ."
 check-clean-workspace = "git diff --exit-code"
 ruff-lint = "ruff check . --fix"
 pylint = "pylint --version && echo 'running pylint...' && pylint $(git ls-files '*.py')"
-lint = { depends_on = ["ruff-lint", "pylint"] }
-style = { depends_on = ["format", "lint"] }
+lint = { depends-on = ["ruff-lint", "pylint"] }
+style = { depends-on = ["format", "lint"] }
 commit-format = "git commit -a -m'autoformat code' || true"
 test = "pytest"
 coverage = "coverage run -m pytest && coverage xml -o coverage.xml"
 coverage-report = "coverage report -m"
 update-lock = "pixi update && git commit -a -m'update pixi.lock' || true"
 push = "git push"
-update-lock-push = { depends_on = ["update-lock", "push"] }
-fix = { depends_on = ["update-lock", "pre-commit", "format", "ruff-lint"] }
-fix-commit-push = { depends_on = ["fix", "commit-format", "update-lock-push"] }
-ci-no-cover = { depends_on = ["style", "test"] }
-ci = { depends_on = [
+update-lock-push = { depends-on = ["update-lock", "push"] }
+fix = { depends-on = ["update-lock", "format", "ruff-lint", "pre-commit"] }
+fix-commit-push = { depends-on = ["fix", "commit-format", "update-lock-push"] }
+ci-no-cover = { depends-on = ["style", "test"] }
+ci = { depends-on = [
     "format",
     "ruff-lint",
     "pylint",
     "coverage",
     "coverage-report",
 ] }
-ci-push = { depends_on = ["format", "ruff-lint", "update-lock", "ci", "push"] }
+ci-push = { depends-on = ["format", "ruff-lint", "update-lock", "ci", "push"] }
 clear-pixi = "rm -rf .pixi pixi.lock"
 setup-git-merge-driver = "git config merge.ours.driver true"
 update-from-template-repo = "./scripts/update_from_template.sh"
@@ -121,7 +121,7 @@ extension-pkg-whitelist = ["numpy", "scipy"]
 jobs = 16                                    #detect number of cores
 
 [tool.pylint.'MESSAGES CONTROL']
-disable = "C,logging-fstring-interpolation,line-too-long,fixme,missing-module-docstring,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-locals,too-many-branches,too-many-statements,use-dict-literal,duplicate-code,too-many-public-methods,too-many-nested-blocks,cyclic-import, too-many-positional-arguments"
+disable = "C,logging-fstring-interpolation,line-too-long,fixme,missing-module-docstring,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-locals,too-many-branches,too-many-statements,use-dict-literal,duplicate-code,too-many-public-methods,too-many-nested-blocks,cyclic-import,too-many-positional-arguments"
 enable = "no-else-return,consider-using-in"
 
 [tool.coverage.run]