holobench 1.3.5__py3-none-any.whl → 1.22.2__py3-none-any.whl

bencher/__init__.py

@@ -11,6 +11,7 @@ from .variables.results import (
     ResultVar,
     ResultVec,
     ResultHmap,
+    ResultPath,
     ResultVideo,
     ResultImage,
     ResultString,
@@ -30,6 +31,7 @@ from .utils import (
     gen_image_path,
     gen_video_path,
     lerp,
+    tabs_in_markdown,
 )
 from .variables.parametrised_sweep import ParametrizedSweep
 from .caching import CachedParams
@@ -39,3 +41,4 @@ from .results.holoview_result import ReduceType, HoloviewResult
 from .bench_report import BenchReport
 from .job import Executors
 from .video_writer import VideoWriter, add_image
+from .class_enum import ClassEnum, ExampleEnum

bencher/bench_cfg.py

@@ -38,10 +38,6 @@ class BenchRunCfg(BenchPlotSrvCfg):
         doc="If true each time the function is called it will plot a timeseries of historical and the latest result.",
     )
 
-    debug: bool = param.Boolean(
-        False, doc="Debug the sampling faster by reducing the dimension sampling resolution"
-    )
-
     use_optuna: bool = param.Boolean(False, doc="show optuna plots")
 
     summarise_constant_inputs = param.Boolean(
@@ -303,6 +299,11 @@ class BenchCfg(BenchRunCfg):
         doc="store the hash value of the config to avoid having to hash multiple times",
     )
 
+    plot_callbacks = param.List(
+        None,
+        doc="A callable that takes a BenchResult and returns panel representation of the results",
+    )
+
     def __init__(self, **params):
         super().__init__(**params)
         self.plot_lib = None
@@ -328,7 +329,6 @@ class BenchCfg(BenchRunCfg):
                 hash_sha1(str(self.title)),
                 hash_sha1(self.over_time),
                 repeats_hash,
-                hash_sha1(self.debug),
                 hash_sha1(self.tag),
             )
         )
@@ -344,9 +344,13 @@ class BenchCfg(BenchRunCfg):
     def inputs_as_str(self) -> List[str]:
         return [i.name for i in self.input_vars]
 
-    def describe_sweep(self, width: int = 800) -> pn.pane.Markdown:
+    def describe_sweep(self, width: int = 800, accordion=True) -> pn.pane.Markdown:
         """Produce a markdown summary of the sweep settings"""
-        return pn.pane.Markdown(self.describe_benchmark(), width=width)
+
+        desc = pn.pane.Markdown(self.describe_benchmark(), width=width)
+        if accordion:
+            return pn.Accordion(("Data Collection Parameters", desc))
+        return desc
 
     def describe_benchmark(self) -> str:
         """Generate a string summary of the inputs and results from a BenchCfg
@@ -359,37 +363,30 @@ class BenchCfg(BenchRunCfg):
 
         benchmark_sampling_str.append("Input Variables:")
         for iv in self.input_vars:
-            benchmark_sampling_str.extend(describe_variable(iv, self.debug, True))
+            benchmark_sampling_str.extend(describe_variable(iv, True))
 
         if self.const_vars and (self.summarise_constant_inputs):
             benchmark_sampling_str.append("\nConstants:")
             for cv in self.const_vars:
-                benchmark_sampling_str.extend(describe_variable(cv[0], False, False, cv[1]))
+                benchmark_sampling_str.extend(describe_variable(cv[0], False, cv[1]))
 
         benchmark_sampling_str.append("\nResult Variables:")
         for rv in self.result_vars:
-            benchmark_sampling_str.extend(describe_variable(rv, self.debug, False))
-
-        print_meta = True
-        # if len(self.meta_vars) == 1:
-        #     mv = self.meta_vars[0]
-        #     if mv.name == "repeat" and mv.samples == 1:
-        #         print_meta = False
-
-        if print_meta:
-            benchmark_sampling_str.append("\nMeta Variables:")
-            benchmark_sampling_str.append(f"    run date: {self.run_date}")
-            if self.run_tag is not None and len(self.run_tag) > 0:
-                benchmark_sampling_str.append(f"    run tag: {self.run_tag}")
-            if self.level is not None:
-                benchmark_sampling_str.append(f"    bench level: {self.level}")
-            benchmark_sampling_str.append(f"    use_cache: {self.use_cache}")
-            benchmark_sampling_str.append(f"    use_sample_cache: {self.use_sample_cache}")
-            benchmark_sampling_str.append(f"    only_hash_tag: {self.only_hash_tag}")
-            benchmark_sampling_str.append(f"    parallel: {self.executor}")
-
-            for mv in self.meta_vars:
-                benchmark_sampling_str.extend(describe_variable(mv, self.debug, True))
+            benchmark_sampling_str.extend(describe_variable(rv, False))
+
+        benchmark_sampling_str.append("\nMeta Variables:")
+        benchmark_sampling_str.append(f"    run date: {self.run_date}")
+        if self.run_tag:
+            benchmark_sampling_str.append(f"    run tag: {self.run_tag}")
+        if self.level is not None:
+            benchmark_sampling_str.append(f"    bench level: {self.level}")
+        benchmark_sampling_str.append(f"    use_cache: {self.use_cache}")
+        benchmark_sampling_str.append(f"    use_sample_cache: {self.use_sample_cache}")
+        benchmark_sampling_str.append(f"    only_hash_tag: {self.only_hash_tag}")
+        benchmark_sampling_str.append(f"    executor: {self.executor}")
+
+        for mv in self.meta_vars:
+            benchmark_sampling_str.extend(describe_variable(mv, True))
 
         benchmark_sampling_str.append("```")
 
@@ -427,7 +424,6 @@ class BenchCfg(BenchRunCfg):
         if self.description is not None and description:
             col.append(self.to_description())
         if describe_sweep:
-            col.append(pn.pane.Markdown("## Data Collection Parameters:"))
             col.append(self.describe_sweep())
         if results_suffix:
             col.append(pn.pane.Markdown("## Results:"))
@@ -451,7 +447,7 @@ class DimsCfg:
         self.dims_name = [i.name for i in bench_cfg.all_vars]
 
         self.dim_ranges = []
-        self.dim_ranges = [i.values(bench_cfg.debug) for i in bench_cfg.all_vars]
+        self.dim_ranges = [i.values() for i in bench_cfg.all_vars]
         self.dims_size = [len(p) for p in self.dim_ranges]
         self.dim_ranges_index = [list(range(i)) for i in self.dims_size]
         self.dim_ranges_str = [f"{s}\n" for s in self.dim_ranges]

bencher/bench_plot_server.py

@@ -1,4 +1,5 @@
 """A server for display plots of benchmark results"""
+
 import logging
 import os
 from typing import List, Tuple
@@ -86,6 +87,10 @@ class BenchPlotServer:
             port (int): use a fixed port to lauch the server
         """
 
+        # suppress verbose tornado and bokeh output
+        for logger in ["tornado", "bokeh"]:
+            logging.getLogger(logger).setLevel(logging.WARNING)
+
         if port is not None:
             return pn.serve(
                 plots_instance,
@@ -96,5 +101,4 @@ class BenchPlotServer:
                 show=show,
             )
 
-        logging.getLogger().setLevel(logging.WARNING)
         return pn.serve(plots_instance, title=bench_name, threaded=True, show=show)

bencher/bench_report.py

@@ -3,7 +3,8 @@ from typing import Callable
 import os
 import panel as pn
 from pathlib import Path
-import shutil
+import tempfile
+
 from threading import Thread
 
 from bencher.results.bench_result import BenchResult
@@ -47,7 +48,7 @@ class BenchReport(BenchPlotServer):
         self.pane.append(col)
 
     def append_result(self, bench_res: BenchResult) -> None:
-        self.append_tab(bench_res.to_auto_plots(), bench_res.bench_cfg.title)
+        self.append_tab(bench_res.plot(), bench_res.bench_cfg.title)
 
     def append_tab(self, pane: pn.panel, name: str = None) -> None:
         if pane is not None:
@@ -137,24 +138,23 @@ class BenchReport(BenchPlotServer):
 
         remote, publish_url = remote_callback(branch_name)
 
-        directory = "tmpgit"
-        report_path = self.save(directory, filename="index.html", in_html_folder=False)
-        logging.info(f"created report at: {report_path.absolute()}")
+        with tempfile.TemporaryDirectory() as td:
+            directory = td
+            report_path = self.save(directory, filename="index.html", in_html_folder=False)
+            logging.info(f"created report at: {report_path.absolute()}")
 
-        cd_dir = f"cd {directory} &&"
+            cd_dir = f"cd {directory} &&"
 
-        os.system(f"{cd_dir} git init")
-        os.system(f"{cd_dir} git checkout -b {branch_name}")
-        os.system(f"{cd_dir} git add index.html")
-        os.system(f'{cd_dir} git commit -m "publish {branch_name}"')
-        os.system(f"{cd_dir} git remote add origin {remote}")
-        os.system(f"{cd_dir} git push --set-upstream origin {branch_name} -f")
+            os.system(f"{cd_dir} git init")
+            os.system(f"{cd_dir} git checkout -b {branch_name}")
+            os.system(f"{cd_dir} git add index.html")
+            os.system(f'{cd_dir} git commit -m "publish {branch_name}"')
+            os.system(f"{cd_dir} git remote add origin {remote}")
+            os.system(f"{cd_dir} git push --set-upstream origin {branch_name} -f")
 
         logging.info("Published report @")
         logging.info(publish_url)
 
-        shutil.rmtree(directory)
-
         return publish_url
 
 

bencher/bench_runner.py

@@ -103,7 +103,7 @@ class BenchRunner:
         for r in range(1, repeats + 1):
             for lvl in range(min_level, max_level + 1):
                 if grouped:
-                    report_level = BenchReport(self.name)
+                    report_level = BenchReport(f"{run_cfg.run_tag}_{self.name}")
 
                 for bch_fn in self.bench_fns:
                     run_lvl = deepcopy(run_cfg)
@@ -114,6 +114,7 @@ class BenchRunner:
                         res = bch_fn(run_lvl, report_level)
                     else:
                         res = bch_fn(run_lvl, BenchReport())
+                        res.report.bench_name = f"{run_cfg.run_tag}_{res.report.bench_name}"
                         self.show_publish(res.report, show, publish, save, debug)
                     self.results.append(res)
                 if grouped:

bencher/bencher.py

@@ -10,6 +10,7 @@ import xarray as xr
 from diskcache import Cache
 from contextlib import suppress
 from functools import partial
+import panel as pn
 
 from bencher.worker_job import WorkerJob
 
@@ -23,6 +24,7 @@ from bencher.variables.results import (
     ResultVar,
     ResultVec,
     ResultHmap,
+    ResultPath,
     ResultVideo,
     ResultImage,
     ResultString,
@@ -32,6 +34,7 @@ from bencher.variables.results import (
 from bencher.results.bench_result import BenchResult
 from bencher.variables.parametrised_sweep import ParametrizedSweep
 from bencher.job import Job, FutureCache, JobFuture, Executors
+from bencher.utils import params_to_str
 
 # Customize the formatter
 formatter = logging.Formatter("%(levelname)s: %(message)s")
@@ -164,6 +167,23 @@ class Bench(BenchPlotServer):
 
         self.cache_size = int(100e9)  # default to 100gb
 
+        # self.bench_cfg = BenchCfg()
+
+        # Maybe put this in SweepCfg
+        self.input_vars = None
+        self.result_vars = None
+        self.const_vars = None
+        self.plot_callbacks = []
+        self.plot = True
+
+    def add_plot_callback(self, callback: Callable[[BenchResult], pn.panel], **kwargs) -> None:
+        """Add a plotting callback that will be called on any result produced when calling a sweep funciton.  You can pass additional arguments to the plotting function with kwargs.  e.g.  add_plot_callback(bch.BenchResult.to_video_grid,)
+
+        Args:
+            callback (Callable[[BenchResult], pn.panel]): _description_
+        """
+        self.plot_callbacks.append(partial(callback, **kwargs))
+
     def set_worker(self, worker: Callable, worker_input_cfg: ParametrizedSweep = None) -> None:
         """Set the benchmark worker function and optionally the type the worker expects
 
@@ -187,34 +207,6 @@ class Bench(BenchPlotServer):
             logging.info(f"setting worker {worker}")
         self.worker_input_cfg = worker_input_cfg
 
-    def sweep(
-        self,
-        input_vars: List[ParametrizedSweep] = None,
-        result_vars: List[ParametrizedSweep] = None,
-        const_vars: List[ParametrizedSweep] = None,
-        time_src: datetime = None,
-        description: str = None,
-        post_description: str = None,
-        pass_repeat: bool = False,
-        tag: str = "",
-        run_cfg: BenchRunCfg = None,
-        plot: bool = False,
-    ) -> BenchResult:
-        title = "Sweeping " + " vs ".join([self.get_name(i) for i in input_vars])
-        return self.plot_sweep(
-            title,
-            input_vars=input_vars,
-            result_vars=result_vars,
-            const_vars=const_vars,
-            time_src=time_src,
-            description=description,
-            post_description=post_description,
-            pass_repeat=pass_repeat,
-            tag=tag,
-            run_cfg=run_cfg,
-            plot=plot,
-        )
-
     def sweep_sequential(
         self,
         title="",
@@ -226,15 +218,14 @@ class Bench(BenchPlotServer):
         group_size: int = 1,
         iterations: int = 1,
         relationship_cb=None,
+        plot_callbacks: List | bool = None,
     ) -> List[BenchResult]:
         results = []
         if relationship_cb is None:
             relationship_cb = combinations
         for it in range(iterations):
             for input_group in relationship_cb(input_vars, group_size):
-                title_gen = (
-                    title + "Sweeping " + " vs ".join([self.get_name(i) for i in input_group])
-                )
+                title_gen = title + "Sweeping " + " vs ".join(params_to_str(input_group))
                 if iterations > 1:
                     title_gen += f" iteration:{it}"
                 res = self.plot_sweep(
@@ -243,8 +234,9 @@ class Bench(BenchPlotServer):
                     result_vars=result_vars,
                     const_vars=const_vars,
                     run_cfg=run_cfg,
-                    plot=True,
+                    plot_callbacks=plot_callbacks,
                 )
+
                 if optimise_var is not None:
                     const_vars = res.get_optimal_inputs(optimise_var, True)
                 results.append(res)
@@ -262,7 +254,7 @@ class Bench(BenchPlotServer):
         pass_repeat: bool = False,
         tag: str = "",
         run_cfg: BenchRunCfg = None,
-        plot: bool = True,
+        plot_callbacks: List | bool = None,
     ) -> BenchResult:
         """The all in 1 function benchmarker and results plotter.
 
@@ -277,7 +269,8 @@ class Bench(BenchPlotServer):
             pass_repeat (bool,optional) By default do not pass the kwarg 'repeat' to the benchmark function.  Set to true if
             you want the benchmark function to be passed the repeat number
             tag (str,optional): Use tags to group different benchmarks together.
-            run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run and plotted
+            run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run
+            plot_callbacks: (List | bool) A list of plot callbacks to call on the results. Pass false or an empty list to turn off plotting
         Raises:
             ValueError: If a result variable is not set
 
@@ -290,18 +283,26 @@ class Bench(BenchPlotServer):
                 logging.info(
                     "No input variables passed, using all param variables in bench class as inputs"
                 )
-                input_vars = self.worker_class_instance.get_inputs_only()
+                if self.input_vars is None:
+                    input_vars = self.worker_class_instance.get_inputs_only()
+                else:
+                    input_vars = self.input_vars
                 for i in input_vars:
                     logging.info(f"input var: {i.name}")
             if result_vars is None:
                 logging.info(
                     "No results variables passed, using all result variables in bench class:"
                 )
-                result_vars = self.worker_class_instance.get_results_only()
-                for r in result_vars:
-                    logging.info(f"result var: {r.name}")
+                if self.result_vars is None:
+                    result_vars = self.worker_class_instance.get_results_only()
+                else:
+                    result_vars = self.result_vars
+
             if const_vars is None:
-                const_vars = self.worker_class_instance.get_input_defaults()
+                if self.const_vars is None:
+                    const_vars = self.worker_class_instance.get_input_defaults()
+                else:
+                    const_vars = self.const_vars
         else:
             if input_vars is None:
                 input_vars = []
@@ -317,6 +318,9 @@ class Bench(BenchPlotServer):
         for i in range(len(result_vars)):
             result_vars[i] = self.convert_vars_to_params(result_vars[i], "result")
 
+        for r in result_vars:
+            logging.info(f"result var: {r.name}")
+
         if isinstance(const_vars, dict):
             const_vars = list(const_vars.items())
 
@@ -380,6 +384,14 @@ class Bench(BenchPlotServer):
                 "## Results Description\nPlease set post_description to explain these results"
             )
 
+        if plot_callbacks is None:
+            if self.plot_callbacks is not None and len(self.plot_callbacks) == 0:
+                plot_callbacks = [BenchResult.to_auto_plots]
+            else:
+                plot_callbacks = self.plot_callbacks
+        elif isinstance(plot_callbacks, bool):
+            plot_callbacks = [BenchResult.to_auto_plots] if plot_callbacks else []
+
         bench_cfg = BenchCfg(
             input_vars=input_vars,
             result_vars=result_vars_only,
@@ -391,7 +403,13 @@ class Bench(BenchPlotServer):
             title=title,
             pass_repeat=pass_repeat,
             tag=run_cfg.run_tag + tag,
+            plot_callbacks=plot_callbacks,
         )
+        return self.run_sweep(bench_cfg, run_cfg, time_src)
+
+    def run_sweep(
+        self, bench_cfg: BenchCfg, run_cfg: BenchRunCfg, time_src: datetime
+    ) -> BenchResult:
         print("tag", bench_cfg.tag)
 
         bench_cfg.param.update(run_cfg.param.values())
@@ -446,16 +464,12 @@ class Bench(BenchPlotServer):
 
         bench_res.post_setup()
 
-        if plot and bench_res.bench_cfg.auto_plot:
+        if bench_cfg.auto_plot:
             self.report.append_result(bench_res)
+
         self.results.append(bench_res)
         return bench_res
 
-    def get_name(self, var):
-        if isinstance(var, param.Parameter):
-            return var.name
-        return var
-
     def convert_vars_to_params(self, variable: param.Parameter, var_type: str):
         """check that a variable is a subclass of param
 
@@ -544,7 +558,7 @@ class Bench(BenchPlotServer):
             time_src (datetime | str): a representation of the sample time
 
         Returns:
-            _type_: _description_
+            tuple[BenchResult, List, List]: bench_result, function intputs, dimension names
         """
 
         if time_src is None:
@@ -552,13 +566,11 @@ class Bench(BenchPlotServer):
         bench_cfg.meta_vars = self.define_extra_vars(bench_cfg, bench_cfg.repeats, time_src)
 
         bench_cfg.all_vars = bench_cfg.input_vars + bench_cfg.meta_vars
-
         # bench_cfg.all_vars = bench_cfg.iv_time + bench_cfg.input_vars +[ bench_cfg.iv_repeat]
-
         # bench_cfg.all_vars = [ bench_cfg.iv_repeat] +bench_cfg.input_vars + bench_cfg.iv_time
 
         for i in bench_cfg.all_vars:
-            logging.info(i.sampling_str(bench_cfg.debug))
+            logging.info(i.sampling_str())
 
         dims_cfg = DimsCfg(bench_cfg)
         function_inputs = list(
@@ -574,7 +586,9 @@ class Bench(BenchPlotServer):
             if isinstance(rv, ResultReference):
                 result_data = np.full(dims_cfg.dims_size, -1, dtype=int)
                 data_vars[rv.name] = (dims_cfg.dims_name, result_data)
-            if isinstance(rv, (ResultVideo, ResultImage, ResultString, ResultContainer)):
+            if isinstance(
+                rv, (ResultPath, ResultVideo, ResultImage, ResultString, ResultContainer)
+            ):
                 result_data = np.full(dims_cfg.dims_size, "NAN", dtype=object)
                 data_vars[rv.name] = (dims_cfg.dims_name, result_data)
             elif type(rv) == ResultVec:
@@ -616,7 +630,6 @@ class Bench(BenchPlotServer):
             default=repeats,
             bounds=[1, repeats],
             samples=repeats,
-            samples_debug=2 if repeats > 2 else 1,
             units="repeats",
             doc="The number of times a sample was measured",
         )
@@ -712,7 +725,15 @@ class Bench(BenchPlotServer):
                     logging.info(f"{rv.name}: {result_value}")
 
                 if isinstance(
-                    rv, (ResultVar, ResultVideo, ResultImage, ResultString, ResultContainer)
+                    rv,
+                    (
+                        ResultVar,
+                        ResultVideo,
+                        ResultImage,
+                        ResultString,
+                        ResultContainer,
+                        ResultPath,
+                    ),
                 ):
                     set_xarray_multidim(bench_res.ds[rv.name], worker_job.index_tuple, result_value)
                 elif isinstance(rv, ResultReference):
@@ -803,3 +824,7 @@ class Bench(BenchPlotServer):
 
     def get_result(self, index: int = -1) -> BenchResult:
         return self.results[index]
+
+    def publish(self, remote_callback: Callable) -> str:
+        branch_name = f"{self.bench_name}_{self.run_cfg.run_tag}"
+        return self.report.publish(remote_callback, branch_name=branch_name)

bencher/class_enum.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+from strenum import StrEnum
+from typing import Any
+import importlib
+from abc import abstractmethod
+from dataclasses import dataclass
+from enum import auto
+
+
+class ClassEnum(StrEnum):
+    """A ClassEnum is a pattern to make it easier to create factory a factory method that converts from an enum to a corresponding class.  Subclasses should implement to_class(enum_instance:EnumType) which takes an enum returns the corresponding instance of that class."""
+
+    @classmethod
+    def to_class_generic(cls, module_import: str, class_name: str) -> Any:
+        """Create an instance of the class referred to by this enum
+
+        Returns:
+            Any: instance of the class
+        """
+
+        class_def = getattr(importlib.import_module(module_import), class_name)
+        return class_def()
+
+    @classmethod
+    @abstractmethod
+    def to_class(cls, enum_val: ClassEnum) -> Any:
+        """Subclasses should overrides this method to take  an enum returns the corresponding instance of that class."""
+        raise NotImplementedError()
+
+
+@dataclass
+class BaseClass:
+    baseclassname: str = "class0"
+
+
+@dataclass
+class Class1(BaseClass):
+    classname: str = "class1"
+
+
+@dataclass
+class Class2(BaseClass):
+    classname: str = "class2"
+
+
+class ExampleEnum(ClassEnum):
+    Class1 = auto()
+    Class2 = auto()
+
+    @classmethod
+    def to_class(cls, enum_val: ExampleEnum) -> BaseClass:
+        return cls.to_class_generic("bencher.class_enum", enum_val)

bencher/job.py

@@ -80,7 +80,8 @@ class FutureCache:
         size_limit: int = int(20e9),  # 20 GB
         use_cache=True,
     ):
-        self.executor = Executors.factory(executor)
+        self.executor_type = executor
+        self.executor = None
         if use_cache:
             self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
             logging.info(f"cache dir: {self.cache.directory}")
@@ -110,6 +111,9 @@ class FutureCache:
 
         self.worker_fn_call_count += 1
 
+        if self.executor_type is not Executors.SERIAL:
+            if self.executor is None:
+                self.executor = Executors.factory(self.executor_type)
         if self.executor is not None:
             self.overwrite_msg(job, " starting parallel job...")
             return JobFuture(
@@ -148,9 +152,7 @@ class FutureCache:
             self.cache.close()
         if self.executor:
             self.executor.shutdown()
-
-    # def __del__(self):
-    #     self.close()
+            self.executor = None
 
     def stats(self) -> str:
         logging.info(f"job calls: {self.worker_wrapper_call_count}")

bencher/optuna_conversions.py

@@ -30,7 +30,7 @@ def optuna_grid_search(bench_cfg: BenchCfg) -> optuna.Study:
     """
     search_space = {}
     for iv in bench_cfg.all_vars:
-        search_space[iv.name] = iv.values(bench_cfg.debug)
+        search_space[iv.name] = iv.values()
     directions = []
     for rv in bench_cfg.optuna_targets(True):
         directions.append(rv.direction)