holobench 1.3.6__py2.py3-none-any.whl

bencher/example/shelved/example_float2D_scatter.py

@@ -0,0 +1,109 @@
+# import random
+
+# import bencher as bch
+
+
+# class GaussianDist(bch.ParametrizedSweep):
+#     """A class to represent a gaussian distribution."""
+
+#     mean = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], doc="mean of the gaussian distribution", samples=3
+#     )
+#     sigma = bch.FloatSweep(
+#         default=1, bounds=[0, 1.0], doc="standard deviation of gaussian distribution", samples=4
+#     )
+
+
+# class Example2DGaussianResult(bch.ParametrizedSweep):
+#     """A class to represent the properties of a volume sample."""
+
+#     gauss_x = bch.ResultVar("m", doc="x value of the 2D gaussian")
+#     gauss_y = bch.ResultVar("m", doc="y value of the 2D gaussian")
+
+#     point2D = bch.ResultVec(2, "m", doc="2D vector of the point")
+
+
+# def bench_fn(dist: GaussianDist) -> Example2DGaussianResult:
+#     """This function samples a point from a gaussian distribution.
+
+#     Args:
+#         dist (GaussianDist): Sample point
+
+#     Returns:
+#         Example2DGaussianResult: Value at that point
+#     """
+#     output = Example2DGaussianResult()
+
+#     output.gauss_x = random.gauss(dist.mean, dist.sigma)
+#     output.gauss_y = random.gauss(dist.mean, dist.sigma)
+#     output.point2D = [output.gauss_x, output.gauss_y]
+
+#     return output
+
+
+# def example_floats2D_scatter(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+#     """Example of how to perform a 3D floating point parameter sweep
+
+#     Args:
+#         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+#     Returns:
+#         Bench: results of the parameter sweep
+#     """
+#     bench = bch.Bench(
+#         "Bencher_Example_Floats_Scatter", bench_fn, GaussianDist, plot_lib=bch.PlotLibrary.default()
+#     )
+
+#     bench.plot_sweep(
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter Example",
+#         run_cfg=run_cfg,
+#     )
+
+#     bench.plot_sweep(
+#         input_vars=[GaussianDist.param.mean],
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter With Changing Mean",
+#         run_cfg=run_cfg,
+#     )
+
+#     bench.plot_sweep(
+#         input_vars=[GaussianDist.param.sigma],
+#         result_vars=[
+#             Example2DGaussianResult.param.point2D,
+#             Example2DGaussianResult.param.gauss_x,
+#             Example2DGaussianResult.param.gauss_y,
+#         ],
+#         title="Float 2D Scatter With Changing Sigma",
+#         run_cfg=run_cfg,
+#     )
+
+#     # future work
+#     # bench.plot_sweep(
+#     #     input_vars=[GaussianDist.param.mean, GaussianDist.param.sigma],
+#     #     result_vars=[
+#     #         GaussianResult.param.point2D,
+#     #         GaussianResult.param.gauss_x,
+#     #         GaussianResult.param.gauss_y,
+#     #     ],
+#     #     title="Float 2D Scatter With Changing Sigma",
+#     #     run_cfg=run_cfg,
+#     # )
+
+#     return bench
+
+
+# if __name__ == "__main__":
+#     ex_run_cfg = bch.BenchRunCfg()
+#     ex_run_cfg.repeats = 50
+#     ex_run_cfg.over_time = True
+#     # ex_run_cfg.clear_history = True
+#     example_floats2D_scatter(ex_run_cfg).report.show()

bencher/example/shelved/example_float3D_cone.py

@@ -0,0 +1,96 @@
+# import numpy as np
+
+# import bencher as bch
+
+
+# class VolumeSample(bch.ParametrizedSweep):
+#     """A class to represent a 3D point in space."""
+
+#     pos_samples = 1
+
+#     x = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="x coordinate of the sample volume"
+#     )
+#     y = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="y coordinate of the sample volume"
+#     )
+#     z = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=pos_samples, doc="z coordinate of the sample volume"
+#     )
+
+#     vec_samples = 5
+
+#     vec_x = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="x coordinate of the sample volume"
+#     )
+#     vec_y = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="y coordinate of the sample volume"
+#     )
+#     vec_z = bch.FloatSweep(
+#         default=0, bounds=[-1.0, 1.0], samples=vec_samples, doc="z coordinate of the sample volume"
+#     )
+
+
+# class VolumeResult(bch.ParametrizedSweep):
+#     """A class to represent the properties of a volume sample."""
+
+#     vec_dir = bch.ResultVec(3, "vec", doc="A vector field with an interesting shape")
+
+
+# def bench_fn(point: VolumeSample, normalise=False) -> VolumeResult:
+#     """This function takes a 3D point as input and returns distance of that point to the origin.
+
+#     Args:
+#         point (VolumeSample): Sample point
+
+#     Returns:
+#         VolumeResult: Value at that point
+#     """
+#     output = VolumeResult()
+
+#     vec = np.array([point.vec_x, point.vec_y, point.vec_z])
+
+#     if normalise:
+#         norm = np.linalg.norm(vec)
+#         if norm > 0:
+#             vec /= norm
+
+#     output.vec_dir = list(vec)
+#     return output
+
+
+# def example_cone(run_cfg: bch.BenchRunCfg) -> bch.Bench:
+#     """Example of how to perform a 3D floating point parameter sweep
+
+#     Args:
+#         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+
+#     Returns:
+#         Bench: results of the parameter sweep
+#     """
+#     bench = bch.Bench("Bencher_Example_Cone", bench_fn, VolumeSample)
+
+#     bench.plot_sweep(
+#         input_vars=[
+#             VolumeSample.param.x,
+#             VolumeSample.param.y,
+#             VolumeSample.param.z,
+#             VolumeSample.param.vec_x,
+#             VolumeSample.param.vec_y,
+#             VolumeSample.param.vec_z,
+#         ],
+#         result_vars=[
+#             VolumeResult.param.vec_dir,
+#         ],
+#         title="Float 3D cone Example",
+#         description="""This example shows how to sample 3 floating point variables and plot a vector field representation of the results.  The benchmark function returns the distance to the origin""",
+#         run_cfg=run_cfg,
+#     )
+
+#     return bench
+
+
+# if __name__ == "__main__":
+#     ex_run_cfg = bch.BenchRunCfg()
+#     ex_run_cfg.use_cache = True
+#     example_cone(ex_run_cfg).report.show()

bencher/example/shelved/example_kwargs.py

@@ -0,0 +1,63 @@
+# import math
+
+# import bencher as bch
+
+
+# def bench_function(
+#     theta: float = 0,
+#     offset: float = 0,
+#     scale: float = 1.0,
+#     trig_func: str = "sin",
+#     **kwargs,  # pylint: disable=unused-argument
+# ) -> dict:
+#     """All the other examples use classes and parameters to define the inputs and outputs to the function. However it makes the code less flexible when integrating with other systems, so this example shows a more basic interface that accepts and returns dictionaries.  The classes still need to be defined however because that is how the sweep and plotting settings are calcuated"""
+#     output = {}
+
+#     if trig_func == "sin":
+#         output["voltage"] = offset + math.sin(theta) * scale
+#     elif trig_func == "cos":
+#         output["voltage"] = offset + math.cos(theta) * scale
+
+#     return output
+
+
+# class InputCfg(bch.ParametrizedSweep):
+#     """This class is used to define the default values and bounds of the variables to benchmark."""
+
+#     theta = bch.FloatSweep(
+#         default=0.0,
+#         bounds=[0.0, 6.0],
+#         doc="Input angle to the trig function",
+#         units="rad",
+#         samples=10,
+#     )
+
+#     offset = bch.FloatSweep(
+#         default=0.0,
+#         bounds=[0.0, 3.0],
+#         doc="Add an offset voltage to the result of the trig function",
+#         units="v",
+#         samples=5,
+#     )
+
+#     trig_func = bch.StringSweep(["sin", "cos"], doc="Select what trigonometric function use")
+
+
+# class OutputVoltage(bch.ParametrizedSweep):
+#     voltage = bch.ResultVar(units="v", doc="Output voltage")
+
+
+# if __name__ == "__main__":
+#     # pass the objective function you have defined to bencher.  The other examples pass the InputCfg type, but this benchmark function accepts a kwargs dictionary so you don't need to pass the inputCfg type.
+#     bench = bch.Bench("Bencher_Example_Categorical", bench_function)
+
+#     # Bencher needs to know the metadata of the variable in order to automatically sweep and plot it, so it is passed by using param's metadata syntax.  InputCfg.param.* is how to access the metadata defined in the class description.
+#     bench.plot_sweep(
+#         input_vars=[InputCfg.param.theta, InputCfg.param.offset],
+#         result_vars=[OutputVoltage.param.voltage],
+#         title="Example with kwarg inputs and dict output",
+#         description=bench_function.__doc__,
+#     )
+
+#     # launch web server and view
+#     bench.report.show()

bencher/job.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+from typing import Callable
+from sortedcontainers import SortedDict
+import logging
+from diskcache import Cache
+from concurrent.futures import Future, ProcessPoolExecutor
+from .utils import hash_sha1
+from strenum import StrEnum
+from enum import auto
+
+try:
+    from scoop import futures as scoop_future_executor
+except ImportError as e:
+    logging.warning(e.msg)
+    scoop_future_executor = None
+
+
+class Job:
+    def __init__(
+        self, job_id: str, function: Callable, job_args: dict, job_key=None, tag=""
+    ) -> None:
+        self.job_id = job_id
+        self.function = function
+        self.job_args = job_args
+        if job_key is None:
+            self.job_key = hash_sha1(tuple(SortedDict(self.job_args).items()))
+        else:
+            self.job_key = job_key
+        self.tag = tag
+
+
+# @dataclass
+class JobFuture:
+    def __init__(self, job: Job, res: dict = None, future: Future = None, cache=None) -> None:
+        self.job = job
+        self.res = res
+        self.future = future
+        # either a result or a future needs to be passed
+        assert self.res is not None or self.future is not None
+        self.cache = cache
+
+    def result(self):
+        if self.future is not None:
+            self.res = self.future.result()
+        if self.cache is not None and self.res is not None:
+            self.cache.set(self.job.job_key, self.res, tag=self.job.tag)
+        return self.res
+
+
+def run_job(job: Job) -> dict:
+    result = job.function(**job.job_args)
+    return result
+
+
+class Executors(StrEnum):
+    SERIAL = auto()  # slow but reliable
+    MULTIPROCESSING = auto()  # breaks for large number of futures
+    SCOOP = auto()  # requires running with python -m scoop your_file.py
+    # THREADS=auto() #not that useful as most bench code is cpu bound
+
+    @staticmethod
+    def factory(provider: Executors) -> Future():
+        providers = {
+            Executors.SERIAL: None,
+            Executors.MULTIPROCESSING: ProcessPoolExecutor(),
+            Executors.SCOOP: scoop_future_executor,
+        }
+        return providers[provider]
+
+
+class FutureCache:
+    """The aim of this class is to provide a unified interface for running jobs.  T"""
+
+    def __init__(
+        self,
+        executor=Executors.SERIAL,
+        overwrite: bool = True,
+        cache_name: str = "fcache",
+        tag_index: bool = True,
+        size_limit: int = int(20e9),  # 20 GB
+        use_cache=True,
+    ):
+        self.executor = Executors.factory(executor)
+        if use_cache:
+            self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
+            logging.info(f"cache dir: {self.cache.directory}")
+        else:
+            self.cache = None
+
+        self.overwrite = overwrite
+        self.call_count = 0
+        self.size_limit = size_limit
+
+        self.worker_wrapper_call_count = 0
+        self.worker_fn_call_count = 0
+        self.worker_cache_call_count = 0
+
+    def submit(self, job: Job) -> JobFuture:
+        self.worker_wrapper_call_count += 1
+
+        if self.cache is not None:
+            if not self.overwrite and job.job_key in self.cache:
+                logging.info(f"Found job: {job.job_id} in cache, loading...")
+                # logging.info(f"Found key: {job.job_key} in cache")
+                self.worker_cache_call_count += 1
+                return JobFuture(
+                    job=job,
+                    res=self.cache[job.job_key],
+                )
+
+        self.worker_fn_call_count += 1
+
+        if self.executor is not None:
+            self.overwrite_msg(job, " starting parallel job...")
+            return JobFuture(
+                job=job,
+                future=self.executor.submit(run_job, job),
+                cache=self.cache,
+            )
+        self.overwrite_msg(job, " starting serial job...")
+        return JobFuture(
+            job=job,
+            res=run_job(job),
+            cache=self.cache,
+        )
+
+    def overwrite_msg(self, job: Job, suffix: str) -> None:
+        msg = "OVERWRITING" if self.overwrite else "NOT in"
+        logging.info(f"{job.job_id} {msg} cache{suffix}")
+
+    def clear_call_counts(self) -> None:
+        """Clear the worker and cache call counts, to help debug and assert caching is happening properly"""
+        self.worker_wrapper_call_count = 0
+        self.worker_fn_call_count = 0
+        self.worker_cache_call_count = 0
+
+    def clear_cache(self) -> None:
+        if self.cache:
+            self.cache.clear()
+
+    def clear_tag(self, tag: str) -> None:
+        logging.info(f"clearing the sample cache for tag: {tag}")
+        removed_vals = self.cache.evict(tag)
+        logging.info(f"removed: {removed_vals} items from the cache")
+
+    def close(self) -> None:
+        if self.cache:
+            self.cache.close()
+        if self.executor:
+            self.executor.shutdown()
+
+    # def __del__(self):
+    #     self.close()
+
+    def stats(self) -> str:
+        logging.info(f"job calls: {self.worker_wrapper_call_count}")
+        logging.info(f"cache calls: {self.worker_cache_call_count}")
+        logging.info(f"worker calls: {self.worker_fn_call_count}")
+        if self.cache:
+            return f"cache size :{int(self.cache.volume() / 1000000)}MB / {int(self.size_limit/1000000)}MB"
+        return ""
+
+
+class JobFunctionCache(FutureCache):
+    def __init__(
+        self,
+        function: Callable,
+        overwrite=False,
+        executor: bool = False,
+        cache_name: str = "fcache",
+        tag_index: bool = True,
+        size_limit: int = int(100e8),
+    ):
+        super().__init__(
+            executor=executor,
+            cache_name=cache_name,
+            tag_index=tag_index,
+            size_limit=size_limit,
+            overwrite=overwrite,
+        )
+        self.function = function
+
+    def call(self, **kwargs) -> JobFuture:
+        return self.submit(Job(self.call_count, self.function, kwargs))

bencher/optuna_conversions.py

@@ -0,0 +1,168 @@
+from typing import List
+
+import optuna
+import panel as pn
+import param
+from optuna.visualization import (
+    plot_param_importances,
+    plot_pareto_front,
+    plot_optimization_history,
+)
+
+from bencher.bench_cfg import BenchCfg
+
+
+from bencher.variables.inputs import IntSweep, FloatSweep, StringSweep, EnumSweep, BoolSweep
+from bencher.variables.time import TimeSnapshot, TimeEvent
+
+from bencher.variables.parametrised_sweep import ParametrizedSweep
+
+
+# BENCH_CFG
+def optuna_grid_search(bench_cfg: BenchCfg) -> optuna.Study:
+    """use optuna to perform a grid search
+
+    Args:
+        bench_cfg (BenchCfg): setting for grid search
+
+    Returns:
+        optuna.Study: results of grid search
+    """
+    search_space = {}
+    for iv in bench_cfg.all_vars:
+        search_space[iv.name] = iv.values(bench_cfg.debug)
+    directions = []
+    for rv in bench_cfg.optuna_targets(True):
+        directions.append(rv.direction)
+
+    study = optuna.create_study(
+        sampler=optuna.samplers.GridSampler(search_space),
+        directions=directions,
+        study_name=bench_cfg.title,
+    )
+    return study
+
+
+# BENCH_CFG
+def param_importance(bench_cfg: BenchCfg, study: optuna.Study) -> pn.Row:
+    col_importance = pn.Column()
+    for tgt in bench_cfg.optuna_targets():
+        col_importance.append(
+            pn.Column(
+                pn.pane.Markdown(f"## Parameter importance for: {tgt}"),
+                plot_param_importances(study, target=lambda t: t.values[0], target_name=tgt),
+            )
+        )
+    return col_importance
+
+
+# BENCH_CFG
+def summarise_trial(trial: optuna.trial, bench_cfg: BenchCfg) -> List[str]:
+    """Given a trial produce a string summary of the best results
+
+    Args:
+        trial (optuna.trial): trial to summarise
+        bench_cfg (BenchCfg): info about the trial
+
+    Returns:
+        List[str]: Summary of trial
+    """
+    sep = "    "
+    output = []
+    output.append(f"Trial id:{trial.number}:")
+    output.append(f"{sep}Inputs:")
+    for k, v in trial.params.items():
+        output.append(f"{sep}{sep}{k}:{v}")
+    output.append(f"{sep}Results:")
+    for it, rv in enumerate(bench_cfg.optuna_targets()):
+        output.append(f"{sep}{sep}{rv}:{trial.values[it]}")
+    return output
+
+
+def sweep_var_to_optuna_dist(var: param.Parameter) -> optuna.distributions.BaseDistribution:
+    """Convert a sweep var to an optuna distribution
+
+    Args:
+        var (param.Parameter): A sweep var
+
+    Raises:
+        ValueError: Unsupported var type
+
+    Returns:
+        optuna.distributions.BaseDistribution: Optuna representation of a sweep var
+    """
+
+    iv_type = type(var)
+    if iv_type == IntSweep:
+        return optuna.distributions.IntDistribution(var.bounds[0], var.bounds[1])
+    if iv_type == FloatSweep:
+        return optuna.distributions.FloatDistribution(var.bounds[0], var.bounds[1])
+    if iv_type in (EnumSweep, StringSweep):
+        return optuna.distributions.CategoricalDistribution(var.objects)
+    if iv_type == BoolSweep:
+        return optuna.distributions.CategoricalDistribution([False, True])
+    if iv_type == TimeSnapshot:
+        # return optuna.distributions.IntDistribution(0, sys.maxsize)
+        return optuna.distributions.FloatDistribution(0, 1e20)
+        # return optuna.distributions.CategoricalDistribution([])
+    # elif iv_type == TimeEvent:
+    #     pass
+    # return optuna.distributions.CategoricalDistribution(["now"])
+
+    raise ValueError(f"This input type {iv_type} is not supported")
+
+
+def sweep_var_to_suggest(iv: ParametrizedSweep, trial: optuna.trial) -> object:
+    """Converts from a sweep var to an optuna
+
+    Args:
+        iv (ParametrizedSweep): A parametrized sweep input variable
+        trial (optuna.trial): Optuna trial used to define the sample
+
+    Raises:
+        ValueError: Unsupported var type
+
+    Returns:
+        Any: A sampled variable (can be any type)
+    """
+    iv_type = type(iv)
+
+    if iv_type == IntSweep:
+        return trial.suggest_int(iv.name, iv.bounds[0], iv.bounds[1])
+    if iv_type == FloatSweep:
+        return trial.suggest_float(iv.name, iv.bounds[0], iv.bounds[1])
+    if iv_type in (EnumSweep, StringSweep):
+        return trial.suggest_categorical(iv.name, iv.objects)
+    if iv_type in (TimeSnapshot, TimeEvent):
+        pass  # optuna does not like time
+    if iv_type == BoolSweep:
+        return trial.suggest_categorical(iv.name, [True, False])
+    raise ValueError(f"This input type {iv_type} is not supported")
+
+
+def cfg_from_optuna_trial(
+    trial: optuna.trial, bench_cfg: BenchCfg, cfg_type: ParametrizedSweep
+) -> ParametrizedSweep:
+    cfg = cfg_type()
+    for iv in bench_cfg.input_vars:
+        cfg.param.set_param(iv.name, sweep_var_to_suggest(iv, trial))
+    for mv in bench_cfg.meta_vars:
+        sweep_var_to_suggest(mv, trial)
+    return cfg
+
+
+def summarise_optuna_study(study: optuna.study.Study) -> pn.pane.panel:
+    """Summarise an optuna study in a panel format"""
+    row = pn.Column(name="Optimisation Results")
+    row.append(plot_optimization_history(study))
+    row.append(plot_param_importances(study))
+    try:
+        row.append(plot_pareto_front(study))
+    except Exception:
+        pass
+
+    row.append(
+        pn.pane.Markdown(f"```\nBest value: {study.best_value}\nParams: {study.best_params}```")
+    )
+
+    return row