holobench 1.40.1__py3-none-any.whl → 1.42.0__py3-none-any.whl

bencher/example/example_pareto.py

@@ -1,51 +1,127 @@
-# pylint: disable=duplicate-code
+"""
+Advanced Pareto Optimization Example with Randomness
+
+This example demonstrates multi-objective optimization using Optuna with the Bencher framework.
+It shows how to:
+1. Define a problem with multiple competing objectives
+2. Use Optuna's multi-objective optimization capabilities
+3. Visualize and analyze the Pareto front
+4. Compare different optimization approaches
+5. Demonstrate the effect of randomness on Pareto optimization
+"""
 
 import bencher as bch
+import numpy as np
+
+np.random.seed(0)
+
+
+class EngineeringDesignProblem(bch.ParametrizedSweep):
+    """
+    A simplified engineering design problem with two competing objectives.
+
+    This example simulates a common engineering trade-off problem:
+    - Performance vs Cost
+
+    This is a classic multi-objective optimization scenario.
+    The problem includes controlled randomness to simulate real-world variability
+    in manufacturing processes and materials.
+    """
+
+    # Input design parameters - reduced to just 2
+    material_quality = bch.FloatSweep(
+        default=0.5, bounds=[0.1, 1.0], doc="Quality of the material (dimensionless)", samples=20
+    )
+    thickness = bch.FloatSweep(
+        default=0.05, bounds=[0.01, 0.2], doc="Component thickness (m)", samples=20
+    )
+
+    # Result variables - reduced to just 2 objectives to be optimized
+    performance = bch.ResultVar("score", bch.OptDir.maximize, doc="Performance metric (maximize)")
+    cost = bch.ResultVar("$", bch.OptDir.minimize, doc="Manufacturing cost (minimize)")
+
+    def __call__(self, **kwargs) -> dict:
+        """
+        Calculate the multi-objective outcomes based on input parameters.
 
-# All the examples will be using the data structures and benchmark function defined in this file
-from bencher.example.benchmark_data import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
+        This simulates an engineering design problem where various objectives
+        compete with each other:
+        - Higher quality materials improve performance but increase cost
+        - Thicker material improves performance but increases cost
+
+        Includes inherent randomness to simulate:
+        - Manufacturing variability
+        - Material property variations
+        - Measurement uncertainty
+        """
+        self.update_params_from_kwargs(**kwargs)
+
+        # Base performance calculation
+        base_performance = self.material_quality * 80 + self.thickness * 50
+
+        # Add significant randomness (standard deviation = 10% of the base value)
+        # This will create noticeably different results on each run
+        performance_variability = 0.15 * base_performance
+        self.performance = base_performance + np.random.normal(0, performance_variability)
+
+        # Introduce a 30% chance of failure (e.g., due to manufacturing defects)
+        if np.random.rand() < 0.3:
+            self.performance = np.nan
+
+        # Base cost calculation
+        base_cost = self.material_quality * 100 + 10 / (self.thickness + 0.01)
+
+        # Add randomness to cost (standard deviation = 8% of the base value)
+        # Manufacturing costs can vary significantly in real-world scenarios
+        cost_variability = 0.12 * base_cost
+        self.cost = base_cost + np.random.normal(0, cost_variability)
+
+        return self.get_results_values_as_dict()
 
 
 def example_pareto(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None) -> bch.Bench:
-    """Example of how to calculate the pareto front of a parameter sweep
+    """
+    Advanced example of multi-objective Pareto optimization using Optuna.
+
+    This function demonstrates:
+    1. Grid search approach to visualize the entire parameter space
+    2. True multi-objective optimization with Optuna
+    3. Analysis of the Pareto front
+    4. Effect of randomness on the Pareto front
 
     Args:
-        run_cfg (BenchRunCfg): configuration of how to perform the param sweep
+        run_cfg (BenchRunCfg): Configuration for the benchmark run
+        report (BenchReport): Report object to store results
 
     Returns:
-        Bench: results of the parameter sweep
+        Bench: Benchmark object with results
     """
+    if run_cfg is None:
+        run_cfg = bch.BenchRunCfg()
+        run_cfg.repeats = 5  # Multiple repeats to demonstrate randomness effects
+        run_cfg.level = 4
+
+    # Set up Optuna for multi-objective optimization
     run_cfg.use_optuna = True
 
-    bench = bch.Bench(
-        "Multi-objective optimisation",
-        bench_function,
-        ExampleBenchCfgIn,
-        run_cfg=run_cfg,
-        report=report,
-    )
+    # Important: Set multiple repeats to demonstrate the effect of randomness
+    # The framework will automatically calculate and plot both individual runs and averages
+    run_cfg.repeats = 5
+
+    # Create problem definition and benchmark
+    bench = EngineeringDesignProblem().to_bench(run_cfg, report)
 
-    res = bench.plot_sweep(
-        title="Pareto Optimisation with Optuna",
-        description="This example shows how to plot the pareto front of the tradeoff between multiple criteria.  When multiple result variable are defined, and use_optuna=True a pareto plot and the relative importance of each input variable on the output criteria is plotted. A summary of the points on the pareto front is printed as well.  You can use the pareto plot to decide the how to trade off one objective for another.  Pareto plots are supported for 2D and 3D.  If you have more than 3 result variables the first 3 are selected for the pareto plot.  Plotting 4D surfaces is left as an exercise to the reader",
-        input_vars=[
-            ExampleBenchCfgIn.param.theta,
-            ExampleBenchCfgIn.param.offset,
-        ],
-        result_vars=[ExampleBenchCfgOut.param.out_sin, ExampleBenchCfgOut.param.out_cos],
-        const_vars=ExampleBenchCfgIn.get_input_defaults(
-            [ExampleBenchCfgIn.param.noisy.with_const(True)]
-        ),
-        post_description="""# Post Description 
-This is a slightly unusual way of doing pareto optimisation as we are not using a typical multi-objective optimisation algorithm [TODO, add example].  Instead we are performing a grid search and looking at the resulting pareto plot.  The reason for doing a grid search instead of standard pareto optimisation is that we can produce more isolated plots of how an input affects an output which can help understanding of the parameter space.  Future examples will show how to use grid search to bootstrap further optimisation with a multi objective optimiser""",
+    # Perform grid search on our two input variables
+    grid_result = bench.plot_sweep(
+        title="Parameter Space Exploration with Variability",
+        description="Exploring how material quality and thickness affect performance and cost with inherent randomness",
     )
 
-    bench.report.append(res.to_optuna_plots())
+    # Add Optuna-specific visualizations
+    bench.report.append(grid_result.to_optuna_plots())
+
     return bench
 
 
 if __name__ == "__main__":
-    run_cfg_ex = bch.BenchRunCfg()
-    run_cfg_ex.repeats = 2
-    run_cfg_ex.level = 2
-    example_pareto(run_cfg_ex).report.show()
+    example_pareto().report.show()

bencher/example/example_rerun2.py

@@ -23,7 +23,7 @@ else:
     # publish data to a github branch
     bch.publish_and_view_rrd(
         file_path,
-        remote="https://github.com/dyson-ai/bencher.git",
+        remote="https://github.com/blooop/bencher.git",
         branch_name="test_rrd",
         content_callback=bch.github_content,
     ).show()

bencher/example/example_simple_bool.py

@@ -16,14 +16,14 @@ def example_1D_bool(run_cfg: bch.BenchRunCfg) -> bch.Bench:
     )
 
     # here we sample the input variable theta and plot the value of output1. The (noisy) function is sampled 20 times so you can see the distribution
-    res = bench.plot_sweep(
+    bench.plot_sweep(
         title="Example 1D Bool",
         input_vars=[ExampleBenchCfgIn.param.noisy],
         result_vars=[ExampleBenchCfgOut.param.out_sin],
         description=example_1D_bool.__doc__,
         run_cfg=run_cfg,
     )
-    bench.report.append(res.to_bar())
+    bench.add(bch.BarResult)
 
     return bench
 

bencher/example/example_simple_float2d.py

@@ -21,7 +21,12 @@ def example_2D_float(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport =
     """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
 
     bench = SimpleFloat().to_bench(run_cfg, report)
-    bench.plot_sweep()
+    res = bench.plot_sweep()
+
+    bench.add(bch.CurveResult)
+    bench.report.append(res.to(bch.CurveResult))
+    bench.report.append(res.to(bch.HeatmapResult))
+    bench.add(bch.BarResult)
     return bench
 
 

bencher/example/example_video.py

@@ -1,9 +1,24 @@
 import bencher as bch
 import numpy as np
-import matplotlib.pyplot as plt
+from PIL import Image
+import colorcet as cc
+import numpy.typing as npt
+
+
+def apply_colormap(data: npt.NDArray) -> npt.NDArray:
+    """Apply a perceptually uniform colormap to the data"""
+    # Normalize data to [0, 1]
+    normalized = (data - data.min()) / (data.max() - data.min())
+    # Convert hex colors to RGB values using numpy's frombuffer
+    colors = np.array(
+        [np.frombuffer(bytes.fromhex(c.lstrip("#")), dtype=np.uint8) for c in cc.rainbow]
+    )
+    # Map normalized values to colormap indices
+    indices = (normalized * (len(colors) - 1)).astype(int)
+    # Create RGB array from the colormap
+    return colors[indices]
 
 
-# code from https://ipython-books.github.io/124-simulating-a-partial-differential-equation-reaction-diffusion-systems-and-turing-patterns/
 class TuringPattern(bch.ParametrizedSweep):
     alpha = bch.FloatSweep(default=2.8e-4, bounds=(2e-4, 5e-3))
     beta = bch.FloatSweep(default=5e-3, bounds=(1e-3, 9e-3))
@@ -17,6 +32,7 @@ class TuringPattern(bch.ParametrizedSweep):
     video = bch.ResultVideo()
     score = bch.ResultVar()
     img = bch.ResultImage()
+    img_extracted = bch.ResultImage()
 
     def laplacian(self, Z, dx):
         Ztop = Z[0:-2, 1:-1]
@@ -49,28 +65,28 @@ class TuringPattern(bch.ParametrizedSweep):
     def __call__(self, **kwargs):
         self.update_params_from_kwargs(**kwargs)
 
-        n = int(self.time / self.dt)  # number of iterations
-        dx = 2.0 / self.size  # space step
+        n = int(self.time / self.dt)
+        dx = 2.0 / self.size
 
         U = np.random.rand(self.size, self.size)
         V = np.random.rand(self.size, self.size)
 
-        fig, ax = plt.subplots(frameon=False, figsize=(2, 2))
-        fig.set_tight_layout(True)
-        ax.set_axis_off()
         vid_writer = bch.VideoWriter()
         for i in range(n):
             self.update(U, V, dx)
             if i % 500 == 0:
-                ax.imshow(U)
-                fig.canvas.draw()
-                rgb = np.array(fig.canvas.renderer.buffer_rgba())
-                vid_writer.append(rgb)
-
-        self.img = bch.add_image(rgb)
-
+                # Apply colormap to create RGB image
+                rgb = apply_colormap(U)
+                # Create PIL image with alpha channel
+                img = Image.fromarray(rgb, "RGB").convert("RGBA")
+                img = img.resize((200, 200), Image.Resampling.LANCZOS)
+                rgb_alpha = np.array(img)
+                vid_writer.append(rgb_alpha)
+
+        self.img = bch.add_image(rgb_alpha)
         self.video = vid_writer.write()
-
+        self.img_extracted = bch.video_writer.VideoWriter.extract_frame(self.video)
+        print("img path", self.img_extracted)
         self.score = self.alpha + self.beta
         return super().__call__()
 
@@ -103,14 +119,16 @@ def example_video_tap(
         )
     )
 
+    bench.worker_class_instance.get_results_only()
+
     return bench
 
 
 if __name__ == "__main__":
     run_cfg_ex = bch.BenchRunCfg()
     run_cfg_ex.level = 2
-    run_cfg_ex.cache_samples = True
-    run_cfg_ex.only_hash_tag = True
+    # run_cfg_ex.cache_samples = True
+    # run_cfg_ex.only_hash_tag = True
 
     # example_video(run_cfg_ex).report.show()
     example_video_tap(run_cfg_ex).report.show()

bencher/example/experimental/example_hvplot_explorer.py

@@ -1,6 +1,5 @@
 # THIS IS NOT A WORKING EXAMPLE YET
 # pylint: disable=duplicate-code
-import hvplot
 import bencher as bch
 from bencher import ExampleBenchCfgIn, ExampleBenchCfgOut, bench_function
 
@@ -8,7 +7,7 @@ bench = bch.Bench("Bencher_Example_Simple", bench_function, ExampleBenchCfgIn)
 
 
 if __name__ == "__main__":
-    bench_out = bench.plot_sweep(
+    res = bench.plot_sweep(
         input_vars=[ExampleBenchCfgIn.param.theta, ExampleBenchCfgIn.param.offset],
         result_vars=[ExampleBenchCfgOut.param.out_sin],
         title="Float 1D Example",
@@ -35,5 +34,5 @@ if __name__ == "__main__":
         ),
     )
 
-    hvexplorer = hvplot.explorer(bench_out.get_dataframe())
-    hvexplorer.show()
+    bench.add(bch.ExplorerResult)
+    bench.report.show()

bencher/example/inputs_0D/example_0_in_1_out.py

@@ -5,13 +5,13 @@ import random
 
 
 class SimpleFloat0D(bch.ParametrizedSweep):
-    """This class has 0 input dimensions and 1 output dimensions.  It samples from a gaussian distribution"""
+    """This class has 0 input dimensions and 1 output dimension. It samples from a Gaussian distribution"""
 
     # This defines a variable that we want to plot
     output = bch.ResultVar(units="ul", doc="a sample from a gaussian distribution")
 
     def __call__(self, **kwargs) -> dict:
-        """Generate a sample from a uniform distribution
+        """Generate a sample from a Gaussian distribution
 
         Returns:
             dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
@@ -24,31 +24,41 @@ class SimpleFloat0D(bch.ParametrizedSweep):
 def example_0_in_1_out(
     run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
 ) -> bch.Bench:
-    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+    """This example shows how to sample a 0-dimensional variable (no input parameters)
+    and plot the result of that sampling operation.
+
+    Args:
+        run_cfg: Configuration for the benchmark run
+        report: Report to append the results to
+
+    Returns:
+        bch.Bench: The benchmark object
+    """
 
     bench = SimpleFloat0D().to_bench(run_cfg, report)
     bench.plot_sweep()
 
-    bench.report.append(bench.get_result().to_table())
+    bench.add(bch.TableResult)
     return bench
 
 
 if __name__ == "__main__":
-    run_config = bch.BenchRunCfg(repeats=100)
-    reprt = bch.BenchReport()
-    # example_0_in_1_out(run_cfg, report).report.show()
+    example_0_in_1_out().report.show()
+    # run_config = bch.BenchRunCfg(repeats=100)
+    # report_obj = bch.BenchReport()
+    # example_0_in_1_out(run_config, report_obj).report.show()
 
     # run_cfg.over_time = True
     # run_cfg.cache_samples = True
     # for i in range(4):
-    #     example_0_in_1_out(run_cfg, report)
+    #     example_0_in_1_out(run_cfg, report_obj)
 
-    run_config.over_time = True
-    run_config.auto_plot = False
-    for _ in range(4):
-        example_0_in_1_out(run_config, reprt)
+    # run_config.over_time = True
+    # run_config.auto_plot = False
+    # for _ in range(4):
+    #     example_0_in_1_out(run_config, report_obj)
 
-    run_config.auto_plot = True
-    example_0_in_1_out(run_config, reprt)
+    # run_config.auto_plot = True
+    # example_0_in_1_out(run_config, report_obj)
 
-    reprt.show()
+    # report_obj.show()

bencher/example/inputs_0D/example_0_in_2_out.py

@@ -5,14 +5,14 @@ import random
 
 
 class SimpleFloat0D(bch.ParametrizedSweep):
-    """This class has 0 input dimensions and 1 output dimensions.  It samples from a gaussian distribution"""
+    """This class has 0 input dimensions and 2 output dimensions. It samples from Gaussian distributions"""
 
     # This defines a variable that we want to plot
     output1 = bch.ResultVar(units="ul", doc="a sample from a gaussian distribution")
     output2 = bch.ResultVar(units="ul", doc="a sample from a gaussian distribution")
 
     def __call__(self, **kwargs) -> dict:
-        """Generate a sample from a uniform distribution
+        """Generate samples from Gaussian distributions
 
         Returns:
             dict: a dictionary with all the result variables in the ParametrisedSweep class as named key value pairs.
@@ -26,7 +26,16 @@ class SimpleFloat0D(bch.ParametrizedSweep):
 def example_0_in_2_out(
     run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
 ) -> bch.Bench:
-    """This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
+    """This example shows how to sample a 0-dimensional variable (no input parameters)
+    that produces two output values and plot the results.
+
+    Args:
+        run_cfg: Configuration for the benchmark run
+        report: Report to append the results to
+
+    Returns:
+        bch.Bench: The benchmark object
+    """
 
     bench = SimpleFloat0D().to_bench(run_cfg, report)
     bench.plot_sweep()

bencher/example/inputs_0_float/example_0_cat_in_2_out.py

@@ -0,0 +1,88 @@
+"""This file demonstrates benchmarking with 0 categorical inputs and 2 output variables.
+
+It benchmarks a single Python operation configuration to showcase output variations
+using simulated performance data to illustrate how benchmarking works with fixed inputs.
+"""
+
+import random
+import bencher as bch
+
+random.seed(0)
+
+
+class PythonOperations0CatBenchmark(bch.ParametrizedSweep):
+    """Example class for benchmarking with no categorical variables.
+
+    This class demonstrates how to structure a benchmark with fixed inputs
+    and multiple output metrics. It uses simulated performance data that follows realistic
+    patterns while being deterministic and reproducible.
+    """
+
+    # All inputs are fixed (list data structure, read operation, medium size)
+
+    execution_time = bch.ResultVar(units="ms", doc="Execution time in milliseconds")
+    memory_peak = bch.ResultVar(units="KB", doc="Peak memory usage in kilobytes")
+
+    def __call__(self, **kwargs) -> dict:
+        """Execute the benchmark for the given set of parameters.
+
+        Args:
+            **kwargs: Parameters to update before executing
+
+        Returns:
+            dict: Dictionary containing the benchmark results
+        """
+        self.update_params_from_kwargs(**kwargs)
+
+        # Base values for list read operation on medium data
+        base_time = 28.0  # ms
+        base_memory = 960.0  # KB
+
+        # Add significant variance to show distribution of results
+        # even with fixed inputs
+        self.execution_time = base_time * random.gauss(0.75, 1.25)
+        self.memory_peak = base_memory * random.gauss(0.80, 1.20)
+
+        return super().__call__(**kwargs)
+
+
+def example_0_cat_in_2_out(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    """This example demonstrates benchmarking with no categorical variables and multiple output metrics.
+
+    It creates a synthetic benchmark that simulates performance variations when repeatedly running
+    the same operation (list read operation on medium data). This example shows that even with
+    fixed inputs, repeated benchmark runs produce variations in performance metrics.
+
+    Args:
+        run_cfg: Configuration for the benchmark run
+        report: Report to append the results to
+
+    Returns:
+        bch.Bench: The benchmark object
+    """
+
+    if run_cfg is None:
+        run_cfg = bch.BenchRunCfg()
+    run_cfg.repeats = 100  # More repeats to show distribution
+    bench = PythonOperations0CatBenchmark().to_bench(run_cfg, report)
+    bench.plot_sweep(
+        title="Python List Read Operation Performance (Fixed Inputs)",
+        description="Distribution of execution time and peak memory usage across multiple runs",
+        post_description="""
+        This benchmark illustrates how performance metrics vary even with fixed inputs.
+        
+        Key observations:
+        - Each run uses the same configuration: list data structure, read operation, medium data size
+        - Despite identical inputs, performance metrics show natural variations
+        - This variance simulates real-world system fluctuations that occur in benchmarking
+        - With no categorical variables, the benchmark helps establish baseline performance
+          distribution for a single configuration
+        """,
+    )
+    return bench
+
+
+if __name__ == "__main__":
+    example_0_cat_in_2_out().report.show()

bencher/example/inputs_0_float/example_1_cat_in_2_out.py

@@ -0,0 +1,98 @@
+"""This file demonstrates benchmarking with 1 categorical input and 2 output variables.
+
+It benchmarks different Python data structures to compare their performance characteristics
+using simulated performance data to illustrate how benchmarking works.
+"""
+
+import random
+import bencher as bch
+
+random.seed(0)
+
+
+class PythonOperations1CatBenchmark(bch.ParametrizedSweep):
+    """Example class for benchmarking different Python data structures using 1 categorical variable.
+
+    This class demonstrates how to structure a benchmark with a single input parameter
+    and multiple output metrics. It uses simulated performance data that follows realistic
+    patterns while being deterministic and reproducible.
+    """
+
+    data_structure = bch.StringSweep(["list", "dict"], doc="Type of data structure to operate on")
+
+    # Using fixed read operation and medium size data
+
+    execution_time = bch.ResultVar(units="ms", doc="Execution time in milliseconds")
+    memory_peak = bch.ResultVar(units="KB", doc="Peak memory usage in kilobytes")
+
+    def __call__(self, **kwargs) -> dict:
+        """Execute the benchmark for the given set of parameters.
+
+        Args:
+            **kwargs: Parameters to update before executing
+
+        Returns:
+            dict: Dictionary containing the benchmark results
+        """
+        self.update_params_from_kwargs(**kwargs)
+
+        # Use deterministic fake data based on parameters
+        # Base values (for read operation on medium data)
+        base_time = 35.0  # ms (medium size, read operation base)
+        base_memory = 800.0  # KB (medium size, read operation base)
+
+        # Adjust for data structure (lists are generally faster but use more memory)
+        if self.data_structure == "list":
+            time_factor = 0.8
+            memory_factor = 1.2
+        else:  # dict
+            time_factor = 1.2
+            memory_factor = 0.9
+
+        # Calculate final metrics with significant variance to show differences
+        self.execution_time = base_time * time_factor * random.gauss(0.80, 1.20)
+        self.memory_peak = base_memory * memory_factor * random.gauss(0.85, 1.15)
+
+        return super().__call__(**kwargs)
+
+
+def example_1_cat_in_2_out(
+    run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
+) -> bch.Bench:
+    """This example demonstrates benchmarking with 1 categorical variable and multiple output metrics.
+
+    It creates a synthetic benchmark that simulates performance characteristics of different
+    Python data structures (list vs dict) using a fixed read operation and medium data size.
+    The benchmark produces realistic patterns of execution time and memory usage without
+    actually executing real operations, making it ideal for learning and demonstration.
+
+    Args:
+        run_cfg: Configuration for the benchmark run
+        report: Report to append the results to
+
+    Returns:
+        bch.Bench: The benchmark object
+    """
+
+    if run_cfg is None:
+        run_cfg = bch.BenchRunCfg()
+    run_cfg.repeats = 5  # Fewer repeats for a quicker benchmark
+    bench = PythonOperations1CatBenchmark().to_bench(run_cfg, report)
+    bench.plot_sweep(
+        title="Python Data Structure Performance Benchmark (1 Variable)",
+        description="Comparing execution time and peak memory usage between lists and dictionaries",
+        post_description="""
+        This benchmark illustrates how different data structures affect performance.
+        
+        Key observations:
+        - Lists generally process faster than dictionaries for read operations
+        - However, lists consume more memory than dictionaries
+        - All tests were performed with read operations on medium-sized datasets
+        - Note that variance in the results simulates real-world measurement fluctuations
+        """,
+    )
+    return bench
+
+
+if __name__ == "__main__":
+    example_1_cat_in_2_out().report.show()