holobench 1.41.0__py3-none-any.whl → 1.43.0__py3-none-any.whl

bencher/example/meta/generate_examples.py

@@ -2,13 +2,15 @@ import nbformat as nbf
 from pathlib import Path
 
 
-def convert_example_to_jupyter_notebook(filename: str, output_path: str):
-    # print
+def convert_example_to_jupyter_notebook(
+    filename: str, output_path: str, repeats: int = None
+) -> None:
     source_path = Path(filename)
 
     nb = nbf.v4.new_notebook()
     title = source_path.stem
-    function_name = f"{source_path.stem}()"
+    repeat_exr = f"bch.BenchRunCfg(repeats={repeats})" if repeats else ""
+    function_name = f"{source_path.stem}({repeat_exr})"
     text = f"""# {title}"""
 
     code = "%%capture\n"
@@ -34,19 +36,128 @@ bench.get_result().to_auto_plots()
     ]
     output_path = Path(f"docs/reference/{output_path}/ex_{title}.ipynb")
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(nbf.writes(nb), encoding="utf-8")
+    # Add a newline character at the end to ensure proper end-of-file
+    notebook_content = nbf.writes(nb) + "\n"
+    output_path.write_text(notebook_content, encoding="utf-8")
 
 
 if __name__ == "__main__":
+    # Examples with different numbers of categorical variables in increasing order
     convert_example_to_jupyter_notebook(
-        "/workspaces/bencher/bencher/example/inputs_1D/example_1_in_1_out.py", "1D"
+        "/workspaces/bencher/bencher/example/inputs_0_float/example_0_cat_in_2_out.py",
+        "inputs_0_float",
+        repeats=100,
     )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_0_float/example_1_cat_in_2_out.py",
+        "inputs_0_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_0_float/example_2_cat_in_2_out.py",
+        "inputs_0_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_0_float/example_3_cat_in_2_out.py",
+        "inputs_0_float",
+    )
+
+    # Examples with 1 float input plus varying categorical inputs
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1_float/example_1_float_0_cat_in_2_out.py",
+        "inputs_1_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1_float/example_1_float_1_cat_in_2_out.py",
+        "inputs_1_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1_float/example_1_float_2_cat_in_2_out.py",
+        "inputs_1_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1_float/example_1_float_3_cat_in_2_out.py",
+        "inputs_1_float",
+    )
+
+    # Example with 2 float inputs plus categorical inputs
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_2_float/example_2_float_3_cat_in_2_out.py",
+        "inputs_2_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_2_float/example_2_float_2_cat_in_2_out.py",
+        "inputs_2_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_2_float/example_2_float_1_cat_in_2_out.py",
+        "inputs_2_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_2_float/example_2_float_0_cat_in_2_out.py",
+        "inputs_2_float",
+    )
+
+    # Examples with 3 float inputs plus categorical inputs
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_3_float/example_3_float_3_cat_in_2_out.py",
+        "inputs_3_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_3_float/example_3_float_2_cat_in_2_out.py",
+        "inputs_3_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_3_float/example_3_float_1_cat_in_2_out.py",
+        "inputs_3_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_3_float/example_3_float_0_cat_in_2_out.py",
+        "inputs_3_float",
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_0D/example_0_in_1_out.py", "0D", repeats=100
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_0D/example_0_in_2_out.py", "0D", repeats=100
+    )
+
+    # Other 1D examples
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1D/example_1_int_in_1_out.py", "1D"
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1D/example_1_int_in_2_out.py", "1D"
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1D/example_1_int_in_2_out_repeats.py", "1D"
+    )
+
+    convert_example_to_jupyter_notebook(
+        "/workspaces/bencher/bencher/example/inputs_1D/example_1_cat_in_2_out_repeats.py", "1D"
+    )
+
     convert_example_to_jupyter_notebook(
-        "/workspaces/bencher/bencher/example/inputs_1D/example_1_in_2_out.py", "1D"
+        "/workspaces/bencher/bencher/example/inputs_2D/example_2_cat_in_4_out_repeats.py", "1D"
     )
 
     convert_example_to_jupyter_notebook(
-        "/workspaces/bencher/bencher/example/inputs_1D/example_1_in_2_out_repeats.py", "1D"
+        "/workspaces/bencher/bencher/example/example_levels.py", "Levels"
     )
 
     # todo, enable

bencher/example/meta/generate_meta.py

@@ -9,22 +9,68 @@ from bencher.example.meta.example_meta import BenchableObject
 class BenchMetaGen(bch.ParametrizedSweep):
     """This class uses bencher to display the multidimensional types bencher can represent"""
 
-    float_vars = bch.IntSweep(
-        default=0, bounds=(0, 4), doc="The number of floating point variables that are swept"
+    # Benchable object to use
+    benchable_obj = None  # Will store the benchable object instance
+
+    # Variables for controlling the sweep
+    float_vars_count = bch.IntSweep(
+        default=0, bounds=(0, 3), doc="The number of floating point variables that are swept"
     )
-    categorical_vars = bch.IntSweep(
-        default=0, bounds=(0, 3), doc="The number of categorical variables that are swept"
+    categorical_vars_count = bch.IntSweep(
+        default=0, bounds=(0, 2), doc="The number of categorical variables that are swept"
     )
-    sample_with_repeats = bch.IntSweep(default=1, bounds=(1, 100))
 
-    sample_over_time = bch.BoolSweep(default=False)
+    # Lists to store the actual variable names
+    float_var_names = []  # Will store float variable names
+    categorical_var_names = []  # Will store categorical variable names
+    result_var_names = []  # Will store result variable names
 
+    # Configuration options
+    sample_with_repeats = bch.IntSweep(default=1, bounds=(1, 100))
+    sample_over_time = bch.BoolSweep(default=False)
     level = bch.IntSweep(default=2, units="level", bounds=(2, 5))
-
     run_bench = False
 
     plots = bch.ResultReference(units="int")
 
+    def __init__(
+        self, benchable_obj=None, float_vars=None, categorical_vars=None, result_vars=None, **params
+    ):
+        """Initialize with a benchable object and variable specifications
+
+        Args:
+            benchable_obj: The benchable object to use
+            float_vars: List of float variable names to sweep
+            categorical_vars: List of categorical variable names to sweep
+            result_vars: List of result variable names to plot
+            **params: Additional parameters
+        """
+        super().__init__(**params)
+
+        # Set the benchable object
+        self.benchable_obj = benchable_obj if benchable_obj is not None else BenchableObject()
+
+        # Dynamically discover parameters from the benchable object
+        float_param_names = []
+        categorical_param_names = []
+        result_param_names = []
+
+        # Check all parameters of the benchable object
+        for name, param in self.benchable_obj.__class__.param.objects().items():
+            if hasattr(param, "bounds") and isinstance(param, bch.FloatSweep):
+                float_param_names.append(name)
+            elif isinstance(param, (bch.BoolSweep, bch.EnumSweep, bch.StringSweep)):
+                categorical_param_names.append(name)
+            elif isinstance(param, bch.ResultVar):
+                result_param_names.append(name)
+
+        # Use provided parameter lists or discovered ones
+        self.float_var_names = float_vars if float_vars is not None else float_param_names
+        self.categorical_var_names = (
+            categorical_vars if categorical_vars is not None else categorical_param_names
+        )
+        self.result_var_names = result_vars if result_vars is not None else result_param_names
+
     def __call__(self, **kwargs: Any) -> Any:
         self.update_params_from_kwargs(**kwargs)
 
@@ -34,50 +80,52 @@ class BenchMetaGen(bch.ParametrizedSweep):
         run_cfg.over_time = self.sample_over_time
         run_cfg.plot_size = 500
 
-        # bench = bch.Bench("benchable", BenchableObject(), run_cfg=run_cfg)
+        # Limit the variables to the specified counts
+        float_vars = []
+        # Apply the max_level=3 limit to the 3rd and subsequent float variables
+        for i, var_name in enumerate(self.float_var_names[: self.float_vars_count]):
+            if i >= 2:  # Third variable (index 2) and beyond
+                float_vars.append(bch.p(var_name, max_level=3))
+            else:
+                float_vars.append(var_name)
 
-        inputs_vars_float = [
-            "float1",
-            "float2",
-            bch.p("float3", max_level=3),
-            "sigma",
-        ]
+        categorical_vars = self.categorical_var_names[: self.categorical_vars_count]
+        input_vars = float_vars + categorical_vars
 
-        inputs_vars_cat = [
-            "noisy",
-            "noise_distribution",
-            "negate_output",
-        ]
-
-        input_vars = inputs_vars_float[: self.float_vars] + inputs_vars_cat[: self.categorical_vars]
-
-        if self.run_bench:
-            bench = BenchableObject().to_bench(run_cfg)
+        if self.run_bench and input_vars and self.result_var_names:
+            bench = self.benchable_obj.to_bench(run_cfg)
             res = bench.plot_sweep(
                 "test",
                 input_vars=input_vars,
-                result_vars=["distance", "sample_noise"],
+                result_vars=self.result_var_names,
                 plot_callbacks=False,
             )
             self.plots = bch.ResultReference()
             self.plots.obj = res.to_auto()
 
-        title = f"{self.float_vars}_float_{self.categorical_vars}_cat"
+        title = f"{self.float_vars_count}_float_{self.categorical_vars_count}_cat"
 
         nb = nbf.v4.new_notebook()
         text = f"""# {title}"""
 
+        # Customize notebook generation to use the actual benchmark object and variables
+        module_import = "import bencher as bch\n"
+        if self.benchable_obj.__class__.__module__ != "__main__":
+            benchmark_import = f"from {self.benchable_obj.__class__.__module__} import {self.benchable_obj.__class__.__name__}\n"
+        else:
+            # If it's from main, use BenchableObject as fallback
+            benchmark_import = "from bencher.example.meta.example_meta import BenchableObject\n"
+
         code_gen = f"""
 %%capture
-import bencher as bch
-from bencher.example.meta.example_meta import BenchableObject
-
+{module_import}
+{benchmark_import}
 run_cfg = bch.BenchRunCfg()
 run_cfg.repeats = {self.sample_with_repeats}
 run_cfg.level = 4 
-bench = BenchableObject().to_bench(run_cfg)
+bench = {self.benchable_obj.__class__.__name__}().to_bench(run_cfg)
 res=bench.plot_sweep(input_vars={input_vars},
-                    result_vars=["distance","sample_noise"])
+                    result_vars={self.result_var_names})
 """
         code_results = """
 from bokeh.io import output_notebook
@@ -107,14 +155,14 @@ def example_meta(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
         description="""## All Combinations of Variable Sweeps and Resulting Plots
 This uses bencher to display all the combinations of plots bencher is able to produce""",
         input_vars=[
-            bch.p("float_vars", [0, 1]),
-            "categorical_vars",
+            bch.p("float_vars_count", [0, 1]),
+            "categorical_vars_count",
             bch.p("sample_with_repeats", [1, 20]),
         ],
         const_vars=[
-            # BenchMeta.param.float_vars.with_const(1),
+            # BenchMeta.param.float_vars_count.with_const(1),
             # BenchMeta.param.sample_with_repeats.with_const(2),
-            # BenchMeta.param.categorical_vars.with_const(2),
+            # BenchMeta.param.categorical_vars_count.with_const(2),
             # BenchMeta.param.sample_over_time.with_const(True),
         ],
     )
@@ -123,8 +171,8 @@ This uses bencher to display all the combinations of plots bencher is able to pr
         description="""## All Combinations of Variable Sweeps and Resulting Plots
 This uses bencher to display all the combinations of plots bencher is able to produce""",
         input_vars=[
-            bch.p("float_vars", [2, 3]),
-            "categorical_vars",
+            bch.p("float_vars_count", [2, 3]),
+            "categorical_vars_count",
         ],
     )
 
@@ -133,13 +181,13 @@ This uses bencher to display all the combinations of plots bencher is able to pr
     #         description="""## All Combinations of Variable Sweeps and Resulting Plots
     # This uses bencher to display all the combinations of plots bencher is able to produce""",
     #         input_vars=[
-    #             bch.p("float_vars", [2, 3]),
-    #             "categorical_vars",
+    #             bch.p("float_vars_count", [2, 3]),
+    #             "categorical_vars_count",
     #         ],
     #         const_vars=[
     #             dict(level=3)
     #             # BenchMeta.param.sample_with_repeats.with_const(2),
-    #             # BenchMeta.param.categorical_vars.with_const(2),
+    #             # BenchMeta.param.categorical_vars_count.with_const(2),
     #             # BenchMeta.param.sample_over_time.with_const(True),
     #         ],
     #     )

bencher/job.py

@@ -14,9 +14,32 @@ except ImportError as e:
 
 
 class Job:
+    """Represents a benchmarking job to be executed or retrieved from cache.
+
+    A Job encapsulates a function, its arguments, and metadata for caching
+    and tracking purposes.
+
+    Attributes:
+        job_id (str): A unique identifier for the job, used for logging
+        function (Callable): The function to be executed
+        job_args (dict): Arguments to pass to the function
+        job_key (str): A hash key for caching, derived from job_args if not provided
+        tag (str): Optional tag for grouping related jobs
+    """
+
     def __init__(
-        self, job_id: str, function: Callable, job_args: dict, job_key=None, tag=""
+        self, job_id: str, function: Callable, job_args: dict, job_key: str = None, tag: str = ""
     ) -> None:
+        """Initialize a Job with function and arguments.
+
+        Args:
+            job_id (str): A unique identifier for this job
+            function (Callable): The function to execute
+            job_args (dict): Arguments to pass to the function
+            job_key (str, optional): Cache key for this job. If None, will be generated
+                from job_args. Defaults to None.
+            tag (str, optional): Tag for grouping related jobs. Defaults to "".
+        """
         self.job_id = job_id
         self.function = function
         self.job_args = job_args
@@ -27,9 +50,32 @@ class Job:
         self.tag = tag
 
 
-# @dataclass
 class JobFuture:
+    """A wrapper for a job result or future that handles caching.
+
+    This class provides a unified interface for handling both immediate results
+    and futures (for asynchronous execution). It also handles caching results
+    when they become available.
+
+    Attributes:
+        job (Job): The job this future corresponds to
+        res (dict): The result, if available immediately
+        future (Future): The future representing the pending job, if executed asynchronously
+        cache: The cache to store results in when they become available
+    """
+
     def __init__(self, job: Job, res: dict = None, future: Future = None, cache=None) -> None:
+        """Initialize a JobFuture with either an immediate result or a future.
+
+        Args:
+            job (Job): The job this future corresponds to
+            res (dict, optional): The immediate result, if available. Defaults to None.
+            future (Future, optional): The future representing the pending result. Defaults to None.
+            cache (Cache, optional): The cache to store results in. Defaults to None.
+
+        Raises:
+            AssertionError: If neither res nor future is provided
+        """
         self.job = job
         self.res = res
         self.future = future
@@ -40,7 +86,16 @@ class JobFuture:
 
         self.cache = cache
 
-    def result(self):
+    def result(self) -> dict:
+        """Get the job result, waiting for completion if necessary.
+
+        If the result is not immediately available (i.e., it's a future),
+        this method will wait for the future to complete. Once the result
+        is available, it will be cached if a cache is provided.
+
+        Returns:
+            dict: The job result
+        """
         if self.future is not None:
             self.res = self.future.result()
         if self.cache is not None and self.res is not None:
@@ -49,18 +104,42 @@ class JobFuture:
 
 
 def run_job(job: Job) -> dict:
+    """Execute a job by calling its function with the provided arguments.
+
+    This is a helper function used primarily by executors to run jobs.
+
+    Args:
+        job (Job): The job to execute
+
+    Returns:
+        dict: The result of the job execution
+    """
     result = job.function(**job.job_args)
     return result
 
 
 class Executors(StrEnum):
+    """Enumeration of available execution strategies for benchmark jobs.
+
+    This enum defines the execution modes for running benchmark jobs
+    and provides a factory method to create appropriate executors.
+    """
+
     SERIAL = auto()  # slow but reliable
     MULTIPROCESSING = auto()  # breaks for large number of futures
     SCOOP = auto()  # requires running with python -m scoop your_file.py
     # THREADS=auto() #not that useful as most bench code is cpu bound
 
     @staticmethod
-    def factory(provider: Executors) -> Future:
+    def factory(provider: "Executors") -> Future | None:
+        """Create an executor instance based on the specified execution strategy.
+
+        Args:
+            provider (Executors): The type of executor to create
+
+        Returns:
+            Future | None: The executor instance, or None for serial execution
+        """
         providers = {
             Executors.SERIAL: None,
             Executors.MULTIPROCESSING: ProcessPoolExecutor(),
@@ -70,7 +149,23 @@ class Executors(StrEnum):
 
 
 class FutureCache:
-    """The aim of this class is to provide a unified interface for running jobs.  T"""
+    """A cache system for benchmark job results with executor support.
+
+    This class provides a unified interface for running benchmark jobs either serially
+    or in parallel, with optional caching of results. It manages the execution strategy,
+    caching policy, and tracks statistics about job execution.
+
+    Attributes:
+        executor_type (Executors): The execution strategy to use
+        executor: The executor instance, created on demand
+        cache (Cache): Cache for storing job results
+        overwrite (bool): Whether to overwrite existing cached results
+        call_count (int): Counter for job calls
+        size_limit (int): Maximum size of the cache in bytes
+        worker_wrapper_call_count (int): Number of job submissions
+        worker_fn_call_count (int): Number of actual function executions
+        worker_cache_call_count (int): Number of cache hits
+    """
 
     def __init__(
         self,
@@ -79,8 +174,18 @@ class FutureCache:
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(20e9),  # 20 GB
-        cache_results=True,
+        cache_results: bool = True,
     ):
+        """Initialize a FutureCache with optional caching and execution settings.
+
+        Args:
+            executor (Executors, optional): The execution strategy to use. Defaults to Executors.SERIAL.
+            overwrite (bool, optional): Whether to overwrite existing cached results. Defaults to True.
+            cache_name (str, optional): Base name for the cache directory. Defaults to "fcache".
+            tag_index (bool, optional): Whether to enable tag-based indexing in the cache. Defaults to True.
+            size_limit (int, optional): Maximum size of the cache in bytes. Defaults to 20GB.
+            cache_results (bool, optional): Whether to cache results at all. Defaults to True.
+        """
         self.executor_type = executor
         self.executor = None
         if cache_results:
@@ -98,6 +203,18 @@ class FutureCache:
         self.worker_cache_call_count = 0
 
     def submit(self, job: Job) -> JobFuture:
+        """Submit a job for execution, with caching if enabled.
+
+        This method first checks if the job result is already in the cache (if caching is enabled
+        and overwrite is False). If not found in the cache, it executes the job either serially
+        or using the configured executor.
+
+        Args:
+            job (Job): The job to submit
+
+        Returns:
+            JobFuture: A future representing the job execution
+        """
         self.worker_wrapper_call_count += 1
 
         if self.cache is not None:
@@ -130,25 +247,38 @@ class FutureCache:
         )
 
     def overwrite_msg(self, job: Job, suffix: str) -> None:
+        """Log a message about overwriting or using cache.
+
+        Args:
+            job (Job): The job being executed
+            suffix (str): Additional text to add to the log message
+        """
         msg = "OVERWRITING" if self.overwrite else "NOT in"
         logging.info(f"{job.job_id} {msg} cache{suffix}")
 
     def clear_call_counts(self) -> None:
-        """Clear the worker and cache call counts, to help debug and assert caching is happening properly"""
+        """Clear the worker and cache call counts, to help debug and assert caching is happening properly."""
         self.worker_wrapper_call_count = 0
         self.worker_fn_call_count = 0
         self.worker_cache_call_count = 0
 
     def clear_cache(self) -> None:
+        """Clear all entries from the cache."""
         if self.cache:
             self.cache.clear()
 
     def clear_tag(self, tag: str) -> None:
+        """Remove all cache entries with the specified tag.
+
+        Args:
+            tag (str): The tag identifying entries to remove from the cache
+        """
         logging.info(f"clearing the sample cache for tag: {tag}")
         removed_vals = self.cache.evict(tag)
         logging.info(f"removed: {removed_vals} items from the cache")
 
     def close(self) -> None:
+        """Close the cache and shutdown the executor if they exist."""
         if self.cache:
             self.cache.close()
         if self.executor:
@@ -156,6 +286,11 @@ class FutureCache:
             self.executor = None
 
     def stats(self) -> str:
+        """Get statistics about cache usage.
+
+        Returns:
+            str: A string with cache size information
+        """
         logging.info(f"job calls: {self.worker_wrapper_call_count}")
         logging.info(f"cache calls: {self.worker_cache_call_count}")
         logging.info(f"worker calls: {self.worker_fn_call_count}")
@@ -165,15 +300,35 @@ class FutureCache:
 
 
 class JobFunctionCache(FutureCache):
+    """A specialized cache for a specific function with various input parameters.
+
+    This class simplifies caching results for a specific function called with
+    different sets of parameters. It wraps the general FutureCache with a focus
+    on a single function.
+
+    Attributes:
+        function (Callable): The function to cache results for
+    """
+
     def __init__(
         self,
         function: Callable,
-        overwrite=False,
-        executor: bool = False,
+        overwrite: bool = False,
+        executor: Executors = Executors.SERIAL,
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(100e8),
     ):
+        """Initialize a JobFunctionCache for a specific function.
+
+        Args:
+            function (Callable): The function to cache results for
+            overwrite (bool, optional): Whether to overwrite existing cached results. Defaults to False.
+            executor (Executors, optional): The execution strategy to use. Defaults to Executors.SERIAL.
+            cache_name (str, optional): Base name for the cache directory. Defaults to "fcache".
+            tag_index (bool, optional): Whether to enable tag-based indexing in the cache. Defaults to True.
+            size_limit (int, optional): Maximum size of the cache in bytes. Defaults to 10GB.
+        """
         super().__init__(
             executor=executor,
             cache_name=cache_name,
@@ -184,4 +339,14 @@ class JobFunctionCache(FutureCache):
         self.function = function
 
     def call(self, **kwargs) -> JobFuture:
+        """Call the wrapped function with the provided arguments.
+
+        This method creates a Job for the function call and submits it through the cache.
+
+        Args:
+            **kwargs: Arguments to pass to the function
+
+        Returns:
+            JobFuture: A future representing the function call
+        """
         return self.submit(Job(self.call_count, self.function, kwargs))