holobench 1.40.1__py3-none-any.whl → 1.42.0__py3-none-any.whl

bencher/example/meta/generate_meta.py

@@ -9,22 +9,68 @@ from bencher.example.meta.example_meta import BenchableObject
 class BenchMetaGen(bch.ParametrizedSweep):
     """This class uses bencher to display the multidimensional types bencher can represent"""
 
-    float_vars = bch.IntSweep(
-        default=0, bounds=(0, 4), doc="The number of floating point variables that are swept"
+    # Benchable object to use
+    benchable_obj = None  # Will store the benchable object instance
+
+    # Variables for controlling the sweep
+    float_vars_count = bch.IntSweep(
+        default=0, bounds=(0, 3), doc="The number of floating point variables that are swept"
     )
-    categorical_vars = bch.IntSweep(
-        default=0, bounds=(0, 3), doc="The number of categorical variables that are swept"
+    categorical_vars_count = bch.IntSweep(
+        default=0, bounds=(0, 2), doc="The number of categorical variables that are swept"
     )
-    sample_with_repeats = bch.IntSweep(default=1, bounds=(1, 100))
 
-    sample_over_time = bch.BoolSweep(default=False)
+    # Lists to store the actual variable names
+    float_var_names = []  # Will store float variable names
+    categorical_var_names = []  # Will store categorical variable names
+    result_var_names = []  # Will store result variable names
 
+    # Configuration options
+    sample_with_repeats = bch.IntSweep(default=1, bounds=(1, 100))
+    sample_over_time = bch.BoolSweep(default=False)
     level = bch.IntSweep(default=2, units="level", bounds=(2, 5))
-
     run_bench = False
 
     plots = bch.ResultReference(units="int")
 
+    def __init__(
+        self, benchable_obj=None, float_vars=None, categorical_vars=None, result_vars=None, **params
+    ):
+        """Initialize with a benchable object and variable specifications
+
+        Args:
+            benchable_obj: The benchable object to use
+            float_vars: List of float variable names to sweep
+            categorical_vars: List of categorical variable names to sweep
+            result_vars: List of result variable names to plot
+            **params: Additional parameters
+        """
+        super().__init__(**params)
+
+        # Set the benchable object
+        self.benchable_obj = benchable_obj if benchable_obj is not None else BenchableObject()
+
+        # Dynamically discover parameters from the benchable object
+        float_param_names = []
+        categorical_param_names = []
+        result_param_names = []
+
+        # Check all parameters of the benchable object
+        for name, param in self.benchable_obj.__class__.param.objects().items():
+            if hasattr(param, "bounds") and isinstance(param, bch.FloatSweep):
+                float_param_names.append(name)
+            elif isinstance(param, (bch.BoolSweep, bch.EnumSweep, bch.StringSweep)):
+                categorical_param_names.append(name)
+            elif isinstance(param, bch.ResultVar):
+                result_param_names.append(name)
+
+        # Use provided parameter lists or discovered ones
+        self.float_var_names = float_vars if float_vars is not None else float_param_names
+        self.categorical_var_names = (
+            categorical_vars if categorical_vars is not None else categorical_param_names
+        )
+        self.result_var_names = result_vars if result_vars is not None else result_param_names
+
     def __call__(self, **kwargs: Any) -> Any:
         self.update_params_from_kwargs(**kwargs)
 
@@ -34,50 +80,52 @@ class BenchMetaGen(bch.ParametrizedSweep):
         run_cfg.over_time = self.sample_over_time
         run_cfg.plot_size = 500
 
-        # bench = bch.Bench("benchable", BenchableObject(), run_cfg=run_cfg)
+        # Limit the variables to the specified counts
+        float_vars = []
+        # Apply the max_level=3 limit to the 3rd and subsequent float variables
+        for i, var_name in enumerate(self.float_var_names[: self.float_vars_count]):
+            if i >= 2:  # Third variable (index 2) and beyond
+                float_vars.append(bch.p(var_name, max_level=3))
+            else:
+                float_vars.append(var_name)
 
-        inputs_vars_float = [
-            "float1",
-            "float2",
-            bch.p("float3", max_level=3),
-            "sigma",
-        ]
+        categorical_vars = self.categorical_var_names[: self.categorical_vars_count]
+        input_vars = float_vars + categorical_vars
 
-        inputs_vars_cat = [
-            "noisy",
-            "noise_distribution",
-            "negate_output",
-        ]
-
-        input_vars = inputs_vars_float[: self.float_vars] + inputs_vars_cat[: self.categorical_vars]
-
-        if self.run_bench:
-            bench = BenchableObject().to_bench(run_cfg)
+        if self.run_bench and input_vars and self.result_var_names:
+            bench = self.benchable_obj.to_bench(run_cfg)
             res = bench.plot_sweep(
                 "test",
                 input_vars=input_vars,
-                result_vars=["distance", "sample_noise"],
+                result_vars=self.result_var_names,
                 plot_callbacks=False,
             )
             self.plots = bch.ResultReference()
             self.plots.obj = res.to_auto()
 
-        title = f"{self.float_vars}_float_{self.categorical_vars}_cat"
+        title = f"{self.float_vars_count}_float_{self.categorical_vars_count}_cat"
 
         nb = nbf.v4.new_notebook()
         text = f"""# {title}"""
 
+        # Customize notebook generation to use the actual benchmark object and variables
+        module_import = "import bencher as bch\n"
+        if self.benchable_obj.__class__.__module__ != "__main__":
+            benchmark_import = f"from {self.benchable_obj.__class__.__module__} import {self.benchable_obj.__class__.__name__}\n"
+        else:
+            # If it's from main, use BenchableObject as fallback
+            benchmark_import = "from bencher.example.meta.example_meta import BenchableObject\n"
+
         code_gen = f"""
 %%capture
-import bencher as bch
-from bencher.example.meta.example_meta import BenchableObject
-
+{module_import}
+{benchmark_import}
 run_cfg = bch.BenchRunCfg()
 run_cfg.repeats = {self.sample_with_repeats}
 run_cfg.level = 4 
-bench = BenchableObject().to_bench(run_cfg)
+bench = {self.benchable_obj.__class__.__name__}().to_bench(run_cfg)
 res=bench.plot_sweep(input_vars={input_vars},
-                    result_vars=["distance","sample_noise"])
+                    result_vars={self.result_var_names})
 """
         code_results = """
 from bokeh.io import output_notebook
@@ -107,14 +155,14 @@ def example_meta(run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
         description="""## All Combinations of Variable Sweeps and Resulting Plots
 This uses bencher to display all the combinations of plots bencher is able to produce""",
         input_vars=[
-            bch.p("float_vars", [0, 1]),
-            "categorical_vars",
+            bch.p("float_vars_count", [0, 1]),
+            "categorical_vars_count",
             bch.p("sample_with_repeats", [1, 20]),
         ],
         const_vars=[
-            # BenchMeta.param.float_vars.with_const(1),
+            # BenchMeta.param.float_vars_count.with_const(1),
             # BenchMeta.param.sample_with_repeats.with_const(2),
-            # BenchMeta.param.categorical_vars.with_const(2),
+            # BenchMeta.param.categorical_vars_count.with_const(2),
             # BenchMeta.param.sample_over_time.with_const(True),
         ],
     )
@@ -123,8 +171,8 @@ This uses bencher to display all the combinations of plots bencher is able to pr
         description="""## All Combinations of Variable Sweeps and Resulting Plots
 This uses bencher to display all the combinations of plots bencher is able to produce""",
         input_vars=[
-            bch.p("float_vars", [2, 3]),
-            "categorical_vars",
+            bch.p("float_vars_count", [2, 3]),
+            "categorical_vars_count",
         ],
     )
 
@@ -133,13 +181,13 @@ This uses bencher to display all the combinations of plots bencher is able to pr
     #         description="""## All Combinations of Variable Sweeps and Resulting Plots
     # This uses bencher to display all the combinations of plots bencher is able to produce""",
     #         input_vars=[
-    #             bch.p("float_vars", [2, 3]),
-    #             "categorical_vars",
+    #             bch.p("float_vars_count", [2, 3]),
+    #             "categorical_vars_count",
     #         ],
     #         const_vars=[
     #             dict(level=3)
     #             # BenchMeta.param.sample_with_repeats.with_const(2),
-    #             # BenchMeta.param.categorical_vars.with_const(2),
+    #             # BenchMeta.param.categorical_vars_count.with_const(2),
     #             # BenchMeta.param.sample_over_time.with_const(True),
     #         ],
     #     )

bencher/job.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 from typing import Callable
-from sortedcontainers import SortedDict
 import logging
 from diskcache import Cache
 from concurrent.futures import Future, ProcessPoolExecutor
@@ -11,27 +10,72 @@ from enum import auto
 try:
     from scoop import futures as scoop_future_executor
 except ImportError as e:
-    logging.warning(e.msg)
     scoop_future_executor = None
 
 
 class Job:
+    """Represents a benchmarking job to be executed or retrieved from cache.
+
+    A Job encapsulates a function, its arguments, and metadata for caching
+    and tracking purposes.
+
+    Attributes:
+        job_id (str): A unique identifier for the job, used for logging
+        function (Callable): The function to be executed
+        job_args (dict): Arguments to pass to the function
+        job_key (str): A hash key for caching, derived from job_args if not provided
+        tag (str): Optional tag for grouping related jobs
+    """
+
     def __init__(
-        self, job_id: str, function: Callable, job_args: dict, job_key=None, tag=""
+        self, job_id: str, function: Callable, job_args: dict, job_key: str = None, tag: str = ""
     ) -> None:
+        """Initialize a Job with function and arguments.
+
+        Args:
+            job_id (str): A unique identifier for this job
+            function (Callable): The function to execute
+            job_args (dict): Arguments to pass to the function
+            job_key (str, optional): Cache key for this job. If None, will be generated
+                from job_args. Defaults to None.
+            tag (str, optional): Tag for grouping related jobs. Defaults to "".
+        """
         self.job_id = job_id
         self.function = function
         self.job_args = job_args
         if job_key is None:
-            self.job_key = hash_sha1(tuple(SortedDict(self.job_args).items()))
+            self.job_key = hash_sha1(tuple(sorted(self.job_args.items())))
         else:
             self.job_key = job_key
         self.tag = tag
 
 
-# @dataclass
 class JobFuture:
+    """A wrapper for a job result or future that handles caching.
+
+    This class provides a unified interface for handling both immediate results
+    and futures (for asynchronous execution). It also handles caching results
+    when they become available.
+
+    Attributes:
+        job (Job): The job this future corresponds to
+        res (dict): The result, if available immediately
+        future (Future): The future representing the pending job, if executed asynchronously
+        cache: The cache to store results in when they become available
+    """
+
     def __init__(self, job: Job, res: dict = None, future: Future = None, cache=None) -> None:
+        """Initialize a JobFuture with either an immediate result or a future.
+
+        Args:
+            job (Job): The job this future corresponds to
+            res (dict, optional): The immediate result, if available. Defaults to None.
+            future (Future, optional): The future representing the pending result. Defaults to None.
+            cache (Cache, optional): The cache to store results in. Defaults to None.
+
+        Raises:
+            AssertionError: If neither res nor future is provided
+        """
         self.job = job
         self.res = res
         self.future = future
@@ -42,7 +86,16 @@ class JobFuture:
 
         self.cache = cache
 
-    def result(self):
+    def result(self) -> dict:
+        """Get the job result, waiting for completion if necessary.
+
+        If the result is not immediately available (i.e., it's a future),
+        this method will wait for the future to complete. Once the result
+        is available, it will be cached if a cache is provided.
+
+        Returns:
+            dict: The job result
+        """
         if self.future is not None:
             self.res = self.future.result()
         if self.cache is not None and self.res is not None:
@@ -51,18 +104,42 @@ class JobFuture:
 
 
 def run_job(job: Job) -> dict:
+    """Execute a job by calling its function with the provided arguments.
+
+    This is a helper function used primarily by executors to run jobs.
+
+    Args:
+        job (Job): The job to execute
+
+    Returns:
+        dict: The result of the job execution
+    """
     result = job.function(**job.job_args)
     return result
 
 
 class Executors(StrEnum):
+    """Enumeration of available execution strategies for benchmark jobs.
+
+    This enum defines the execution modes for running benchmark jobs
+    and provides a factory method to create appropriate executors.
+    """
+
     SERIAL = auto()  # slow but reliable
     MULTIPROCESSING = auto()  # breaks for large number of futures
     SCOOP = auto()  # requires running with python -m scoop your_file.py
     # THREADS=auto() #not that useful as most bench code is cpu bound
 
     @staticmethod
-    def factory(provider: Executors) -> Future:
+    def factory(provider: "Executors") -> Future | None:
+        """Create an executor instance based on the specified execution strategy.
+
+        Args:
+            provider (Executors): The type of executor to create
+
+        Returns:
+            Future | None: The executor instance, or None for serial execution
+        """
         providers = {
             Executors.SERIAL: None,
             Executors.MULTIPROCESSING: ProcessPoolExecutor(),
@@ -72,7 +149,23 @@ class Executors(StrEnum):
 
 
 class FutureCache:
-    """The aim of this class is to provide a unified interface for running jobs.  T"""
+    """A cache system for benchmark job results with executor support.
+
+    This class provides a unified interface for running benchmark jobs either serially
+    or in parallel, with optional caching of results. It manages the execution strategy,
+    caching policy, and tracks statistics about job execution.
+
+    Attributes:
+        executor_type (Executors): The execution strategy to use
+        executor: The executor instance, created on demand
+        cache (Cache): Cache for storing job results
+        overwrite (bool): Whether to overwrite existing cached results
+        call_count (int): Counter for job calls
+        size_limit (int): Maximum size of the cache in bytes
+        worker_wrapper_call_count (int): Number of job submissions
+        worker_fn_call_count (int): Number of actual function executions
+        worker_cache_call_count (int): Number of cache hits
+    """
 
     def __init__(
         self,
@@ -81,8 +174,18 @@ class FutureCache:
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(20e9),  # 20 GB
-        cache_results=True,
+        cache_results: bool = True,
     ):
+        """Initialize a FutureCache with optional caching and execution settings.
+
+        Args:
+            executor (Executors, optional): The execution strategy to use. Defaults to Executors.SERIAL.
+            overwrite (bool, optional): Whether to overwrite existing cached results. Defaults to True.
+            cache_name (str, optional): Base name for the cache directory. Defaults to "fcache".
+            tag_index (bool, optional): Whether to enable tag-based indexing in the cache. Defaults to True.
+            size_limit (int, optional): Maximum size of the cache in bytes. Defaults to 20GB.
+            cache_results (bool, optional): Whether to cache results at all. Defaults to True.
+        """
         self.executor_type = executor
         self.executor = None
         if cache_results:
@@ -100,6 +203,18 @@ class FutureCache:
         self.worker_cache_call_count = 0
 
     def submit(self, job: Job) -> JobFuture:
+        """Submit a job for execution, with caching if enabled.
+
+        This method first checks if the job result is already in the cache (if caching is enabled
+        and overwrite is False). If not found in the cache, it executes the job either serially
+        or using the configured executor.
+
+        Args:
+            job (Job): The job to submit
+
+        Returns:
+            JobFuture: A future representing the job execution
+        """
         self.worker_wrapper_call_count += 1
 
         if self.cache is not None:
@@ -132,25 +247,38 @@ class FutureCache:
         )
 
     def overwrite_msg(self, job: Job, suffix: str) -> None:
+        """Log a message about overwriting or using cache.
+
+        Args:
+            job (Job): The job being executed
+            suffix (str): Additional text to add to the log message
+        """
         msg = "OVERWRITING" if self.overwrite else "NOT in"
         logging.info(f"{job.job_id} {msg} cache{suffix}")
 
     def clear_call_counts(self) -> None:
-        """Clear the worker and cache call counts, to help debug and assert caching is happening properly"""
+        """Clear the worker and cache call counts, to help debug and assert caching is happening properly."""
         self.worker_wrapper_call_count = 0
         self.worker_fn_call_count = 0
         self.worker_cache_call_count = 0
 
     def clear_cache(self) -> None:
+        """Clear all entries from the cache."""
         if self.cache:
             self.cache.clear()
 
     def clear_tag(self, tag: str) -> None:
+        """Remove all cache entries with the specified tag.
+
+        Args:
+            tag (str): The tag identifying entries to remove from the cache
+        """
         logging.info(f"clearing the sample cache for tag: {tag}")
         removed_vals = self.cache.evict(tag)
         logging.info(f"removed: {removed_vals} items from the cache")
 
     def close(self) -> None:
+        """Close the cache and shutdown the executor if they exist."""
         if self.cache:
             self.cache.close()
         if self.executor:
@@ -158,6 +286,11 @@ class FutureCache:
             self.executor = None
 
     def stats(self) -> str:
+        """Get statistics about cache usage.
+
+        Returns:
+            str: A string with cache size information
+        """
         logging.info(f"job calls: {self.worker_wrapper_call_count}")
         logging.info(f"cache calls: {self.worker_cache_call_count}")
         logging.info(f"worker calls: {self.worker_fn_call_count}")
@@ -167,15 +300,35 @@ class FutureCache:
 
 
 class JobFunctionCache(FutureCache):
+    """A specialized cache for a specific function with various input parameters.
+
+    This class simplifies caching results for a specific function called with
+    different sets of parameters. It wraps the general FutureCache with a focus
+    on a single function.
+
+    Attributes:
+        function (Callable): The function to cache results for
+    """
+
     def __init__(
         self,
         function: Callable,
-        overwrite=False,
-        executor: bool = False,
+        overwrite: bool = False,
+        executor: Executors = Executors.SERIAL,
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(100e8),
     ):
+        """Initialize a JobFunctionCache for a specific function.
+
+        Args:
+            function (Callable): The function to cache results for
+            overwrite (bool, optional): Whether to overwrite existing cached results. Defaults to False.
+            executor (Executors, optional): The execution strategy to use. Defaults to Executors.SERIAL.
+            cache_name (str, optional): Base name for the cache directory. Defaults to "fcache".
+            tag_index (bool, optional): Whether to enable tag-based indexing in the cache. Defaults to True.
+            size_limit (int, optional): Maximum size of the cache in bytes. Defaults to 10GB.
+        """
         super().__init__(
             executor=executor,
             cache_name=cache_name,
@@ -186,4 +339,14 @@ class JobFunctionCache(FutureCache):
         self.function = function
 
     def call(self, **kwargs) -> JobFuture:
+        """Call the wrapped function with the provided arguments.
+
+        This method creates a Job for the function call and submits it through the cache.
+
+        Args:
+            **kwargs: Arguments to pass to the function
+
+        Returns:
+            JobFuture: A future representing the function call
+        """
         return self.submit(Job(self.call_count, self.function, kwargs))

bencher/plotting/plot_filter.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from typing import Optional
 from dataclasses import dataclass
 from bencher.plotting.plt_cnt_cfg import PltCntCfg
+import logging
 import panel as pn
 
 
@@ -43,9 +44,20 @@ class VarRange:
 
         return lower_match and upper_match
 
-    def matches_info(self, val, name):
+    def matches_info(self, val: int, name: str) -> tuple[bool, str]:
+        """Get matching info for a value with a descriptive name.
+
+        Args:
+            val (int): A positive integer to check against the range
+            name (str): A descriptive name for the value being checked, used in the output string
+
+        Returns:
+            tuple[bool, str]: A tuple containing:
+                - bool: True if the value matches the range, False otherwise
+                - str: A formatted string describing the match result
+        """
         match = self.matches(val)
-        info = f"{name}\t{match}\t{self.lower_bound}>= {val} <={self.upper_bound}"
+        info = f"{name}\t{self.lower_bound}>= {val} <={self.upper_bound} is {match}"
         return match, info
 
     def __str__(self) -> str:
@@ -65,13 +77,21 @@ class PlotFilter:
     input_range: VarRange = VarRange(1, None)
 
     def matches_result(
-        self, plt_cnt_cfg: PltCntCfg, plot_name: str, override: bool = False
+        self, plt_cnt_cfg: PltCntCfg, plot_name: str, override: bool
     ) -> PlotMatchesResult:
-        """Checks if the result data signature matches the type of data the plot is able to display."""
+        """Checks if the result data signature matches the type of data the plot is able to display.
+
+        Args:
+            plt_cnt_cfg (PltCntCfg): Configuration containing counts of different plot elements
+            plot_name (str): Name of the plot being checked
+            override (bool): Whether to override filter matching rules
+
+        Returns:
+            PlotMatchesResult: Object containing match results and information
+        """
         return PlotMatchesResult(self, plt_cnt_cfg, plot_name, override)
 
 
-# @dataclass
 class PlotMatchesResult:
     """Stores information about which properties match the requirements of a particular plotter"""
 
@@ -80,12 +100,20 @@ class PlotMatchesResult:
         plot_filter: PlotFilter,
         plt_cnt_cfg: PltCntCfg,
         plot_name: str,
-        override: bool = False,
-    ):
-        match_info = []
-        matches = []
+        override: bool,
+    ) -> None:
+        """Initialize a PlotMatchesResult with filter matching information.
 
-        match_candidates = [
+        Args:
+            plot_filter (PlotFilter): The filter defining acceptable ranges for plot properties
+            plt_cnt_cfg (PltCntCfg): Configuration containing counts of different plot elements
+            plot_name (str): Name of the plot being checked
+            override (bool): Whether to override filter matching rules
+        """
+        match_info: list[str] = []
+        matches: list[bool] = []
+
+        match_candidates: list[tuple[VarRange, int, str]] = [
             (plot_filter.float_range, plt_cnt_cfg.float_cnt, "float"),
             (plot_filter.cat_range, plt_cnt_cfg.cat_cnt, "cat"),
             (plot_filter.vector_len, plt_cnt_cfg.vector_len, "vec"),
@@ -99,7 +127,7 @@ class PlotMatchesResult:
             match, info = m.matches_info(cnt, name)
             matches.append(match)
             if not match:
-                match_info.append(info)
+                match_info.append(f"\t{info}")
         if override:
             match_info.append(f"override: {override}")
             self.overall = True
@@ -107,15 +135,22 @@ class PlotMatchesResult:
             self.overall = all(matches)
 
         match_info.insert(0, f"plot {plot_name} matches: {self.overall}")
-        self.matches_info = "\n".join(match_info).strip()
-        self.plt_cnt_cfg = plt_cnt_cfg
+        self.matches_info: str = "\n".join(match_info).strip()
+        self.plt_cnt_cfg: PltCntCfg = plt_cnt_cfg
 
-        if self.plt_cnt_cfg.print_debug:
-            print(f"checking {plot_name} result: {self.overall}")
-            if not self.overall:
-                print(self.matches_info)
+        # if self.plt_cnt_cfg.print_debug:
+        logging.info(self.matches_info)
 
     def to_panel(self, **kwargs) -> Optional[pn.pane.Markdown]:
+        """Convert match information to a Panel Markdown pane if debug mode is enabled.
+
+        Args:
+            **kwargs: Additional keyword arguments to pass to the Panel Markdown constructor
+
+        Returns:
+            Optional[pn.pane.Markdown]: A Markdown pane containing match information if in debug mode,
+                                        None otherwise
+        """
         if self.plt_cnt_cfg.print_debug:
             return pn.pane.Markdown(self.matches_info, **kwargs)
         return None