benchstats 3.3.2__tar.gz → 3.4.0__tar.gz

{benchstats-3.3.2 → benchstats-3.4.0}/CHANGELOG.md

@@ -1,3 +1,19 @@
+# 3.4.0
+- `qbench.showBench()` can take benchmark results as a dict of per-name 2D `[reps, iters]` arrays
+(with `bm_names=None` and a non-empty `alt_delimiter`), not only a single 3D numpy stack.
+- `qbench.showBench()` gained `allow_inplace_reshuffle`: opt in to allow in-place shuffling/copy
+avoidance when bootstrapping or normalizing inputs, instead of always working on a private copy.
+- Restored default value for `qbench.showBench(pvalue_stats_bootstrap=)` to 1000.
+- `CompareStatsResult` now exposes `comparisons` (which original benchmark names each row compares)
+and `comparison_indices` (which rows of the stacked `showBench` input they correspond to; absent
+for dict-input mode), for easier downstream automation.
+
+# 3.3.3
+- breaking change, though unlikely important for anyone: due to T-test also being affected by
+bad data, had to rename `compareStats()`'s parameter `brunnermunzel_workaround` to `edge_cases_workaround`
+and extended its effect on t-test.
+- Fix `_at_least_one_differs` flag handling in `CompareStatsResult::updatePvalStats()`.
+
 # 3.3.1-.2
 - Changed the order of default metrics of `qbench.showBench()` to make `mean` the leading metric, as
 it's typically the most correct one.

{benchstats-3.3.2/src/benchstats.egg-info → benchstats-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchstats
-Version: 3.3.2
+Version: 3.4.0
 Summary: Statistical Testing for Benchmark Results Comparison
 Author-email: Aleksei Rechinskii <5dpea9nhd@mozmail.com>
 License-Expression: MIT

{benchstats-3.3.2 → benchstats-3.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "benchstats"
-version = "3.3.2"
+version = "3.4.0"
 description = "Statistical Testing for Benchmark Results Comparison"
 readme = "README.md"
 authors = [{ name = "Aleksei Rechinskii", email = "5dpea9nhd@mozmail.com" }]

{benchstats-3.3.2 → benchstats-3.4.0}/src/benchstats/compare.py

@@ -39,7 +39,7 @@ class BmCompResult(
     def __new__(
         cls,
         res: str,
-        pval: float | int,
+        pvalue: float | int,
         v0: float | int | np.ndarray,
         v1: float | int | np.ndarray,
         siz0: int,
@@ -52,8 +52,8 @@ class BmCompResult(
             > when set1 is stochastically greater than set2,
             ~ when set1 is not stochastically less or greater than set2
 
-        - pval/pvalue is a pvalue associated with less or greater comparison result, or a minimum of
-            pvalues for less/greater comparison
+        - pvalue is a p-value associated with less or greater comparison result, or a minimum of
+            p-values for less/greater comparison
 
         - v0/val_set0 and v1/val_set1 are either representative values (mean) of a corresponding
             set, or the whole set iself, depending on `store_sets` flag value of compareStats()
@@ -62,10 +62,10 @@ class BmCompResult(
         """
 
         # assert isinstance(rel, bool) and isinstance(pval, (float, np.floating))
-        if isinstance(pval, int):
-            pval = float(pval)
-        assert isinstance(pval, kAllowedFpTypes)
-        assert 0 <= pval <= 1
+        if isinstance(pvalue, int):
+            pvalue = float(pvalue)
+        assert isinstance(pvalue, kAllowedFpTypes)
+        assert 0 <= pvalue <= 1
         if isinstance(v0, int):
             v0 = float(v0)
         if isinstance(v1, int):
@@ -80,7 +80,7 @@ class BmCompResult(
         return super().__new__(
             cls,
             res,
-            float(pval),
+            float(pvalue),
             v0 if isinstance(v0, np.ndarray) else float(v0),
             v1 if isinstance(v1, np.ndarray) else float(v1),
             siz0,
@@ -95,10 +95,12 @@ class CompareStatsResult:
         method: str,
         alpha: float,
         at_least_one_differs: bool,
+        comparisons: dict[str, tuple[str, str]],
     ):
         """Constructor to verify initialization correctness.
-        - results field is a mapping {benchmark_name -> {metric_name -> CompResult}}. Keys of the
+        - results is a mapping {benchmark_name -> {metric_name -> CompResult}}. Keys of the
             top-level dict are common between the data sources sg0 and sg1.
+        - comparisons is a mapping {benchmark_name -> (bm_name_in_set0, bm_name_in_set1)}
         """
         assert isinstance(results, dict) and isinstance(method, str) and method
         assert isinstance(alpha, kAllowedFpTypes)
@@ -107,6 +109,8 @@ class CompareStatsResult:
         self._method: str = method
         self._alpha: float = float(alpha)
         self._at_least_one_differs = bool(at_least_one_differs)
+        self._comparisons: dict[str, tuple[str, str]] = comparisons
+        self._comparison_indices: dict[str, tuple[int, int]] | None = None
         # {benchmark_name -> {metric_name -> {result_type -> list of pvalues}}}
         self._pval_stats: None | dict[str, dict[str, dict[str, list[float]]]] = None
 
@@ -122,10 +126,33 @@ class CompareStatsResult:
     def alpha(self) -> float:
         return self._alpha
 
+    @property
+    def comparisons(self) -> dict[str, tuple[str, str]]:
+        """Returns a mapping between a benchmark name in the results and a tuple of names of the
+        original data in the original data sets"""
+        return self._comparisons
+
     @property
     def at_least_one_differs(self) -> bool:
+        """Returns True if at least one benchmark/metric comparison result is different from "~"
+        (no difference) for any metric from `main_metrics` list (or for any metric if `main_metrics`
+        is None)"""
         return self._at_least_one_differs
 
+    @property
+    def comparison_indices(self) -> dict[str, tuple[int, int]] | None:
+        """Returns a mapping between a benchmark name in the results and a tuple of indices of the
+        original data in the original data sets"""
+        return self._comparison_indices
+
+    def setComparisonIndices(self, comparison_indices: dict[str, tuple[int, int]]):
+        """Sets a mapping between a benchmark name in the results and a tuple of indices of the
+        original data of the original data sets (when qbench's form of benchmarks is used)"""
+        # assert isinstance(comparison_indices, dict)
+        # assert all(isinstance(k, str) and isinstance(v, tuple) and len(v) == 2 and all(isinstance(i, int) for i in v) for k, v in comparison_indices.items())
+        # assert frozenset(comparison_indices.keys()) == frozenset(self._comparisons.keys())
+        self._comparison_indices = comparison_indices
+
     def getMetrics(self) -> tuple[str]:
         return tuple(next(iter(self._results.values())).keys())
 
@@ -134,13 +161,13 @@ class CompareStatsResult:
 
     def areAllSame(self) -> bool:
         """Tests if all benchmarks over all metrics compare same"""
-        return all([
+        return all(
             "~" == cr.result for bm_res in self._results.values() for cr in bm_res.values()
-        ])
+        )
 
     def areMetricsSame(self, metrics: Iterable[str]) -> bool:
         """Tests if all benchmarks over specified metrics compare same"""
-        return all(["~" == bm_res[m].result for bm_res in self._results.values() for m in metrics])
+        return all("~" == bm_res[m].result for bm_res in self._results.values() for m in metrics)
 
     @property
     def pval_stats_available(self) -> bool:
@@ -162,6 +189,8 @@ class CompareStatsResult:
         assert isinstance(other, CompareStatsResult)
         assert other.method == self.method and other.alpha == self.alpha
         assert self._results.keys() == other._results.keys()
+        assert self._comparisons == other._comparisons
+        self._at_least_one_differs = self._at_least_one_differs or other._at_least_one_differs
         for bm, omdict in other._results.items():
             mdict = self._results[bm]
             assert mdict.keys() == omdict.keys()
@@ -182,7 +211,7 @@ def poolBenchmarks(
     sg0: dict[str, dict[str, Iterable[float]]],
     sg1: dict[str, dict[str, Iterable[float]]] | None,
     logger: LoggingConsole | None = None,
-) -> dict[str, dict[str, dict[str, Iterable[float]]]]:
+) -> tuple[dict[str, dict[str, dict[str, Iterable[float]]]], dict[str, dict[str, str]]]:
     """Using the specified delimiter divides each benchmark name into (the longest possible) common
     part and an alternative name part, merges the benchmarks from one or two sets of benchmarks into
     a single pool represented by a dictionary where a key specifies common part of benchmark name
@@ -192,6 +221,9 @@ def poolBenchmarks(
     If a benchmark name doesn't contain the delimiter, then its common name is assumed to be a whole
     benchmark name, and its alternative name is assumed to either "0" or "1" depending on the
     benchmark set (sg0 or sg1) it belongs to.
+
+    Returns a mapping {common_name -> {alternative_name -> {metric_name -> values}}}, and a mapping
+    of original names {common_name -> {alternative_name -> original_name}}
     """
     assert isinstance(alt_delimiter, str) and len(alt_delimiter) > 0
     assert isinstance(sg0, dict) and (isinstance(sg1, dict) or sg1 is None)
@@ -212,12 +244,15 @@ def poolBenchmarks(
         else:
             return bm_name, pfx
 
+    pool = {}
+    original_names = {}
+
     def _do_pool(
-        pool: dict[str, dict[str, dict[str, Iterable[float]]]],
         sg: dict[str, dict[str, Iterable[float]]],
         pfx: str,
     ):
         """Merges the benchmarks from one set of benchmarks into a pool"""
+        nonlocal pool, original_names
         for bm_name, metrics in sg.items():
             common_name, alt_name = splitName(bm_name, pfx)
             cmn_bm = pool.setdefault(common_name, {})
@@ -229,15 +264,15 @@ def poolBenchmarks(
                 )
                 continue
             cmn_bm[alt_name] = metrics
+            original_names.setdefault(common_name, {})[alt_name] = bm_name
 
-    pool = {}
     if sg1 is None:
-        _do_pool(pool, sg0, "")
+        _do_pool(sg0, "")
     else:
-        _do_pool(pool, sg0, "0")
-        _do_pool(pool, sg1, "1")
+        _do_pool(sg0, "0")
+        _do_pool(sg1, "1")
 
-    return pool
+    return pool, original_names
 
 
 def compareStats(
@@ -246,10 +281,10 @@ def compareStats(
     alt_delimiter: str | None = None,
     method: str = next(iter(kMethods.keys())),
     alpha: float = kDefaultAlpha,
-    main_metrics: None | list[str] | tuple[str] = None,
+    main_metrics: None | str | list[str] | tuple[str] = None,
     debug_log: None | bool | LoggingConsole = True,
     store_sets: bool = False,
-    brunnermunzel_workaround: None | bool = None,
+    edge_cases_workaround: None | bool = None,
 ) -> CompareStatsResult:
     """Perform comparison for statistical significance of benchmark results using a chosen
     statistical method.
@@ -273,29 +308,30 @@ def compareStats(
     comparisons: "foo|0bar vs 0baz", "foo|0bar vs 1qux" and "foo|0baz vs 1qux". Numbers 0 and 1 in
     benchmark alternative names references source set of benchmark results (sg0 or sg1).
 
-    `main_metrics` is either a list/tuple of strings describing containing main metrics for the
-    purpose of computing of at_least_one_differs flag, or None (then all metrics are main)
+    `main_metrics` is either a string, a list/tuple of strings describing containing main metrics
+    for the purpose of computing of at_least_one_differs flag, or None (then all metrics are main).
 
     `debug_log` is a bool, but could also be None (==False) or a logger object like LoggingConsole.
 
     `store_sets` is a bool, True will make store whole corresponding dataset into a BmCompResult
     instead of a mean of the set.
 
-    `brunnermunzel_workaround`: Formally speaking, Brunner Munzel test method is "underspecified"
+    `edge_cases_workaround`: Formally speaking, Brunner Munzel test method is "underspecified"
     in a sense that by design it doesn't handle a case of non-intersecting sample sets, or sets
     consisting of the same single element (https://aakinshin.net/posts/brunner-munzel-corner-case/).
     scipy.stats.brunnermunzel() implementation at least in versions 1.10-1.15.2 directly follows the
     words of the original paper and doesn't introduce handling for that cases, which causes warnings
     and nans to appear in results. IMO from an engineering point of view this is more like just a
-    bug (https://github.com/scipy/scipy/issues/22664). `brunnermunzel_workaround` is a flag to
-    handle it: value of None handles the bug only for method == 'brunnermunzel', a bool otherwise
-    controls whether to handle the bug irrespective of the method used (this might give different
-    pvalues for other tests, but this shouldn't affect inference results unless alpha threshold
-    value is too wild). It should be noted that for small sample sizes (under 10 elements in any of
+    bug (https://github.com/scipy/scipy/issues/22664). The same issue also applies to Student's
+    t-test. `edge_cases_workaround` is a flag to handle it: value of None handles the bug only for
+    affected methods, a bool otherwise controls whether to handle the bug irrespective of the method
+    used (this might give different pvalues for other tests, but this shouldn't affect inference
+    results unless alpha threshold value is too wild).
+    It should be noted that for small sample sizes (under 10 elements in any of
     sample sets) this most certainly leads to a wrong p-value result that underestimate a
     probability of error. However, with 10, or preferably more, sample set sizes, this should be
     negligible.
-    While brunnermunzel_workaround allows to disable the workaround, I doubt it has any practical
+    While edge_cases_workaround allows to disable the workaround, I doubt it has any practical
     value in the context of benchmark results comparison.
 
     Returns an instance of CompareStatsResult class
@@ -306,7 +342,9 @@ def compareStats(
     assert isinstance(sg0, dict) and (isinstance(sg1, dict) or (with_alternatives and sg1 is None))
     assert isinstance(method, str) and method in kMethods, "unsupported method"
     assert isinstance(alpha, kAllowedFpTypes) and 0 < alpha and alpha < 0.5
-    assert brunnermunzel_workaround is None or isinstance(brunnermunzel_workaround, bool)
+    assert edge_cases_workaround is None or isinstance(edge_cases_workaround, bool)
+    if isinstance(main_metrics, str):
+        main_metrics = (main_metrics,)
     assert main_metrics is None or isinstance(main_metrics, (list, tuple))
 
     if debug_log is None or (isinstance(debug_log, bool) and not debug_log):
@@ -317,8 +355,8 @@ def compareStats(
         logger = debug_log
         debug_log = True
 
-    if brunnermunzel_workaround is None:
-        brunnermunzel_workaround = method == "brunnermunzel"
+    if edge_cases_workaround is None:
+        edge_cases_workaround = method in ("brunnermunzel", "ttest_ind")
 
     def warn(*args, **kwargs):
         if debug_log:
@@ -330,11 +368,11 @@ def compareStats(
 
     def computePValues(s0, s1):
         """Computes and return pvalues of s0 being stochastically less or greater than s1"""
-        if brunnermunzel_workaround:
+        if edge_cases_workaround:
             # test for edge cases that brunnermunzel can't handle properly
             mn0, mx0, mn1, mx1 = np.min(s0), np.max(s0), np.min(s1), np.max(s1)
             assert np.all(np.isfinite([mn0, mx0, mn1, mx1]))  # sanity check, more asserts below
-            all_eq = np.all([mn0 == mx0, mn0 == mn1, mn0 == mx1])
+            all_eq = all([mn0 == mx0, mn0 == mn1, mn0 == mx1])
             all_less, all_greater = mx0 < mn1, mn0 > mx1
 
             if all_eq:
@@ -348,7 +386,7 @@ def compareStats(
         res_greater = stat_func(s0, s1, alternative="greater")
 
         less_pvalue, greater_pvalue = res_less.pvalue, res_greater.pvalue
-        if brunnermunzel_workaround:
+        if edge_cases_workaround:
             assert np.all(np.isfinite([less_pvalue, greater_pvalue]))
 
         return less_pvalue, greater_pvalue
@@ -447,9 +485,10 @@ def compareStats(
         return bm_results
 
     results = {}
+    comparisons: dict[str, tuple[str, str]] = {}  # {comparison_name -> (bm_name0, bm_name1)}
 
     if with_alternatives:
-        pool = poolBenchmarks(alt_delimiter, sg0, sg1, logger if debug_log else None)
+        pool, orig_names = poolBenchmarks(alt_delimiter, sg0, sg1, logger if debug_log else None)
 
         for common_name, alternatives in pool.items():
             if len(alternatives) <= 1:
@@ -460,12 +499,14 @@ def compareStats(
                 )
                 continue
 
+            onames = orig_names[common_name]
             for altA, altB in itertools.combinations(alternatives.keys(), 2):
                 assert isinstance(altA, str) and isinstance(altB, str)
                 assert altA != altB
                 bm_name = f"{common_name} {alt_delimiter} {altA} vs {altB}"
                 assert bm_name not in results
                 results[bm_name] = compareBenchmark(bm_name, alternatives[altA], alternatives[altB])
+                comparisons[bm_name] = (onames[altA], onames[altB])
 
     else:
         common_bms = sg0.keys() & sg1.keys()
@@ -483,5 +524,6 @@ def compareStats(
                 warn("Key/benchmark name '%s' not found in set2", bm_name)
                 continue
             results[bm_name] = compareBenchmark(bm_name, metrics1, sg1[bm_name])
+            comparisons[bm_name] = (bm_name, bm_name)
 
-    return CompareStatsResult(results, method, alpha, bool(at_least_one_differs))
+    return CompareStatsResult(results, method, alpha, bool(at_least_one_differs), comparisons)

{benchstats-3.3.2 → benchstats-3.4.0}/src/benchstats/qbench.py

@@ -6,6 +6,7 @@ of Python callables. Some parts, however, could be reused outside of Python benc
 import argparse
 from collections import namedtuple
 from collections.abc import Callable, Iterable
+from copy import deepcopy
 import itertools
 import inspect
 import numpy as np
@@ -66,7 +67,8 @@ class BenchmarkDescription(
         args_func: Callable[[], Iterable] | None = None,
         clear_cache_func: Callable[[Iterable], None] | None = None,
         *,
-        wait_complete: Callable[[Any], Any] | None = None,  # sets the default value for the 2 below
+        # wait_complete sets the default value for the other 2 below
+        wait_complete: Callable[[Any], Any] | None = None,
         wait_arg_complete: Callable[[Any], Any] | None = None,
         wait_func_complete: Callable[[Any], Any] | None = None,
     ):
@@ -362,7 +364,7 @@ def bench2(func1, func2, **kwargs):
 
 def resultsToDict(
     results: np.ndarray, bm_names: tuple | list | str = "code", alt_delimiter: str | None = None
-) -> tuple[dict[str, np.ndarray], str]:
+) -> tuple[dict[str, np.ndarray], str, dict[str, str]]:
     """Store benchmark results in a dictionary benchmark_name -> benchmark_results
 
     Arguments:
@@ -385,11 +387,13 @@ def resultsToDict(
                 of functions benchmarked (results.shape[0]), where each string is parsed
                 according to the expected format: "<common_name>{alt_delimiter}<alternative_name>".
                 Benchmarks with the same "<common_name>" are compared pairwise.
-    Returns a tuple of 2 objects:
+    Returns a tuple of 3 objects:
         dict[str, np.ndarray]: A dictionary mapping benchmark names to benchmark results as 2D
             matrix of shape (reps, iters). Note that matricies returned are views into the original
             results array, so modifying them will modify the original results array as well.
         str - possibly modified alt_delimiter value
+        dict[str, int] - mapping of created benchmark names to indices of benchmark data in the
+            results array
     """
     if results.ndim == 2:
         results = np.expand_dims(results, axis=0)
@@ -406,13 +410,17 @@ def resultsToDict(
     n_names = len(bm_names)
     assert n_names > 0, bm_names_err
 
-    def addBenchmark(sg: dict, bm_name: str, b_idx: int):
+    sg = {}
+    bm_idxs = {}
+
+    def addBenchmark(bm_name: str, b_idx: int):
+        nonlocal sg
         assert bm_name not in sg, (
             f"Duplicate benchmark name '{bm_name}' found. Please provide unique names."
         )
         sg[bm_name] = results[b_idx, :, :]  # .copy()
+        bm_idxs[bm_name] = b_idx
 
-    sg = {}
     if alt_delimiter is None:
         if n_funcs % n_names != 0:
             raise ValueError(
@@ -423,8 +431,8 @@ def resultsToDict(
         alt_delimiter = "|"
 
         if n_funcs_per_bm <= 1:
-            addBenchmark(sg, f"{bm_names[0]}{alt_delimiter}0", 0)
-            addBenchmark(sg, f"{bm_names[0]}{alt_delimiter}same", 0)
+            addBenchmark(f"{bm_names[0]}{alt_delimiter}0", 0)
+            addBenchmark(f"{bm_names[0]}{alt_delimiter}same", 0)
         else:
             combination_set = list(itertools.combinations(range(n_funcs_per_bm), 2))
             func_group_idx = 0
@@ -432,10 +440,10 @@ def resultsToDict(
                 for cs in combination_set:
                     n1 = f"{bm_name}{alt_delimiter}{cs[0]}"
                     if n1 not in sg:
-                        addBenchmark(sg, n1, func_group_idx + cs[0])
+                        addBenchmark(n1, func_group_idx + cs[0])
                     n2 = f"{bm_name}{alt_delimiter}{cs[1]}"
                     if n2 not in sg:
-                        addBenchmark(sg, n2, func_group_idx + cs[1])
+                        addBenchmark(n2, func_group_idx + cs[1])
                 func_group_idx += n_funcs_per_bm
 
     else:
@@ -453,9 +461,9 @@ def resultsToDict(
                 raise ValueError(
                     f"Duplicate benchmark name '{bm_name}' found. Please provide unique names."
                 )
-            addBenchmark(sg, bm_name, b_idx)
+            addBenchmark(bm_name, b_idx)
 
-    return sg, alt_delimiter
+    return sg, alt_delimiter, bm_idxs
 
 
 def _splitCompareStats_and_renderArgs(
@@ -495,24 +503,31 @@ def _splitCompareStats_and_renderArgs(
 
 
 def showBench(
-    results: np.ndarray,
+    results: np.ndarray | dict[str, np.ndarray],
     *,
-    bm_names: tuple | list | str = "code",
+    bm_names: tuple | list | str | None = "code",
     alt_delimiter: str | None = None,
     metrics: dict = {"mean": np.mean, "min": np.min},
     console: None | LoggingConsole = None,
-    pvalue_stats_bootstrap: int = 100,
+    pvalue_stats_bootstrap: int = 1000,
     pvalue_stats_bootstrap_seed: Any = None,
+    start_with_reshuffled: bool = False,
     show_progress_each: int = 1,
     render_report: bool = True,
+    same_console_for_progress: bool = False,
+    allow_inplace_reshuffle: bool = False,
     **kwCompareStats_and_renderArgs,
 ) -> CompareStatsResult:
     """
     Displays the benchmark results in a human-readable format.
 
     Args:
-    - results (np.ndarray): The benchmark results as a 3D [nfuncs, reps, iters] or 2D [reps, iters]
-        numpy array. Essentially, the output of the bench() function.
+    - results (np.ndarray | dict[str, np.ndarray]): The benchmark results as a 3D
+        [nfuncs, reps, iters] or 2D [reps, iters] numpy array. Essentially, the output of the
+        bench() function.
+        Alternatively, could be a dictionary mapping benchmark names to benchmark results as 2D
+        matrix of potentially different shapes (reps, iters). In that case, `bm_names` must be None
+        and `alt_delimiter` must be a valid delimiter string.
     - bm_names (tuple|list|str, optional) and alt_delimiter: (str|None, optional): defines the
         names of the benchmarks and how they are compared one against the other. Options are:
         - if alt_delimiter is None:
@@ -546,12 +561,28 @@ def showBench(
         assumption of the statistical tests.
         Note that large values like 1000 are computationally expensive, but provide more stable
         results thanks to a better coverage.
+        By default the first comparison is computed on the original results and the rest
+        `pvalue_stats_bootstrap` comparisons are computed on reshuffled results. This can be changed
+        by setting `start_with_reshuffled` to True, which make even the first comparison computed on
+        reshuffled results, doing `pvalue_stats_bootstrap - 1` iterations of bootstrapping in total.
     - pvalue_stats_bootstrap_seed (Any, default None): if pvalue bootstrapping is enabled, this seed
         will initialize the random number generator for bootstrapping.
+    - start_with_reshuffled (bool, False by default): if True and bootstrapping is enabled, even the
+        first comparison will be computed on reshuffled results. Otherwise it is ignored.
+        Note that benchmarking results rendering doesn't currently support judging the comparison
+        result by the p-value statistics (it always use results of the first comparison instead), so
+        it's not recommended to use this option if your code or rendered results interpretation
+        doesn't have an additional logic to take the bootstrapped results into account.
     - show_progress_each - if p-value bootstrapping is enabled and it's a positive integer,
         show progress bars and update them each that many bootstrapping iterations.
+    - same_console_for_progress (bool, False by default): If True, use the same console for progress
+        bar. Useful for using in scripts, but is disabled by default to prevent effect on exported
+        reports.
     - render_report (bool, True by default): If False, only returns CompareStatsResult object,
          but doesn't print the report.
+    - allow_inplace_reshuffle (bool, False by default): If True, allow the `results` to be
+        reshuffled for bootstrapping (or if .ndim < 3 for array argument) in place. Otherwise,
+        a copy of the results is made when necessary.
     - kwCompareStats_and_renderArgs: Any optional arguments of the compareStats()
         or renderComparisonResults() functions. By default compareStats() also gets
         store_sets=True argument, and renderComparisonResults() gets
@@ -567,18 +598,30 @@ def showBench(
                 kwCompareStats_and_renderArgs["debug_log"] = False
     assert console is None or isinstance(console, LoggingConsole)
 
-    if pvalue_stats_bootstrap > 0 or results.ndim == 2:
-        results = results.copy()
-        show_progress = isinstance(show_progress_each, int) and show_progress_each > 0
-        if results.ndim == 2:
-            results = np.expand_dims(results, axis=0)
+    results_is_dict = isinstance(results, dict)
+    if results_is_dict:
+        assert bm_names is None and isinstance(alt_delimiter, str) and len(alt_delimiter) > 0
+        assert all(isinstance(r, np.ndarray) and r.ndim == 2 for r in results.values())
 
-    assert results.ndim == 3, "results must be a 3D numpy array."
+    if not allow_inplace_reshuffle and (
+        pvalue_stats_bootstrap > 0 or (not results_is_dict and results.ndim < 3)
+    ):
+        results = deepcopy(results)
+    if not results_is_dict and results.ndim < 3:
+        results = np.expand_dims(results, axis=0)
+    assert results_is_dict or results.ndim == 3, "results must be a dict or 3D numpy array."
+
+    if pvalue_stats_bootstrap > 0:
+        show_progress = isinstance(show_progress_each, int) and show_progress_each > 0
 
     compStats_args, render_args = _splitCompareStats_and_renderArgs(
         kwCompareStats_and_renderArgs, console
     )
-    resd, alt_delimiter = resultsToDict(results, bm_names, alt_delimiter)
+
+    if results_is_dict:
+        resd = results
+    else:
+        resd, alt_delimiter, bm_idxs = resultsToDict(results, bm_names, alt_delimiter)
 
     def _applyMetrics(r: dict) -> dict:
         return {
@@ -586,24 +629,41 @@ def showBench(
             for bm_name, res in r.items()
         }
 
+    rng = None
+    if pvalue_stats_bootstrap > 0 and start_with_reshuffled:
+        pvalue_stats_bootstrap -= 1
+        if pvalue_stats_bootstrap <= 0 and console is not None:
+            console.warning(
+                "pvalue_stats_bootstrap is 0, so no bootstrapping will be done even though start_with_reshuffled is True."
+            )
+
+        rng = np.random.default_rng(pvalue_stats_bootstrap_seed)
+        for res in resd.values():
+            rng.shuffle(res.reshape(-1))
+
     sg = _applyMetrics(resd)
     sr = compareStats(sg, None, alt_delimiter=alt_delimiter, **compStats_args)
 
     if pvalue_stats_bootstrap > 0:
-        rng = np.random.default_rng(pvalue_stats_bootstrap_seed)
-        reps, iters = results.shape[1:]
+        if rng is None:
+            rng = np.random.default_rng(pvalue_stats_bootstrap_seed)
 
         if show_progress:
-            progress = Progress(transient=True)
+            progress = Progress(
+                transient=True, console=console if same_console_for_progress else None
+            )
             task = progress.add_task("P-value bootstrapping", total=pvalue_stats_bootstrap)
             progress.start()
 
+        bs_compStats_args = compStats_args.copy()
+        bs_compStats_args["store_sets"] = False
+        bs_compStats_args["debug_log"] = False
+
         for i in range(pvalue_stats_bootstrap):
-            for _, res in resd.items():
+            for res in resd.values():
                 rng.shuffle(res.reshape(-1))
-                assert res.shape == (reps, iters)
             sg = _applyMetrics(resd)
-            cs = compareStats(sg, None, alt_delimiter=alt_delimiter, **compStats_args)
+            cs = compareStats(sg, None, alt_delimiter=alt_delimiter, **bs_compStats_args)
             sr.updatePvalStats(cs)
 
             if show_progress and i % show_progress_each == 0:
@@ -612,6 +672,11 @@ def showBench(
         if show_progress:
             progress.stop()
 
+    if not results_is_dict:
+        sr.setComparisonIndices({
+            bm_name: (bm_idxs[bm0], bm_idxs[bm1]) for bm_name, (bm0, bm1) in sr.comparisons.items()
+        })
+
     if render_report:
         renderComparisonResults(sr, console=console, **render_args)
     return sr
@@ -731,32 +796,32 @@ def makeArgumentParser(parser=None, *, allow_exports=True):
         "--iters",
         type=int,
         default=_g_bench_defaults["iters"],
-        help=f"Iterations per repetition (default: {_g_bench_defaults['iters']})",
+        help="Iterations per repetition (default: %(default)s)",
     )
     parser.add_argument(
         "--reps",
         type=int,
         default=_g_bench_defaults["reps"],
-        help=f"Number of repetitions (default: {_g_bench_defaults['reps']})",
+        help="Number of repetitions (default: %(default)s)",
     )
     parser.add_argument(
         "--warmup",
         type=int,
         default=_g_bench_defaults["warmup"],
-        help=f"Number of warmup iterations (default: {_g_bench_defaults['warmup']})",
+        help="Number of warmup iterations (default: %(default)s)",
     )
     parser.add_argument(
         "--batch_functions",
         action=argparse.BooleanOptionalAction,
         default=_g_bench_defaults["batch_functions"],
         help="If true: outer loop - benchmarks, inner loop - iterations. Otherwise reversed: "
-        f"outer loop - iterations, inner loop - benchmarks. (default: {_g_bench_defaults['batch_functions']})",
+        "outer loop - iterations, inner loop - benchmarks. (default: %(default)s)",
     )
     parser.add_argument(
         "--randomize_iterations",
         action=argparse.BooleanOptionalAction,
         default=_g_bench_defaults["randomize_iterations"],
-        help=f"Randomly shuffle benchmarks order (default: {_g_bench_defaults['randomize_iterations']})",
+        help="Randomly shuffle benchmarks order (default: %(default)s)",
     )
 
     parser.add_argument(
@@ -764,7 +829,15 @@ def makeArgumentParser(parser=None, *, allow_exports=True):
         type=int,
         default=_g_showBench_defaults["pvalue_stats_bootstrap"],
         help="Number of iterations for pvalue statistics bootstrapping (default: "
-        f"{_g_showBench_defaults['pvalue_stats_bootstrap']}). Set to 0 to disable.",
+        "%(default)s). Set to 0 to disable.",
+    )
+
+    parser.add_argument(
+        "--start_with_reshuffled",
+        action=argparse.BooleanOptionalAction,
+        default=_g_showBench_defaults["start_with_reshuffled"],
+        help="If true, start even the first pvalue statistics bootstrapping iteration with "
+        "reshuffled results (default: %(default)s).",
     )
 
     if allow_exports:

{benchstats-3.3.2 → benchstats-3.4.0}/src/benchstats/render.py

@@ -132,7 +132,7 @@ def renderComparisonResults(
     dark_theme: bool = True,
     title: None | bool | str = True,  # None, False - disables title, str - customizes it
     style_overrides: dict = None,  # overrides for kDefaultStyles
-    main_metrics: Iterable[str] = None,
+    main_metrics: str | Iterable[str] | None = None,
     show_sample_sizes: bool = False,
     sample_stats=None,  # or iterable with predefined values: float%, or from kPossibleStatNames.keys()
     expect_same: bool = False,  # if true, show stats from assumption h0 is true
@@ -183,17 +183,6 @@ def renderComparisonResults(
             )
     assert title is None or isinstance(title, str)
 
-    if show_pvalue_stats:
-        if not comp_res.pval_stats_available:
-            show_pvalue_stats = False
-            console.debug("P-value statistics are not available, disabling show_pvalue_stats")
-        else:
-            pvs_iters = None
-            pvs_styles = [  # indexed by is_main
-                [_getFmt(f"metric_{fld}_diff" if i < 2 else f"metric_{fld}_same") for i in range(3)]
-                for fld in ("scnd", "main")
-            ]
-
     delim_space = "\n" if multiline else " "
 
     _column_descr = (
@@ -203,6 +192,8 @@ def renderComparisonResults(
     )
 
     metrics = comp_res.getMetrics()
+    if isinstance(main_metrics, str):
+        main_metrics = (main_metrics,)
     if main_metrics is None or len(main_metrics) < 1:
         main_metrics = [metrics[0]]
     else:
@@ -211,6 +202,17 @@ def renderComparisonResults(
     scnd_metrics = [m for m in metrics if m not in main_metrics]
     iter_metrics = [*main_metrics, *scnd_metrics]
 
+    if show_pvalue_stats:
+        if not comp_res.pval_stats_available:
+            show_pvalue_stats = False
+            console.debug("P-value statistics are not available, disabling show_pvalue_stats")
+        else:
+            pvs_iters = None
+            pvs_styles = [  # indexed by is_main
+                [_getFmt(f"metric_{fld}_diff" if i < 2 else f"metric_{fld}_same") for i in range(3)]
+                for fld in ("scnd", "main")
+            ]
+
     metric_unit_keys = [f"metric_{m}_unit" for m in iter_metrics]
 
     assert isinstance(metric_precision, int)

{benchstats-3.3.2 → benchstats-3.4.0/src/benchstats.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchstats
-Version: 3.3.2
+Version: 3.4.0
 Summary: Statistical Testing for Benchmark Results Comparison
 Author-email: Aleksei Rechinskii <5dpea9nhd@mozmail.com>
 License-Expression: MIT

{benchstats-3.3.2 → benchstats-3.4.0}/tests/test_compare.py

@@ -9,7 +9,7 @@ kAlpha = bsc.kDefaultAlpha
 
 class TestCompareStatsResult(unittest.TestCase):
     def test_updatePvalStats(self):
-        defargs = (kMethods[0], kAlpha, True)
+        defargs = (kMethods[0], kAlpha, True,{})
         cs0 = bsc.CompareStatsResult(
             {
                 "b1": {
@@ -109,4 +109,7 @@ class TestComparisonMethods(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    import sys
+    import pytest
+
+    sys.exit(pytest.main(sys.argv))

{benchstats-3.3.2 → benchstats-3.4.0}/tests/test_qbench.py

@@ -1,8 +1,6 @@
-import sys
 from typing import Iterable
 import unittest
 import numpy as np
-import pytest
 import time
 import traceback
 
@@ -232,7 +230,7 @@ class TestQBench(unittest.TestCase):
         check_reset_counters(7 * 2, 7 * 2, 7 * 2, 0, 7 + 14)
 
     def test_showBench_with_delimiter(self):
-        def_prms = {"alt_delimiter": "|", "render_report": False}
+        def_prms = {"alt_delimiter": "|", "render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -246,13 +244,24 @@ class TestQBench(unittest.TestCase):
             axis=0,
         )
         bm_names = ("alg0|A", "alg0|B", "alg1|A", "alg1|B", "alg2|A", "alg2|B")
-        sr = qb.showBench(results, bm_names=bm_names, **def_prms)
-        assert "<" == sr.results["alg0 | A vs B"]["mean"].result
-        assert ">" == sr.results["alg1 | A vs B"]["mean"].result
-        assert "~" == sr.results["alg2 | A vs B"]["mean"].result
+        cr = qb.showBench(results, bm_names=bm_names, **def_prms)
+        assert "<" == cr.results["alg0 | A vs B"]["mean"].result
+        assert ">" == cr.results["alg1 | A vs B"]["mean"].result
+        assert "~" == cr.results["alg2 | A vs B"]["mean"].result
+        assert cr.at_least_one_differs
+        assert cr.comparisons == {
+            "alg0 | A vs B": ("alg0|A", "alg0|B"),
+            "alg1 | A vs B": ("alg1|A", "alg1|B"),
+            "alg2 | A vs B": ("alg2|A", "alg2|B"),
+        }
+        assert cr.comparison_indices == {
+            "alg0 | A vs B": (0, 1),
+            "alg1 | A vs B": (2, 3),
+            "alg2 | A vs B": (4, 5),
+        }
 
     def test_showBench_one_name(self):
-        def_prms = {"render_report": False}
+        def_prms = {"render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -262,13 +271,24 @@ class TestQBench(unittest.TestCase):
             ),
             axis=0,
         )
-        sr = qb.showBench(results, bm_names="code", **def_prms)
-        assert "<" == sr.results["code | 0 vs 1"]["mean"].result
-        assert ">" == sr.results["code | 1 vs 2"]["mean"].result
-        assert ">" == sr.results["code | 0 vs 2"]["mean"].result
+        cr = qb.showBench(results, bm_names="code", **def_prms)
+        assert "<" == cr.results["code | 0 vs 1"]["mean"].result
+        assert ">" == cr.results["code | 1 vs 2"]["mean"].result
+        assert ">" == cr.results["code | 0 vs 2"]["mean"].result
+        assert cr.at_least_one_differs
+        assert cr.comparisons == {
+            "code | 0 vs 1": ("code|0", "code|1"),
+            "code | 1 vs 2": ("code|1", "code|2"),
+            "code | 0 vs 2": ("code|0", "code|2"),
+        }
+        assert cr.comparison_indices == {
+            "code | 0 vs 1": (0, 1),
+            "code | 1 vs 2": (1, 2),
+            "code | 0 vs 2": (0, 2),
+        }
 
     def test_showBench_tuple(self):
-        def_prms = {"render_report": False}
+        def_prms = {"render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -282,10 +302,21 @@ class TestQBench(unittest.TestCase):
             axis=0,
         )
         bm_names = ("alg0", "alg1", "alg2")
-        sr = qb.showBench(results, bm_names=bm_names, **def_prms)
-        assert "<" == sr.results["alg0 | 0 vs 1"]["mean"].result
-        assert ">" == sr.results["alg1 | 0 vs 1"]["mean"].result
-        assert "~" == sr.results["alg2 | 0 vs 1"]["mean"].result
+        cr = qb.showBench(results, bm_names=bm_names, **def_prms)
+        assert "<" == cr.results["alg0 | 0 vs 1"]["mean"].result
+        assert ">" == cr.results["alg1 | 0 vs 1"]["mean"].result
+        assert "~" == cr.results["alg2 | 0 vs 1"]["mean"].result
+        assert cr.at_least_one_differs
+        assert cr.comparisons == {
+            "alg0 | 0 vs 1": ("alg0|0", "alg0|1"),
+            "alg1 | 0 vs 1": ("alg1|0", "alg1|1"),
+            "alg2 | 0 vs 1": ("alg2|0", "alg2|1"),
+        }
+        assert cr.comparison_indices == {
+            "alg0 | 0 vs 1": (0, 1),
+            "alg1 | 0 vs 1": (2, 3),
+            "alg2 | 0 vs 1": (4, 5),
+        }
 
     def test_showBench_pvalue_stats_bootstrap(self):
         r = np.array([[[1, 3, 3], [1, 3, 3]], [[1, 2, 2], [1, 2, 2]]])
@@ -295,6 +326,8 @@ class TestQBench(unittest.TestCase):
             alpha=0.499,
             render_report=False,
             console=None,
+            show_progress_each=0,
+            start_with_reshuffled=False,
         )
         # seed is chosen to modify the first bootstrap result and produce the same second result
 
@@ -306,12 +339,20 @@ class TestQBench(unittest.TestCase):
             assert len(pvs["~"]) == same_len
             assert all(pvs["~"][i] == 1.0 for i in range(same_len))
 
+        cr = qb.showBench(r, pvalue_stats_bootstrap=0, **cmnargs)
+        assert not cr.at_least_one_differs
+
         cr = qb.showBench(r, pvalue_stats_bootstrap=1, **cmnargs)
+        assert cr.at_least_one_differs
         test_pvs(next(iter(next(iter(cr.pval_stats.values())).values())), 1)
 
         cr = qb.showBench(r, pvalue_stats_bootstrap=2, **cmnargs)
+        assert cr.at_least_one_differs
         test_pvs(next(iter(next(iter(cr.pval_stats.values())).values())), 2)
 
 
 if __name__ == "__main__":
+    import sys
+    import pytest
+
     sys.exit(pytest.main(sys.argv))