benchstats 3.3.2__tar.gz → 3.3.3__tar.gz

{benchstats-3.3.2 → benchstats-3.3.3}/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 3.3.3
+- breaking change, though unlikely important for anyone: due to T-test also being affected by
+bad date, renamed `compareStats()` parameter `brunnermunzel_workaround` to `edge_cases_workaround`
+and extended its effect on t-test.
+- Fix `_at_least_one_differs` flag handling in `CompareStatsResult::updatePvalStats()`.
+
 # 3.3.1-.2
 - Changed the order of default metrics of `qbench.showBench()` to make `mean` the leading metric, as
 it's typically the most correct one.

{benchstats-3.3.2/src/benchstats.egg-info → benchstats-3.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchstats
-Version: 3.3.2
+Version: 3.3.3
 Summary: Statistical Testing for Benchmark Results Comparison
 Author-email: Aleksei Rechinskii <5dpea9nhd@mozmail.com>
 License-Expression: MIT

{benchstats-3.3.2 → benchstats-3.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "benchstats"
-version = "3.3.2"
+version = "3.3.3"
 description = "Statistical Testing for Benchmark Results Comparison"
 readme = "README.md"
 authors = [{ name = "Aleksei Rechinskii", email = "5dpea9nhd@mozmail.com" }]

{benchstats-3.3.2 → benchstats-3.3.3}/src/benchstats/compare.py

@@ -162,6 +162,7 @@ class CompareStatsResult:
         assert isinstance(other, CompareStatsResult)
         assert other.method == self.method and other.alpha == self.alpha
         assert self._results.keys() == other._results.keys()
+        self._at_least_one_differs = self._at_least_one_differs or other._at_least_one_differs
         for bm, omdict in other._results.items():
             mdict = self._results[bm]
             assert mdict.keys() == omdict.keys()
@@ -249,7 +250,7 @@ def compareStats(
     main_metrics: None | list[str] | tuple[str] = None,
     debug_log: None | bool | LoggingConsole = True,
     store_sets: bool = False,
-    brunnermunzel_workaround: None | bool = None,
+    edge_cases_workaround: None | bool = None,
 ) -> CompareStatsResult:
     """Perform comparison for statistical significance of benchmark results using a chosen
     statistical method.
@@ -281,21 +282,22 @@ def compareStats(
     `store_sets` is a bool, True will make store whole corresponding dataset into a BmCompResult
     instead of a mean of the set.
 
-    `brunnermunzel_workaround`: Formally speaking, Brunner Munzel test method is "underspecified"
+    `edge_cases_workaround`: Formally speaking, Brunner Munzel test method is "underspecified"
     in a sense that by design it doesn't handle a case of non-intersecting sample sets, or sets
     consisting of the same single element (https://aakinshin.net/posts/brunner-munzel-corner-case/).
     scipy.stats.brunnermunzel() implementation at least in versions 1.10-1.15.2 directly follows the
     words of the original paper and doesn't introduce handling for that cases, which causes warnings
     and nans to appear in results. IMO from an engineering point of view this is more like just a
-    bug (https://github.com/scipy/scipy/issues/22664). `brunnermunzel_workaround` is a flag to
-    handle it: value of None handles the bug only for method == 'brunnermunzel', a bool otherwise
-    controls whether to handle the bug irrespective of the method used (this might give different
-    pvalues for other tests, but this shouldn't affect inference results unless alpha threshold
-    value is too wild). It should be noted that for small sample sizes (under 10 elements in any of
+    bug (https://github.com/scipy/scipy/issues/22664). The same issue also applies to Student's
+    t-test. `edge_cases_workaround` is a flag to handle it: value of None handles the bug only for
+    affected methods, a bool otherwise controls whether to handle the bug irrespective of the method
+    used (this might give different pvalues for other tests, but this shouldn't affect inference
+    results unless alpha threshold value is too wild).
+    It should be noted that for small sample sizes (under 10 elements in any of
     sample sets) this most certainly leads to a wrong p-value result that underestimate a
     probability of error. However, with 10, or preferably more, sample set sizes, this should be
     negligible.
-    While brunnermunzel_workaround allows to disable the workaround, I doubt it has any practical
+    While edge_cases_workaround allows to disable the workaround, I doubt it has any practical
     value in the context of benchmark results comparison.
 
     Returns an instance of CompareStatsResult class
@@ -306,7 +308,7 @@ def compareStats(
     assert isinstance(sg0, dict) and (isinstance(sg1, dict) or (with_alternatives and sg1 is None))
     assert isinstance(method, str) and method in kMethods, "unsupported method"
     assert isinstance(alpha, kAllowedFpTypes) and 0 < alpha and alpha < 0.5
-    assert brunnermunzel_workaround is None or isinstance(brunnermunzel_workaround, bool)
+    assert edge_cases_workaround is None or isinstance(edge_cases_workaround, bool)
     assert main_metrics is None or isinstance(main_metrics, (list, tuple))
 
     if debug_log is None or (isinstance(debug_log, bool) and not debug_log):
@@ -317,8 +319,8 @@ def compareStats(
         logger = debug_log
         debug_log = True
 
-    if brunnermunzel_workaround is None:
-        brunnermunzel_workaround = method == "brunnermunzel"
+    if edge_cases_workaround is None:
+        edge_cases_workaround = (method in ("brunnermunzel", "ttest_ind"))
 
     def warn(*args, **kwargs):
         if debug_log:
@@ -330,11 +332,11 @@ def compareStats(
 
     def computePValues(s0, s1):
         """Computes and return pvalues of s0 being stochastically less or greater than s1"""
-        if brunnermunzel_workaround:
+        if edge_cases_workaround:
             # test for edge cases that brunnermunzel can't handle properly
             mn0, mx0, mn1, mx1 = np.min(s0), np.max(s0), np.min(s1), np.max(s1)
             assert np.all(np.isfinite([mn0, mx0, mn1, mx1]))  # sanity check, more asserts below
-            all_eq = np.all([mn0 == mx0, mn0 == mn1, mn0 == mx1])
+            all_eq = all([mn0 == mx0, mn0 == mn1, mn0 == mx1])
             all_less, all_greater = mx0 < mn1, mn0 > mx1
 
             if all_eq:
@@ -348,7 +350,7 @@ def compareStats(
         res_greater = stat_func(s0, s1, alternative="greater")
 
         less_pvalue, greater_pvalue = res_less.pvalue, res_greater.pvalue
-        if brunnermunzel_workaround:
+        if edge_cases_workaround:
             assert np.all(np.isfinite([less_pvalue, greater_pvalue]))
 
         return less_pvalue, greater_pvalue

{benchstats-3.3.2 → benchstats-3.3.3/src/benchstats.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchstats
-Version: 3.3.2
+Version: 3.3.3
 Summary: Statistical Testing for Benchmark Results Comparison
 Author-email: Aleksei Rechinskii <5dpea9nhd@mozmail.com>
 License-Expression: MIT

{benchstats-3.3.2 → benchstats-3.3.3}/tests/test_compare.py

@@ -109,4 +109,7 @@ class TestComparisonMethods(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    import sys
+    import pytest
+
+    sys.exit(pytest.main(sys.argv))

{benchstats-3.3.2 → benchstats-3.3.3}/tests/test_qbench.py

@@ -1,8 +1,6 @@
-import sys
 from typing import Iterable
 import unittest
 import numpy as np
-import pytest
 import time
 import traceback
 
@@ -232,7 +230,7 @@ class TestQBench(unittest.TestCase):
         check_reset_counters(7 * 2, 7 * 2, 7 * 2, 0, 7 + 14)
 
     def test_showBench_with_delimiter(self):
-        def_prms = {"alt_delimiter": "|", "render_report": False}
+        def_prms = {"alt_delimiter": "|", "render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -252,7 +250,7 @@ class TestQBench(unittest.TestCase):
         assert "~" == sr.results["alg2 | A vs B"]["mean"].result
 
     def test_showBench_one_name(self):
-        def_prms = {"render_report": False}
+        def_prms = {"render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -268,7 +266,7 @@ class TestQBench(unittest.TestCase):
         assert ">" == sr.results["code | 0 vs 2"]["mean"].result
 
     def test_showBench_tuple(self):
-        def_prms = {"render_report": False}
+        def_prms = {"render_report": False, "show_progress_each": 0}
         nreps = 10
         results = np.stack(
             (
@@ -295,6 +293,7 @@ class TestQBench(unittest.TestCase):
             alpha=0.499,
             render_report=False,
             console=None,
+            show_progress_each=0,
         )
         # seed is chosen to modify the first bootstrap result and produce the same second result
 
@@ -306,12 +305,20 @@ class TestQBench(unittest.TestCase):
             assert len(pvs["~"]) == same_len
             assert all(pvs["~"][i] == 1.0 for i in range(same_len))
 
+        cr = qb.showBench(r, pvalue_stats_bootstrap=0, **cmnargs)
+        assert not cr.at_least_one_differs
+
         cr = qb.showBench(r, pvalue_stats_bootstrap=1, **cmnargs)
+        assert cr.at_least_one_differs
         test_pvs(next(iter(next(iter(cr.pval_stats.values())).values())), 1)
 
         cr = qb.showBench(r, pvalue_stats_bootstrap=2, **cmnargs)
+        assert cr.at_least_one_differs
         test_pvs(next(iter(next(iter(cr.pval_stats.values())).values())), 2)
 
 
 if __name__ == "__main__":
+    import sys
+    import pytest
+
     sys.exit(pytest.main(sys.argv))