evalsuite-python 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

evalsuite/benchmarks.py

@@ -17,8 +17,10 @@ whose reference library is not installed is timed for EvalSuite only.
 from __future__ import annotations
 
 import contextlib
+import importlib
 import json
 import platform
+import re
 import time
 import tracemalloc
 import warnings
@@ -371,6 +373,28 @@ def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, own if own is not None else refs.get(name)) for name, ref, es_fn, own in cases]
 
 
+_GROUPS = (
+    ("Classification and regression", ("binary", "10 classes", "regression")),
+    ("Clinical, calibration and statistics", ("clinical", "calibration", "decision", "statistics", "multiple")),
+    ("Segmentation and object detection", ("segmentation", "detection")),
+)
+MATCH_TOLERANCE = 1e-9
+
+
+def benchmark_group(case: str) -> str:
+    """The suite a benchmark case belongs to, by its name."""
+    lowered = case.lower()
+    for title, prefixes in _GROUPS:
+        if lowered.startswith(prefixes):
+            return title
+    return "Other"
+
+
+def _geomean(values: Sequence[float]) -> Optional[float]:
+    vals = [v for v in values if v is not None and v > 0]
+    return float(np.exp(np.mean(np.log(vals)))) if vals else None
+
+
 @dataclass(frozen=True, eq=False)
 class BenchmarkResult:
     """Rows of timings (milliseconds), peak memory (MiB) and agreement, plus the environment they ran in."""
@@ -382,6 +406,60 @@ class BenchmarkResult:
         object.__setattr__(self, "rows", tuple(MappingProxyType(dict(r)) for r in self.rows))
         object.__setattr__(self, "environment", MappingProxyType(dict(self.environment)))
 
+    def sorted_rows(self) -> list[Any]:
+        """Rows in alphabetical order of case, then by size."""
+        return sorted(self.rows, key=lambda r: (r["case"].lower(), r["n"]))
+
+    def overall(self) -> list[dict[str, Any]]:
+        """One summary row per suite and one for everything: cases, how many agree with the reference
+        (max |difference| <= 1e-9), and the geometric-mean, smallest and largest speed-up."""
+
+        def summarise(name: str, rows: Sequence[Any]) -> dict[str, Any]:
+            compared = [r for r in rows if r["speedup"] is not None]
+            speedups = [r["speedup"] for r in compared]
+            return {
+                "group": name,
+                "cases": len({re.sub(r"\s*\(\d+ images?\)", "", r["case"]) for r in rows}),
+                "rows": len(rows),
+                "compared": len(compared),
+                "matching": sum(
+                    r["max_abs_diff"] is not None and r["max_abs_diff"] <= MATCH_TOLERANCE for r in rows
+                ),
+                "faster": sum(sp > 1 for sp in speedups),
+                "geomean_speedup": _geomean(speedups),
+                "min_speedup": min(speedups) if speedups else None,
+                "max_speedup": max(speedups) if speedups else None,
+            }
+
+        groups = sorted({benchmark_group(r["case"]) for r in self.rows})
+        out = [summarise(g, [r for r in self.rows if benchmark_group(r["case"]) == g]) for g in groups]
+        out.append(summarise("Overall", self.rows))
+        return out
+
+    _OVERALL_TITLES = ("Suite", "Cases", "Rows", "Match reference", "Faster", "Geo-mean speed-up", "Range")
+
+    def _overall_cells(self) -> list[list[str]]:
+        def x(v: Optional[float]) -> str:
+            return "–" if v is None else f"{v:.2f}×"
+
+        return [
+            [
+                o["group"],
+                str(o["cases"]),
+                str(o["rows"]),
+                f"{o['matching']}/{o['compared']}",
+                f"{o['faster']}/{o['compared']}",
+                x(o["geomean_speedup"]),
+                "–" if o["min_speedup"] is None else f"{x(o['min_speedup'])}–{x(o['max_speedup'])}",
+            ]
+            for o in self.overall()
+        ]
+
+    def overall_markdown(self) -> str:
+        """The overall summary as a Markdown table."""
+        lines = ["| " + " | ".join(self._OVERALL_TITLES) + " |", "| --- |" + " ---: |" * 6]
+        return "\n".join(lines + ["| " + " | ".join(c) + " |" for c in self._overall_cells()])
+
     def _cells(self, digits: int) -> list[list[str]]:
         def f(v: Any, d: int = digits) -> str:
             return "–" if v is None else f"{v:.{d}f}"
@@ -398,7 +476,7 @@ class BenchmarkResult:
                 f(r.get("reference_peak_mb"), 2),
                 "–" if r["max_abs_diff"] is None else f"{r['max_abs_diff']:.1e}",
             ]
-            for r in self.rows
+            for r in self.sorted_rows()
         ]
 
     _TITLES = (
@@ -430,7 +508,14 @@ class BenchmarkResult:
             + f" | {env['machine']} | fastest of {env['repeat']} runs"
         )
         note = "Speed-up > 1 means EvalSuite is faster. Max |difference| compares EvalSuite with the reference."
-        return "\n".join([head, "", line(self._TITLES), *(line(c) for c in cells), "", note])
+        ocells = self._overall_cells()
+        owidths = [max(len(t), *(len(c[i]) for c in ocells)) for i, t in enumerate(self._OVERALL_TITLES)]
+
+        def oline(c: Sequence[str]) -> str:
+            return "  ".join(x.ljust(owidths[i]) if i == 0 else x.rjust(owidths[i]) for i, x in enumerate(c))
+
+        overall = ["Overall", oline(self._OVERALL_TITLES), *(oline(c) for c in ocells), ""]
+        return "\n".join([head, "", *overall, line(self._TITLES), *(line(c) for c in cells), "", note])
 
     def __repr__(self) -> str:
         return self.summary()
@@ -438,7 +523,13 @@ class BenchmarkResult:
     def to_dict(self) -> dict[str, Any]:
         return cast(
             "dict[str, Any]",
-            _json_safe({"environment": dict(self.environment), "rows": [dict(r) for r in self.rows]}),
+            _json_safe(
+                {
+                    "environment": dict(self.environment),
+                    "overall": self.overall(),
+                    "rows": [dict(r) for r in self.rows],
+                }
+            ),
         )
 
     def to_json(self, *, indent: Optional[int] = 2) -> str:
@@ -514,7 +605,7 @@ def run_benchmarks(
     if compare_sklearn:
         for mod in versions:
             with contextlib.suppress(ImportError):
-                __import__(mod)
+                importlib.import_module(mod)  # fixed names only: sklearn, statsmodels, pycocotools
                 from importlib.metadata import version as _dist_version
 
                 versions[mod] = _dist_version("scikit-learn" if mod == "sklearn" else mod)

evalsuite/core/validation.py

@@ -45,9 +45,25 @@ def to_numpy(x: ArrayLike, name: str, *, allow_2d: bool = False) -> NDArray[Any]
         raise InputValidationError(f"{name} must be {expected}; received an array with shape {arr.shape}.")
     if arr.shape[0] == 0:
         raise InputValidationError(f"{name} is empty. Provide at least one observation.")
+    if (
+        arr.ndim == 1
+        and arr.dtype.kind in "US"
+        and isinstance(x, (list, tuple))
+        and not all(isinstance(v, (str, bytes)) for v in x)
+    ):
+        # NumPy silently turns [0, "a"] into ["0", "a"]; a mix of numbers and strings is almost always a bug.
+        raise InputValidationError(f"{name} mixes numbers and strings; use one label type throughout.")
     return arr
 
 
+def _label_kind(arr: NDArray[Any]) -> str:
+    if arr.dtype.kind in "US":
+        return "string"
+    if arr.dtype.kind in "biuf":
+        return "number"
+    return "other"
+
+
 def check_finite(arr: NDArray[Any], name: str) -> None:
     if np.issubdtype(arr.dtype, np.number) and not np.all(np.isfinite(arr)):
         n_nan = int(np.isnan(arr).sum())
@@ -126,6 +142,11 @@ def resolve_labels(
         if np.unique(lab).shape[0] != lab.shape[0]:
             raise InputValidationError("labels contains duplicates.")
         return lab
+    if y_pred is not None and {_label_kind(y_true), _label_kind(y_pred)} == {"string", "number"}:
+        raise InputValidationError(
+            f"y_true has {_label_kind(y_true)} labels but y_pred has {_label_kind(y_pred)} labels; "
+            "use the same label type in both (for example map class names to integers first)."
+        )
     present = y_true if y_pred is None else np.concatenate([y_true, y_pred])
     try:
         return unique_labels(present)

evalsuite/regression/metrics.py

@@ -202,7 +202,15 @@ def rmse(
 ) -> MetricResult:
     """Root mean squared error (square root taken per output, then averaged)."""
     inp = _inputs(y_true, y_pred, sample_weight)
-    return _finish(np.sqrt(inp.mean(inp.error**2)), inp, multioutput, "rmse", "RMSE")
+    with np.errstate(over="ignore"):
+        per_output = np.sqrt(inp.mean(inp.error**2))
+    if not np.all(np.isfinite(per_output)):
+        # Errors above ~1e154 overflow when squared although the RMSE itself is representable:
+        # rescale by the largest error and square again.
+        scale = np.abs(inp.error).max(axis=0)
+        scale = np.where(np.isfinite(scale) & (scale > 0), scale, 1.0)
+        per_output = scale * np.sqrt(inp.mean((inp.error / scale) ** 2))
+    return _finish(per_output, inp, multioutput, "rmse", "RMSE")
 
 
 def _r2_per_output(inp: _Inputs) -> FloatArray:

evalsuite/stats/_resolve.py

@@ -10,7 +10,7 @@ from typing import Any, Optional
 import numpy as np
 from numpy.typing import NDArray
 
-from ..core.exceptions import InputValidationError
+from ..core.exceptions import InputValidationError, StatisticalTestError
 from ..core.validation import target_type, to_numpy
 
 MetricFn = Callable[..., Any]
@@ -89,6 +89,11 @@ class MetricCall:
                 f"observations. Received {self.y_true.shape[0]} and {self.second.shape[0]}."
             )
         self.n = self.y_true.shape[0]
+        if self.n < 2:
+            raise StatisticalTestError(
+                f"Resampling needs at least 2 observations; got {self.n}. An interval or test from a single "
+                "observation would have zero width and no meaning."
+            )
         self.weight = None if sample_weight is None else to_numpy(sample_weight, "sample_weight")
         self.kwargs = dict(kwargs or {})
         params = inspect.signature(fn).parameters

evalsuite/version.py

@@ -1,3 +1,3 @@
 """Package version (single source of truth, read by the build backend)."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

evalsuite/vision/segmentation.py

@@ -84,9 +84,17 @@ _REF_HD = (
 
 # ---- input handling ------------------------------------------------------------------------------
 def _as_images(masks: Masks, name: str) -> list[NDArray[Any]]:
+    images = None
     if isinstance(masks, (list, tuple)) or (isinstance(masks, np.ndarray) and masks.dtype == object):
         images = [np.asarray(m) for m in masks]
-    else:
+        if (
+            images
+            and isinstance(masks, (list, tuple))
+            and all(m.ndim == 1 for m in images)
+            and not any(isinstance(m, np.ndarray) for m in masks)
+        ):
+            images = None  # a nested Python list of rows, e.g. [[1, 1], [0, 0]], is one 2-D mask
+    if images is None:
         arr = np.asarray(masks)
         if arr.ndim < 2:
             raise InputValidationError(

{evalsuite_python-0.3.0.dist-info → evalsuite_python-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalsuite-python
-Version: 0.3.0
+Version: 0.3.1
 Summary: A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation.
 Project-URL: Homepage, https://evalsuite-nine.vercel.app
 Project-URL: Documentation, https://evalsuite-nine.vercel.app/docs
@@ -49,7 +49,7 @@ Requires-Dist: pycocotools>=2.0.7; extra == 'dev'
 Requires-Dist: pytest-cov>=4; extra == 'dev'
 Requires-Dist: pytest>=7; extra == 'dev'
 Requires-Dist: ruff>=0.6; extra == 'dev'
-Requires-Dist: scikit-learn>=1.2; extra == 'dev'
+Requires-Dist: scikit-learn>=1.4; extra == 'dev'
 Requires-Dist: statsmodels>=0.13; extra == 'dev'
 Requires-Dist: twine; extra == 'dev'
 Provides-Extra: plot
@@ -70,7 +70,7 @@ Description-Content-Type: text/markdown
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.3.0).** Every item on the 0.1.0, 0.2.0 and 0.3.0 roadmaps is implemented and verified.
+> **Status: stable (0.3.1).** Every item on the 0.1.0, 0.2.0 and 0.3.0 roadmaps is implemented and verified.
 
 ## Installation
 
@@ -283,24 +283,28 @@ Input files can be CSV, TSV, Parquet or JSON. Output format follows `--format` o
 ## Performance
 
 Benchmarked against reference implementations on the same data (fastest of 5 runs; Python 3.12, NumPy 2.5,
-Linux x86_64). Every result agrees with the reference to floating-point rounding (largest difference
-1.4e-14).
-
-| Case | n | Reference | EvalSuite (ms) | Reference (ms) | Speed-up |
-| --- | ---: | --- | ---: | ---: | ---: |
-| 8 binary label metrics via `evaluate()` | 1,000,000 | scikit-learn | 30.2 | 1020.2 | **33.8×** |
-| macro F1, 10 classes | 1,000,000 | scikit-learn | 22.4 | 128.7 | **5.8×** |
-| ROC AUC, binary | 1,000,000 | scikit-learn | 173.2 | 300.6 | **1.7×** |
-| MAE, MSE, RMSE, R² via `evaluate()` | 1,000,000 | scikit-learn | 19.1 | 9.7 | 0.51× |
-| sensitivity, specificity, LR+, LR− | 1,000,000 | scikit-learn | 66.2 | 392.8 | **5.9×** |
-| calibration slope and intercept | 1,000,000 | statsmodels | 178.0 | 1014.8 | **5.7×** |
-| decision curve, 99 thresholds | 1,000,000 | NumPy loop | 155.2 | 174.3 | **1.1×** |
-| diagnostic report (7 CIs) | 1,000,000 | statsmodels | 16.6 | 4.6 | 0.28× |
-| Welch t-test | 1,000,000 | SciPy | 14.5 | 7.3 | 0.51× |
-| Hochberg correction | 1,000,000 | statsmodels | 75.4 | 81.9 | **1.1×** |
-| segmentation Dice and IoU per class | 1,000,000 px | scikit-learn | 40.2 | 282.0 | **7.0×** |
-| COCO detection evaluation (12 numbers) | 1,000 images | pycocotools | 967.4 | 980.6 | **1.0×** |
-| Hausdorff distance | 50 images | SciPy | 26.2 | 20.8 | 0.79× |
+Linux x86_64). **Overall: 45 of 45 rows agree with the reference** (largest
+difference 2.8e-14), 32 are faster, and the geometric-mean speed-up across all
+15 cases and three sizes is **2.22×**. At 1,000,000 samples (pixels for
+segmentation; 1,000 images for detection and 50 for Hausdorff), in alphabetical order:
+
+| Case | Reference | EvalSuite (ms) | Reference (ms) | Speed-up |
+| --- | --- | ---: | ---: | ---: |
+| 10 classes: macro F1 | scikit-learn | 29.3 | 130.8 | **4.5×** |
+| binary: 8 label metrics via evaluate() | scikit-learn | 39.5 | 1084.1 | **27.4×** |
+| binary: ROC AUC | scikit-learn | 195.0 | 348.3 | **1.8×** |
+| calibration: slope and intercept | statsmodels | 193.0 | 1068.0 | **5.5×** |
+| clinical: diagnostic report (7 CIs) | statsmodels | 23.5 | 4.7 | 0.20× |
+| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 69.1 | 414.0 | **6.0×** |
+| decision curve: 99 thresholds | NumPy loop | 156.3 | 181.6 | **1.2×** |
+| detection: COCO evaluation (1000 images) | pycocotools | 817.3 | 983.7 | **1.2×** |
+| multiple testing: Hochberg (n p-values) | statsmodels | 82.0 | 108.0 | **1.3×** |
+| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 23.1 | 10.8 | 0.47× |
+| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 43.0 | 290.0 | **6.8×** |
+| segmentation: Hausdorff distance (50 images) | SciPy | 27.0 | 20.2 | 0.75× |
+| statistics: Cramér's V (5×5 table) | SciPy | 0.5 | 0.4 | 0.88× |
+| statistics: Mann–Whitney U | SciPy | 349.7 | 350.2 | **1.0×** |
+| statistics: Welch t-test | SciPy | 14.5 | 7.2 | 0.49× |
 
 `evaluate()` validates inputs once and builds the confusion matrix once for all metrics, which is where most
 of the speed-up comes from. Hypothesis tests use SciPy underneath, so they match its speed at best; rows

{evalsuite_python-0.3.0.dist-info → evalsuite_python-0.3.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 evalsuite/__init__.py,sha256=5QLHlf6OX5UMBiPPpo9TrIhJYgCmqLXlzOgUkhPxxKw,5267
 evalsuite/__main__.py,sha256=Nq90_8VlUN1mif-AVgX-kZRL63bd94F5-F7F9UkVZD4,117
 evalsuite/api.py,sha256=TU49b4DolAm6dwU3ypJprOEzv5hJOsnRiXLwccyaGew,10170
-evalsuite/benchmarks.py,sha256=aKeukw4y89NjiU2XiA1K5Nwk34bAAcGspomZK1u3xQc,23122
+evalsuite/benchmarks.py,sha256=GzC6yNdxkih6Coe8hNqUSxTc6NLVfOP5MOK1K1TDuUo,26992
 evalsuite/calibration.py,sha256=sP7XO29Lk6wNp7qS5INNDtpz6SuVGGZhYUxSrTUxj94,14979
 evalsuite/plot.py,sha256=tfyuIF_8m-lrn87Eqq7GkQG5OWfD9yQNBRGKHCtYVLk,20016
 evalsuite/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalsuite/reporting.py,sha256=-LISv9vvEnltT26GtzLjVlxOCcAwmjv4-HyB8_1UF5o,9615
-evalsuite/version.py,sha256=LEC6quDhVmFCkRDuLm6La-9rNPb-dPGubQJbxTxof8g,98
+evalsuite/version.py,sha256=FZk8M-bQVGqb0YvmxRi1eu5Z11BEwSB4mexHdE1qj48,98
 evalsuite/classification/__init__.py,sha256=ErBIXbBGnhfW6v-1Cn5SegiERoyxNlmO1tEKcLM15mA,833
 evalsuite/classification/_common.py,sha256=3ANV33ix_mnOzvOAG8glyJO-iYVhhTGO0S3YG0ZddB8,4543
 evalsuite/classification/metrics.py,sha256=8-RPjzuFUKJYlXndrDv6kXlzRdx6aR2lHp7mzKsas2g,36969
@@ -22,11 +22,11 @@ evalsuite/core/export.py,sha256=6M_ktQzW3E7usOrwqv5f5zOEdoi48JD6CvgyRpDkGvM,4303
 evalsuite/core/registry.py,sha256=3l2_o2PlckuKY89HxfnxPLf-sDaCe_-McUcAcUmXusk,2884
 evalsuite/core/result.py,sha256=L_fQZRGRgVgzII2jZb_jpRBK2jy-CdO5u9s-6pUk9fE,14927
 evalsuite/core/types.py,sha256=r6bhUk7CYAcrlQzjmf-6xWQsNqO3EOipZD-e-2yAJTI,720
-evalsuite/core/validation.py,sha256=uCD2QbWQWejctdy8ZwO9oL7NfvpCSYr6ZFc6ZPw18Bk,9071
+evalsuite/core/validation.py,sha256=nWWyISLdW2TtzxUs3gT-Qx2uh6euDOlYMvbnCrFRYME,9975
 evalsuite/regression/__init__.py,sha256=t2XEHUw6e5tyWBHcEeiog2WWd-wwaxf4m5AIX8dJyng,571
-evalsuite/regression/metrics.py,sha256=jVdzmegdpBdyJNCSm1dFR1E-9VHDmQmTVtyrqyjk3KY,22459
+evalsuite/regression/metrics.py,sha256=5i3hmiFfZGfmqtFDaDOxGHRYCXzdt48-GNfgvVOX7CM,22914
 evalsuite/stats/__init__.py,sha256=mtnEqpvlvscQ3C6ceRr8xLKoSHPQWsWB24pEVlvp6fs,1187
-evalsuite/stats/_resolve.py,sha256=SBuKR-jfqhBTZWYcUnD0RsrBryzGqn6QBYwBw48zyTc,4759
+evalsuite/stats/_resolve.py,sha256=QhvI6A2Ui2gbS1qY5AAofz47hYO_DSoyQtKlRA2wZ7M,5036
 evalsuite/stats/compare.py,sha256=2c66hQNgsxl2DBEOeebl-SRxQ83ZkO_71lAfvf9jZEU,17998
 evalsuite/stats/effect.py,sha256=Esw1Di0BSabiErcsxoUinB9V0-ByEQ5fTLzAptf-dBo,6943
 evalsuite/stats/hypothesis.py,sha256=Sd3vcPUYVs_QwZ0IKKgQEFqtFcFan_CTy0EHjHEgIF4,12773
@@ -35,9 +35,9 @@ evalsuite/stats/paired.py,sha256=YK3AFndChKeCEpK512u56VtRlKvhlGdkFLHHYe77zmc,824
 evalsuite/stats/results.py,sha256=nqXHUiW8l0fQJkbMiunU6Gz-2be1NApZ6Fif2B1K_Yw,4407
 evalsuite/vision/__init__.py,sha256=9gpOBTPveM_srE6BM-SgCssdQurya2LPHhoMLGOh2A0,991
 evalsuite/vision/detection.py,sha256=eUnt5iFUVLEoUl1RB51q-vfr5juDCD3YH8ttLFrqxFg,27781
-evalsuite/vision/segmentation.py,sha256=s1dWXtwWd4-hprUOmARQCBhStLQROtc480Z9QNUCgPE,38311
-evalsuite_python-0.3.0.dist-info/METADATA,sha256=p6HSajrShVBi_YkL86kd4NYFwQskW2QRp6ZNzoOgZ8s,17364
-evalsuite_python-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-evalsuite_python-0.3.0.dist-info/entry_points.txt,sha256=YYIwisn4L4sQt3ntzbM0_Z4yZE2KCu3GJ6KfWL1kIwc,54
-evalsuite_python-0.3.0.dist-info/licenses/LICENSE,sha256=cRRQ6Kk8w2qCqtDSSUJQGy-0rqjmjJG09cZ6iIiMvdc,1095
-evalsuite_python-0.3.0.dist-info/RECORD,,
+evalsuite/vision/segmentation.py,sha256=7h0AirBU0G6XMWzdA7MQNO52hXz8oboNAX9s55y6SVQ,38647
+evalsuite_python-0.3.1.dist-info/METADATA,sha256=75uFR4hsGdPZy40k1PwTyyubHsmSBSDOSAJK1Kyho1g,17640
+evalsuite_python-0.3.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+evalsuite_python-0.3.1.dist-info/entry_points.txt,sha256=YYIwisn4L4sQt3ntzbM0_Z4yZE2KCu3GJ6KfWL1kIwc,54
+evalsuite_python-0.3.1.dist-info/licenses/LICENSE,sha256=cRRQ6Kk8w2qCqtDSSUJQGy-0rqjmjJG09cZ6iIiMvdc,1095
+evalsuite_python-0.3.1.dist-info/RECORD,,