evalsuite-python 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

evalsuite/_bench_metrics.py

@@ -903,7 +903,7 @@ def _llm(c: _Cases, n: int, rng: np.random.Generator) -> None:
 
     cider_ref: Optional[Callable[[], float]] = None
     with contextlib.suppress(ImportError):
-        from pycocoevalcap.cider.cider import Cider  # type: ignore[import-untyped]
+        from pycocoevalcap.cider.cider import Cider
 
         def _cider() -> float:
             res = {i: [p] for i, p in enumerate(preds)}

evalsuite/_lazy.py

@@ -0,0 +1,32 @@
+"""Deferred imports for heavy optional-at-import-time dependencies (SciPy submodules).
+
+``import evalsuite`` should not pay for SciPy until a function that needs it runs.
+"""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+from typing import Any
+
+
+class LazyModule(ModuleType):
+    """Module proxy that imports the real module on first attribute access."""
+
+    def __init__(self, name: str) -> None:
+        super().__init__(name)
+        self.__dict__["_mod"] = None
+
+    def _load(self) -> ModuleType:
+        mod = self.__dict__["_mod"]
+        if mod is None:
+            mod = importlib.import_module(self.__name__)
+            self.__dict__["_mod"] = mod
+        return mod  # type: ignore[no-any-return]
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self._load(), attr)
+
+
+def lazy_module(name: str) -> Any:
+    return LazyModule(name)

evalsuite/api.py

@@ -63,7 +63,8 @@ def _classification_registry(
 
 def _hamming(ctx: ClassificationContext) -> MetricResult:
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         wrong = (ctx.y_true != ctx.y_pred).mean(axis=1)
     else:
         wrong = (ctx.true_idx != ctx.pred_idx).astype(float)

evalsuite/benchmarks.py

@@ -524,6 +524,117 @@ def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, own if own is not None else refs.get(name)) for name, ref, es_fn, own in cases]
 
 
+def _llmsys_cases(n: int, rng: np.random.Generator) -> list[Case]:
+    """v0.5.0: LLM systems workloads (code quality, selective prediction, agent tool calls, shift, latency)."""
+    import json as _json
+
+    import evalsuite as es
+
+    from ._bench_metrics import _CODE_SAMPLES
+
+    m = max(10, n // 100)
+    progs = [str(rng.choice(np.array(_CODE_SAMPLES))) for _ in range(max(4, m // 10))]
+    refs_c = progs
+    preds_c = [p.replace("a", "x").replace("s", "t") for p in progs]
+    conf = rng.uniform(0, 1, n)
+    correct = rng.random(n) < conf
+    tools = {
+        t: {"type": "object", "properties": {"q": {"type": "string"}, "k": {"type": "integer"}}, "required": ["q"]}
+        for t in ("search", "calc", "email")
+    }
+    trajs = [
+        [
+            {
+                "name": str(rng.choice(["search", "calc", "email", "bad"])),
+                "arguments": {"q": "x"} if rng.random() > 0.1 else {"k": 1},
+            }
+            for _ in range(int(rng.integers(1, 6)))
+        ]
+        for _ in range(m)
+    ]
+    src, tgt = rng.normal(0.8, 0.1, n), rng.normal(0.75, 0.12, n)
+    lat = rng.lognormal(0, 0.5, n)
+
+    def es_shift() -> list[float]:
+        r = es.distribution_shift_drop(src, tgt)
+        return [r.params["ci_low"], r.params["ci_high"]]
+
+    def es_lat() -> list[float]:
+        r = es.latency_percentiles(lat)
+        return [r.params["p50"], r.params["p95"], r.params["p99"]]
+
+    cases: list[Case] = [
+        (
+            f"systems: maintainability index ({len(progs)} programs)",
+            "radon",
+            lambda: [float(es.code_complexity(progs).params["maintainability_index"])],
+            None,
+        ),
+        (
+            f"systems: CodeBLEU n-gram terms ({len(progs)} programs)",
+            "codebleu",
+            lambda: [es.codebleu(refs_c, preds_c).params[k] for k in ("ngram_match", "weighted_ngram_match")],
+            None,
+        ),
+        (
+            "systems: confidence AUROC (n answers)",
+            "scikit-learn",
+            lambda: [float(es.confidence_accuracy_correlation(correct, conf))],
+            None,
+        ),
+        (
+            f"systems: invalid tool calls vs JSON Schema ({m} tasks)",
+            "jsonschema",
+            lambda: [float(es.invalid_tool_call_rate(trajs, tools))],
+            None,
+        ),
+        ("systems: distribution-shift drop, Welch CI (n scores)", "SciPy", es_shift, None),
+        (
+            "systems: latency p50 / p95 / p99 (n requests)",
+            "NumPy",
+            es_lat,
+            lambda: np.percentile(lat, [50, 95, 99]).tolist(),
+        ),
+    ]
+    refs: dict[str, Callable[[], Any]] = {}
+    with contextlib.suppress(ImportError):
+        from radon.metrics import mi_visit
+
+        refs[cases[0][0]] = lambda: [float(np.mean([mi_visit(p, True) for p in progs]))]
+    with contextlib.suppress(Exception):
+        from codebleu import calc_codebleu
+
+        def cb() -> list[float]:
+            r = calc_codebleu([[x] for x in refs_c], preds_c, lang="python")
+            return [r["ngram_match_score"], r["weighted_ngram_match_score"]]
+
+        cb()
+        refs[cases[1][0]] = cb
+    with contextlib.suppress(ImportError):
+        import sklearn.metrics as skm
+
+        refs[cases[2][0]] = lambda: [float(skm.roc_auc_score(correct, conf))]
+    with contextlib.suppress(ImportError):
+        import jsonschema
+
+        def js_ref() -> list[float]:
+            flags = []
+            for traj in trajs:
+                for c in traj:
+                    sch = tools.get(str(c["name"]))
+                    flags.append(sch is None or not jsonschema.Draft202012Validator(sch).is_valid(c["arguments"]))
+            return [float(np.mean(flags))]
+
+        refs[cases[3][0]] = js_ref
+    from scipy import stats as _st
+
+    res = _st.ttest_ind(src[:3], tgt[:3], equal_var=False)
+    if hasattr(res, "confidence_interval"):
+        refs[cases[4][0]] = lambda: list(_st.ttest_ind(src, tgt, equal_var=False).confidence_interval(0.95))
+    del _json
+    return [(name, ref, es_fn, own if own is not None else refs.get(name)) for name, ref, es_fn, own in cases]
+
+
 _GROUPS = (
     ("Classification and regression", ("binary", "10 classes", "regression", "classification")),
     ("Clinical, calibration and statistics", ("clinical", "calibration", "decision", "statistics", "multiple")),
@@ -532,6 +643,7 @@ _GROUPS = (
     (
         "LLM systems",
         (
+            "systems:",
             "safety.",
             "robustness.",
             "uncertainty.",
@@ -765,8 +877,8 @@ def run_benchmarks(
 
     if repeat < 1:
         raise ValueError("repeat must be at least 1.")
-    if suite not in ("all", "core", "clinical", "vision", "llm", "metrics"):
-        raise ValueError("suite must be 'all', 'core', 'clinical', 'vision', 'llm' or 'metrics'.")
+    if suite not in ("all", "core", "clinical", "vision", "llm", "llmsys", "metrics"):
+        raise ValueError("suite must be 'all', 'core', 'clinical', 'vision', 'llm', 'llmsys' or 'metrics'.")
     rng = np.random.default_rng(random_state)
     rows: list[dict[str, Any]] = []
     versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None, "pycocotools": None}
@@ -782,7 +894,8 @@ def run_benchmarks(
         "clinical": [_clinical_cases],
         "vision": [_vision_cases],
         "llm": [_llm_cases],
-        "all": [_core_cases, _clinical_cases, _vision_cases, _llm_cases],
+        "llmsys": [_llmsys_cases],
+        "all": [_core_cases, _clinical_cases, _vision_cases, _llm_cases, _llmsys_cases],
         "metrics": [metric_cases],
     }[suite]
     for n in sizes:

evalsuite/calibration.py

@@ -14,7 +14,13 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from ._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from .classification.metrics import (
     _binary_target,

evalsuite/classification/metrics.py

@@ -109,7 +109,8 @@ def accuracy(y_true: ArrayLike, y_pred: ArrayLike, *, sample_weight: Optional[Ar
 
 def _accuracy(ctx: ClassificationContext) -> MetricResult:
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101 - ensured by context for label metrics
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         correct = (ctx.y_true == ctx.y_pred).all(axis=1)
     else:
         correct = ctx.true_idx == ctx.pred_idx
@@ -491,7 +492,8 @@ def hamming_loss(
     """Hamming loss."""
     ctx = _ctx(y_true, y_pred, sample_weight=sample_weight)
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         wrong = (ctx.y_true != ctx.y_pred).mean(axis=1)
     else:
         wrong = (ctx.true_idx != ctx.pred_idx).astype(float)

evalsuite/cli/main.py

@@ -559,11 +559,11 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
     p.add_argument(
         "--suite",
-        choices=("all", "core", "clinical", "vision", "llm", "metrics"),
+        choices=("all", "core", "clinical", "vision", "llm", "llmsys", "metrics"),
         default="all",
         help="core: classification/regression; clinical: clinical, calibration, tests; vision: segmentation, "
-        "detection; llm: text, retrieval, agreement, structured output; metrics: one row per metric and "
-        "statistics function (default all)",
+        "detection; llm: text, retrieval, agreement, structured output; llmsys: v0.5.0 LLM systems; "
+        "metrics: one row per metric and statistics function (default all)",
     )
     p.add_argument("--no-sklearn", action="store_true", help="time EvalSuite only (skip reference libraries)")
     p.add_argument("--seed", type=int, default=0)

evalsuite/clinical/report.py

@@ -10,7 +10,13 @@ from typing import TYPE_CHECKING, Any, Optional, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special
+
+if TYPE_CHECKING:
+    from scipy import special
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
 
 from ..core.exceptions import InputValidationError
 from ..core.export import PathLike, csv_text, html_document, html_table, save_as

evalsuite/llmsys/_common.py

@@ -31,6 +31,14 @@ def seq(x: Any, name: str) -> list[Any]:
 
 def bools(x: Any, name: str) -> np.ndarray:
     """A 1-D boolean array from booleans or 0/1 values (anything else is rejected, never coerced)."""
+    if isinstance(x, np.ndarray) and x.ndim == 1 and x.size and x.dtype.kind in "biuf":
+        if x.dtype.kind == "b":
+            return x.astype(bool, copy=False)
+        bad = ~np.isin(x, (0, 1))
+        if bad.any():
+            i = int(np.flatnonzero(bad)[0])
+            raise InputValidationError(f"{name}[{i}] must be a boolean (or 0/1); got {x[i]!r}.")
+        return x.astype(bool)
     items = seq(x, name)
     out = np.empty(len(items), dtype=bool)
     for i, v in enumerate(items):
@@ -45,7 +53,7 @@ def bools(x: Any, name: str) -> np.ndarray:
 
 def floats(x: Any, name: str, *, lo: Optional[float] = None, hi: Optional[float] = None) -> np.ndarray:
     """A finite 1-D float array, optionally bounded."""
-    items = seq(x, name)
+    items: Any = x if isinstance(x, np.ndarray) and x.size and x.dtype.kind in "biuf" else seq(x, name)
     try:
         a = np.asarray(items, dtype=np.float64)
     except (TypeError, ValueError) as exc:

evalsuite/llmsys/code.py

@@ -40,6 +40,10 @@ __all__ = [
 ]
 
 _C = "code"
+
+# Errors CPython raises for source it cannot parse: syntax errors, null bytes (ValueError), and input nested
+# too deeply for the parser (MemoryError "parser stack overflowed") or for the compiler (RecursionError).
+_PARSE_ERRORS = (SyntaxError, ValueError, MemoryError, RecursionError)
 _REF_HUMANEVAL = (
     "Chen M, Tworek J, Jun H, et al. Evaluating large language models trained on code. arXiv:2107.03374. 2021."
 )
@@ -141,7 +145,7 @@ def syntax_validity_rate(
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore", SyntaxWarning)
                     compile(src, "<generated>", "exec", dont_inherit=True)  # parses only; nothing runs
-            except (SyntaxError, ValueError):
+            except _PARSE_ERRORS:
                 return False
             return True
 
@@ -562,12 +566,16 @@ def code_complexity(code: Any) -> MetricResult:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", SyntaxWarning)
                 tree = ast.parse(src)
+            visitor = _Complexity.run(tree)
+            v = _halstead_volume(tree)
         except SyntaxError as exc:
             raise InputValidationError(f"code[{i}] is not valid Python: {exc.msg} (line {exc.lineno}).") from exc
-        visitor = _Complexity.run(tree)
+        except _PARSE_ERRORS as exc:
+            raise InputValidationError(
+                f"code[{i}] is too deeply nested to analyse ({type(exc).__name__})."
+            ) from exc
         blocks = visitor.blocks
         funcs += [c for _, c in blocks]
-        v = _halstead_volume(tree)
         g = visitor.total
         lloc, sloc, comments = _raw(src)
         c_pct = comments / sloc * 100 if sloc else 0.0
@@ -720,38 +728,38 @@ def _parse(src: str) -> Optional[ast.AST]:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", SyntaxWarning)
             return ast.parse(_strip(src))
-    except SyntaxError:
+    except _PARSE_ERRORS:
         return None
 
 
-def _sexp(node: ast.AST, memo: dict[int, str]) -> str:
-    key = id(node)
-    if key in memo:
-        return memo[key]
-    kids = [
-        c
-        for c in ast.iter_child_nodes(node)
-        if not isinstance(c, (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop))
-    ]
-    tag = type(node).__name__
-    if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.AugAssign)):
-        tag += ":" + type(node.op).__name__
-    s = f"({tag}{''.join(' ' + _sexp(c, memo) for c in kids)})"
-    memo[key] = s
-    return s
-
-
-def _subtrees(tree: ast.AST) -> list[str]:
-    memo: dict[int, str] = {}
-    out = []
-    for node in ast.walk(tree):
-        kids = [
-            c
-            for c in ast.iter_child_nodes(node)
-            if not isinstance(c, (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop))
-        ]
+_SKIP_NODES = (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop)
+
+
+def _kids(node: ast.AST) -> list[ast.AST]:
+    return [c for c in ast.iter_child_nodes(node) if not isinstance(c, _SKIP_NODES)]
+
+
+def _subtrees(tree: ast.AST, table: dict[tuple[Any, ...], int]) -> list[int]:
+    """Every subtree with children (and the root), as ids that are equal exactly when the subtrees are
+    structurally equal. Built bottom-up without recursion: ``table`` interns (node type, child ids), so the
+    cost is linear in the tree size (comparing serialised subtrees would be quadratic)."""
+    ids: dict[int, int] = {}
+    out: list[int] = []
+    stack: list[tuple[ast.AST, bool]] = [(tree, False)]
+    while stack:
+        node, ready = stack.pop()
+        kids = _kids(node)
+        if not ready:
+            stack.append((node, True))
+            stack.extend((k, False) for k in kids)
+            continue
+        tag = type(node).__name__
+        if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.AugAssign)):
+            tag += ":" + type(node.op).__name__
+        sid = table.setdefault((tag, tuple(ids[id(k)] for k in kids)), len(table))
+        ids[id(node)] = sid
         if kids or node is tree:
-            out.append(_sexp(node, memo))
+            out.append(sid)
     return out
 
 
@@ -878,15 +886,16 @@ def codebleu(
     ngram = _corpus_bleu(t_refs, t_hyps, None)
     weighted = _corpus_bleu(t_refs, t_hyps, kw)
     match = total = df_match = df_total = 0
+    table: dict[tuple[Any, ...], int] = {}
     for rs, p in zip(refs, preds):
         cand = _parse(p)
-        cand_sub = set(_subtrees(cand)) if cand is not None else set()
+        cand_sub = set(_subtrees(cand, table)) if cand is not None else set()
         cand_flow = _normalize_flow(_dataflow(cand)) if cand is not None else []
         for r in rs:
             ref = _parse(r)
             if ref is None:
                 continue
-            ref_sub = _subtrees(ref)
+            ref_sub = _subtrees(ref, table)
             match += sum(s in cand_sub for s in ref_sub)
             total += len(ref_sub)
             ref_flow = _normalize_flow(_dataflow(ref))

evalsuite/llmsys/safety.py

@@ -362,7 +362,7 @@ def _luhn(digits: str) -> bool:
 
 
 PII_PATTERNS: dict[str, str] = {
-    "email": r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
+    "email": r"(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
     "phone": r"(?<!\d[\s.-])(?<!\w)(?:\+?\d{1,3}[\s.-]?)?(?:\(\d{2,4}\)[\s.-]?)?\d{3,4}[\s.-]\d{3,4}"
     r"(?:[\s.-]\d{2,4})?(?![\s.-]?\d)(?!\w)",
     "credit_card": r"(?<!\d)\d(?:[ -]?\d){12,18}(?!\d)",

evalsuite/llmsys/uncertainty.py

@@ -215,8 +215,12 @@ def confidence_accuracy_correlation(correct: Any, confidence: Any) -> MetricResu
     if np.ptp(p) == 0:
         pb = sp = float("nan")
     else:
-        pb = float(stats.pearsonr(c.astype(float), p)[0])
-        sp = float(stats.spearmanr(c.astype(float), p)[0])
+        cf = c.astype(float)
+        pb = float(np.corrcoef(cf, p)[0, 1])
+        # Spearman = Pearson correlation of average ranks; the ranks of p are already computed
+        n0 = c.size - n1
+        rc = np.where(c, n0 + (n1 + 1) / 2, (n0 + 1) / 2)
+        sp = float(np.corrcoef(rc, ranks)[0, 1])
     return MetricResult(
         "confidence_accuracy_correlation",
         "Confidence–accuracy AUROC",

evalsuite/stats/effect.py

@@ -2,11 +2,17 @@
 
 from __future__ import annotations
 
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special
+
+if TYPE_CHECKING:
+    from scipy import special
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
 
 from ..core.exceptions import InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

evalsuite/stats/hypothesis.py

@@ -8,11 +8,17 @@ implementations are the reference; EvalSuite adds input validation, effect sizes
 from __future__ import annotations
 
 import math
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from .._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

evalsuite/stats/intervals.py

@@ -3,11 +3,18 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any, Literal, Optional
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special, stats
+
+if TYPE_CHECKING:
+    from scipy import special, stats
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import EvalSuiteError, InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

evalsuite/stats/paired.py

@@ -2,10 +2,17 @@
 
 from __future__ import annotations
 
-from typing import Any, Literal, Optional
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 import numpy as np
-from scipy import special, stats
+
+if TYPE_CHECKING:
+    from scipy import special, stats
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError
 from ..core.types import ArrayLike

evalsuite/text/judge.py

@@ -9,10 +9,16 @@ from __future__ import annotations
 
 import math
 from collections.abc import Sequence
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import numpy as np
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from .._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError
 from ..core.registry import register

evalsuite/text/structured.py

@@ -712,7 +712,11 @@ def constraint_satisfaction_rate(checks: Any) -> MetricResult:
 def format_compliance(
     outputs: Any, pattern: str, *, flags: int = re.S, average: Optional[str] = "mean"
 ) -> MetricResult:
-    """``re.fullmatch(pattern, output.strip())`` per output."""
+    """``re.fullmatch(pattern, output.strip())`` per output.
+
+    The pattern is compiled with Python's backtracking ``re`` engine; patterns with nested quantifiers such as
+    ``(a+)+$`` can take exponential time on some outputs, so only pass patterns you trust.
+    """
     outs = _outputs(outputs)
     try:
         rx = re.compile(pattern, flags)

evalsuite/version.py

@@ -1,3 +1,3 @@
 """Package version (single source of truth, read by the build backend)."""
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"

evalsuite/vision/segmentation.py

@@ -27,7 +27,13 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import ndimage
+
+if TYPE_CHECKING:
+    from scipy import ndimage
+else:
+    from .._lazy import lazy_module
+
+    ndimage = lazy_module("scipy.ndimage")
 
 from ..core.exceptions import InputValidationError, UndefinedMetricWarning
 from ..core.export import PathLike, csv_text, html_document, html_table, save_as

{evalsuite_python-0.5.0.dist-info → evalsuite_python-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalsuite-python
-Version: 0.5.0
+Version: 0.5.2
 Summary: A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation.
 Project-URL: Homepage, https://evalsuite-nine.vercel.app
 Project-URL: Documentation, https://evalsuite-nine.vercel.app/docs
@@ -83,7 +83,7 @@ Description-Content-Type: text/markdown
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.5.0).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
+> **Status: stable (0.5.2).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
 > **LLM systems arrive in 0.5.0**: safety, robustness, calibration and uncertainty, agents and tool use,
 > multilingual, code generation, long context and summarization, and inference efficiency and cost.
 
@@ -339,41 +339,47 @@ Input files can be CSV, TSV, Parquet or JSON. Output format follows `--format` o
 
 ## Performance
 
-Benchmarked against reference implementations on the same data (fastest of 5 runs; Python 3.12, NumPy 2.5,
-Linux x86_64). **Overall: 63 of 63 rows agree with the reference** (largest
-difference 1.8e-14), 42 are faster, and the geometric-mean speed-up across all
-21 cases and three sizes is **2.12×**. At 1,000,000 samples (pixels for
+Benchmarked against reference implementations on the same data (fastest of 5 runs; EvalSuite 0.5.1, Python 3.13,
+NumPy 2.5, Linux x86_64). **Overall: 81 of 81 rows agree with the reference** (largest
+difference 2.5e-14), 56 are faster, and the geometric-mean speed-up across all
+27 cases and three sizes is **1.89×**. At 1,000,000 samples (pixels for
 segmentation; 1,000 images for detection and 50 for Hausdorff; 10,000 examples for the LLM cases), in
 alphabetical order:
 
 | Case | Reference | EvalSuite (ms) | Reference (ms) | Speed-up |
 | --- | --- | ---: | ---: | ---: |
-| 10 classes: macro F1 | scikit-learn | 23.8 | 135.5 | **5.7×** |
-| agreement: Krippendorff's alpha, interval (4 raters × 10000 items) | krippendorff | 3.5 | 3.4 | 0.96× |
-| binary: 8 label metrics via evaluate() | scikit-learn | 28.5 | 1023.8 | **36.0×** |
-| binary: ROC AUC | scikit-learn | 187.4 | 327.3 | **1.7×** |
-| calibration: slope and intercept | statsmodels | 205.7 | 1173.7 | **5.7×** |
-| clinical: diagnostic report (7 CIs) | statsmodels | 19.6 | 4.9 | 0.25× |
-| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 75.2 | 411.5 | **5.5×** |
-| decision curve: 99 thresholds | NumPy loop | 165.0 | 201.9 | **1.2×** |
-| detection: COCO evaluation (1000 images) | pycocotools | 838.7 | 848.6 | **1.0×** |
-| multiple testing: Hochberg (n p-values) | statsmodels | 72.1 | 79.0 | **1.1×** |
-| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 22.9 | 10.6 | 0.47× |
-| retrieval: MRR, MAP@20, NDCG@10 (10000 queries) | ranx | 170.4 | 861.0 | **5.1×** |
-| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 34.3 | 256.0 | **7.5×** |
-| segmentation: Hausdorff distance (50 images) | SciPy | 24.2 | 19.8 | 0.82× |
-| statistics: Cramér's V (5×5 table) | SciPy | 0.2 | 0.2 | **1.0×** |
-| statistics: Mann–Whitney U | SciPy | 353.5 | 340.6 | 0.96× |
-| statistics: Welch t-test | SciPy | 16.1 | 8.3 | 0.52× |
-| structured: JSON Schema compliance (10000 documents) | jsonschema | 144.6 | 330.5 | **2.3×** |
-| text: corpus BLEU and chrF (10000 sentences) | sacreBLEU | 2789.8 | 3051.8 | **1.1×** |
-| text: METEOR, exact and stem matches (10000 sentences) | NLTK | 488.1 | 626.7 | **1.3×** |
-| text: ROUGE-1, ROUGE-2, ROUGE-L (10000 sentences) | rouge-score | 1290.1 | 1095.6 | 0.85× |
+| 10 classes: macro F1 | scikit-learn | 41.5 | 137.6 | **3.3×** |
+| agreement: Krippendorff's alpha, interval (4 raters × 10000 items) | krippendorff | 4.5 | 4.5 | **1.0×** |
+| binary: 8 label metrics via evaluate() | scikit-learn | 49.4 | 1056.8 | **21.4×** |
+| binary: ROC AUC | scikit-learn | 200.3 | 338.5 | **1.7×** |
+| calibration: slope and intercept | statsmodels | 344.3 | 1541.7 | **4.5×** |
+| clinical: diagnostic report (7 CIs) | statsmodels | 33.5 | 9.6 | 0.29× |
+| clinical: sensitivity, specificity, LR+, LR− | scikit-learn | 137.9 | 438.1 | **3.2×** |
+| decision curve: 99 thresholds | NumPy loop | 172.1 | 236.2 | **1.4×** |
+| detection: COCO evaluation (1000 images) | pycocotools | 1147.9 | 1351.0 | **1.2×** |
+| multiple testing: Hochberg (n p-values) | statsmodels | 91.9 | 98.4 | **1.1×** |
+| regression: MAE, MSE, RMSE, R² via evaluate() | scikit-learn | 32.0 | 22.6 | 0.71× |
+| retrieval: MRR, MAP@20, NDCG@10 (10000 queries) | ranx | 248.6 | 897.0 | **3.6×** |
+| segmentation: Dice and IoU per class (n = pixels) | scikit-learn | 51.6 | 289.5 | **5.6×** |
+| segmentation: Hausdorff distance (50 images) | SciPy | 31.9 | 23.8 | 0.75× |
+| statistics: Cramér's V (5×5 table) | SciPy | 0.3 | 0.3 | 0.95× |
+| statistics: Mann–Whitney U | SciPy | 390.3 | 425.8 | **1.1×** |
+| statistics: Welch t-test | SciPy | 37.7 | 16.3 | 0.43× |
+| structured: JSON Schema compliance (10000 documents) | jsonschema | 159.4 | 371.7 | **2.3×** |
+| systems: CodeBLEU n-gram terms (1000 programs) | codebleu | 740.3 | 791.9 | **1.1×** |
+| systems: confidence AUROC (n answers) | scikit-learn | 194.4 | 327.7 | **1.7×** |
+| systems: distribution-shift drop, Welch CI (n scores) | SciPy | 13.3 | 16.9 | **1.3×** |
+| systems: invalid tool calls vs JSON Schema (10000 tasks) | jsonschema | 145.0 | 333.2 | **2.3×** |
+| systems: latency p50 / p95 / p99 (n requests) | NumPy | 41.0 | 19.7 | 0.48× |
+| systems: maintainability index (1000 programs) | radon | 138.3 | 168.7 | **1.2×** |
+| text: corpus BLEU and chrF (10000 sentences) | sacreBLEU | 2726.2 | 3121.0 | **1.1×** |
+| text: METEOR, exact and stem matches (10000 sentences) | NLTK | 507.4 | 590.3 | **1.2×** |
+| text: ROUGE-1, ROUGE-2, ROUGE-L (10000 sentences) | rouge-score | 1147.3 | 1010.7 | 0.88× |
 
 **Every metric is benchmarked too** (`evalsuite benchmark --suite metrics`): all 225 registered metrics and
 statistics functions at 10,000 and 100,000 samples. 86 are compared with a reference library, 118 with an
 independent textbook formula and 21 (learned, judge-dependent or randomised) are timed alone; **all 204
-comparisons agree**, and against the libraries the geometric-mean speed-up is **2.65×** (137 of 172
+comparisons agree**, and against the libraries the geometric-mean speed-up is **2.67×** (139 of 172
 measurements faster). Per-metric table in BENCHMARKS.md.
 
 `evaluate()` validates inputs once and builds the confusion matrix once for all metrics, which is where most

{evalsuite_python-0.5.0.dist-info → evalsuite_python-0.5.2.dist-info}/RECORD

@@ -1,21 +1,22 @@
 evalsuite/__init__.py,sha256=nMwLJq_Ni8plZZmq1olG5HwDlBppn0pW-RJyFBzPF7o,5630
 evalsuite/__main__.py,sha256=Nq90_8VlUN1mif-AVgX-kZRL63bd94F5-F7F9UkVZD4,117
-evalsuite/_bench_metrics.py,sha256=lSqirR5rKSzu0Qr4KM8FGT4AtjKfPEpFqI3k0_hKwQo,75824
-evalsuite/api.py,sha256=TU49b4DolAm6dwU3ypJprOEzv5hJOsnRiXLwccyaGew,10170
-evalsuite/benchmarks.py,sha256=zZ8dWCI_kUqFY17SQ9gcckvtlf01Nzf7vlQz3xwZELk,33648
-evalsuite/calibration.py,sha256=sP7XO29Lk6wNp7qS5INNDtpz6SuVGGZhYUxSrTUxj94,14979
+evalsuite/_bench_metrics.py,sha256=GLLa9OhCNmu6IyWkxC0AcNf7p-CTpoOwWUS99C1o2o8,75792
+evalsuite/_lazy.py,sha256=Ddq0uWuiyxtsT0jBtCSITQPvttiQ7znXAbjkMIOVefk,901
+evalsuite/api.py,sha256=DzWwmCgyEiqO2HvPrNg2RIDzqwap4e_UvbxA7PkJ3fE,10227
+evalsuite/benchmarks.py,sha256=TWXIhdYPbdDPanCOFL1lRCo3ppRYaDF58PB-o3ZZyTA,37846
+evalsuite/calibration.py,sha256=jhCPOmCFGEkUkcx-hSAIb-JCMq9oF6JZt4uDGvg5NaU,15083
 evalsuite/plot.py,sha256=7eOyzoRbg6Aad6OhtIQG3laIioQ8fs2K0YzodjG5rHA,26049
 evalsuite/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalsuite/reporting.py,sha256=-LISv9vvEnltT26GtzLjVlxOCcAwmjv4-HyB8_1UF5o,9615
-evalsuite/version.py,sha256=Eww8OsmDsBUNsEa1juWDR77joR6scCsY21vQJUxBtks,98
+evalsuite/version.py,sha256=XXivYAbOhDT-ji15jWK1TxpgAXnosxn_wRb7rCcTkJc,98
 evalsuite/classification/__init__.py,sha256=ErBIXbBGnhfW6v-1Cn5SegiERoyxNlmO1tEKcLM15mA,833
 evalsuite/classification/_common.py,sha256=3ANV33ix_mnOzvOAG8glyJO-iYVhhTGO0S3YG0ZddB8,4543
-evalsuite/classification/metrics.py,sha256=8-RPjzuFUKJYlXndrDv6kXlzRdx6aR2lHp7mzKsas2g,36969
+evalsuite/classification/metrics.py,sha256=h-DYF2wwdN8LA1bPbyIgkfuG4ovwBBgtGZFObEmiHzE,37044
 evalsuite/cli/__init__.py,sha256=1x48hORLcg_Is2X08A7CPBT6VDJ1QL94zaJP1HTFxUY,96
-evalsuite/cli/main.py,sha256=LWy3ThWiGUHgN-3HVwd25jqEEPpqGN6edVSdAqxJUHY,23956
+evalsuite/cli/main.py,sha256=ktvy_5D-KfYevKH5ZxO5f0aVFtNXbGeN8zker0tguT0,23994
 evalsuite/clinical/__init__.py,sha256=xaXLvNvTJsuP525fLb2FyVVNlopIrZczoPrwmEQIvzE,549
 evalsuite/clinical/metrics.py,sha256=Abz4rA9FfOIWn7MXAMnxAH_C3XICY9x-nPQhP1-79xU,13453
-evalsuite/clinical/report.py,sha256=6Pp0NDm24nccmRbicebUfUOyb37U-JtHPD48QeP-h6k,14887
+evalsuite/clinical/report.py,sha256=JaS9PKjUHkiKl_AURC1TwNiDc_V-6B2KNhV4kzHZhdE,14996
 evalsuite/core/__init__.py,sha256=oDHl5Dc2GoZf1tCtJueMQ2KcmOq7zhzMZT9XF4uVC6U,93
 evalsuite/core/context.py,sha256=gvRpH3b-S2oAgAGW9W0XWnBVbuydQP3UB9rgWQNM60g,7874
 evalsuite/core/exceptions.py,sha256=9eZI_Lmgwc9fjkSUY9s3fwjAb7KoqgAlcYetrHRhbBg,1503
@@ -25,30 +26,30 @@ evalsuite/core/result.py,sha256=L_fQZRGRgVgzII2jZb_jpRBK2jy-CdO5u9s-6pUk9fE,1492
 evalsuite/core/types.py,sha256=r6bhUk7CYAcrlQzjmf-6xWQsNqO3EOipZD-e-2yAJTI,720
 evalsuite/core/validation.py,sha256=nWWyISLdW2TtzxUs3gT-Qx2uh6euDOlYMvbnCrFRYME,9975
 evalsuite/llmsys/__init__.py,sha256=NA-kVMc3hdmBSW-ufPKrgfd2NHI15wCXJRn3ZhIVFsE,4653
-evalsuite/llmsys/_common.py,sha256=4V6Yjs7b3_QI8eC9OR3_uKnc53O7a8s0Vi-Yyn5Zdu8,4908
+evalsuite/llmsys/_common.py,sha256=zUJ4R_1fSW79HJ6QvZlSJ9-_VF0Xa7fglyq_fNFJB-U,5380
 evalsuite/llmsys/agents.py,sha256=giiHv3c179fFR6aFUp4VbBjzbdNotS941tZVG7W0UFE,15355
-evalsuite/llmsys/code.py,sha256=bJVpesBy9Vk_xkLBo7FoAVlDPpTRLG30BFeWIb2eJoE,36151
+evalsuite/llmsys/code.py,sha256=6Im0Km5qPHLFTF4S4dnN05OucFFNRixRchDSGHqtJaQ,37004
 evalsuite/llmsys/efficiency.py,sha256=Z6ZrcoT80D3OxNi-1iKHuVe5OQiDnPJthwHEDD84cO0,13884
 evalsuite/llmsys/longcontext.py,sha256=koX58FMot3ZwUTPPiv9tM8fGvBY2ijYg0npA8pG278A,15145
 evalsuite/llmsys/multilingual.py,sha256=IDdV5O5krDQdTximeJbYB3Kmh62qeSbLL--ElM7CGvc,13726
 evalsuite/llmsys/robustness.py,sha256=OPe84s1H7IEX5GcuRVcAp_UA29tx_Qq8WlA88khY2cg,20377
-evalsuite/llmsys/safety.py,sha256=r21haKLDzCU193KCYUYEBMu4cuhS5mO3lDywdQtYZno,22529
-evalsuite/llmsys/uncertainty.py,sha256=ZIcniUdmIAOQpNxpWzM6gL7ibplH4bftxIVE9PKbLsU,9183
+evalsuite/llmsys/safety.py,sha256=7Kcv9zJBhIWHS6GeAa0BedCncFOmsbIpnmUQCMvRMPU,22550
+evalsuite/llmsys/uncertainty.py,sha256=Brm_LT8qHb-EtcPe7YBXq7HYMSTG93OkT3Cnx2SYBts,9367
 evalsuite/regression/__init__.py,sha256=t2XEHUw6e5tyWBHcEeiog2WWd-wwaxf4m5AIX8dJyng,571
 evalsuite/regression/metrics.py,sha256=5i3hmiFfZGfmqtFDaDOxGHRYCXzdt48-GNfgvVOX7CM,22914
 evalsuite/stats/__init__.py,sha256=mtnEqpvlvscQ3C6ceRr8xLKoSHPQWsWB24pEVlvp6fs,1187
 evalsuite/stats/_resolve.py,sha256=oauINI97AairuOB_fvlH5VlkiQnW5aIlR-bQT_zzhQ8,6119
 evalsuite/stats/compare.py,sha256=TzBYHDbfHU1Uz2FtgXbMATTnmHuaDGUbrV9Eq4IphZY,18311
-evalsuite/stats/effect.py,sha256=Esw1Di0BSabiErcsxoUinB9V0-ByEQ5fTLzAptf-dBo,6943
-evalsuite/stats/hypothesis.py,sha256=Sd3vcPUYVs_QwZ0IKKgQEFqtFcFan_CTy0EHjHEgIF4,12773
-evalsuite/stats/intervals.py,sha256=w2bLuFt3p-TTN_ALH1uKPmDMLcTzBBoV2YvcP-ENAOk,13709
-evalsuite/stats/paired.py,sha256=YK3AFndChKeCEpK512u56VtRlKvhlGdkFLHHYe77zmc,8244
+evalsuite/stats/effect.py,sha256=PCzpj_Lf1CdBAJdL8DoL0gEsNVVcwk-gjmlRyrcGAPM,7067
+evalsuite/stats/hypothesis.py,sha256=n-8bUYWwur8ZElhDqRksA8MK09vLuwYd90Vzv1gI9hY,12893
+evalsuite/stats/intervals.py,sha256=0lW87qZsejY8r2xXw0agWasudOGq-rEU1v-9u8OuiGA,13872
+evalsuite/stats/paired.py,sha256=hF4qBAxdYOrKD8JnJJOa9A_Xn8xw08qpMkworXssUuU,8407
 evalsuite/stats/results.py,sha256=nqXHUiW8l0fQJkbMiunU6Gz-2be1NApZ6Fif2B1K_Yw,4407
 evalsuite/text/__init__.py,sha256=_r0Qi6X7cBcaYMAPRyC15CsTj9Tv8VcqMCtLiSJPogY,3632
 evalsuite/text/_common.py,sha256=NeJroFlIFu_1HXNHgnTVZEyaeVScX-o5C0Qbfe3INe8,3659
 evalsuite/text/factuality.py,sha256=Id8LnGjqvaVDYC5qMn4H0UJdjrwp1VrZ_zr_0jct8SM,20772
 evalsuite/text/generation.py,sha256=6eHWccspuQeaup8qP8Pv3k2nx6IejG9UECYHwDlWuHE,42572
-evalsuite/text/judge.py,sha256=AtKZtvrfxeheGAiFOt74IwamR6_XV67uPHOyimAv2jw,22542
+evalsuite/text/judge.py,sha256=ugqsaVih_n_h0jO5GrvIGHqq7XjqGnP7cKd4QbObEw0,22662
 evalsuite/text/qa.py,sha256=rU0ya8TzV5mp7UiWq7w9IqSN9B0tDv8xymSaRzzbzMw,4097
 evalsuite/text/rag.py,sha256=GF8GUp6Vnt9peBCIcdvFUapLUL36_NDApZisYG_rcqg,9641
 evalsuite/text/reasoning.py,sha256=ydaTgAY--Tz_zEXm-Q_jLxJjTLnPgRlv39OzXWXD9yA,8885
@@ -56,12 +57,12 @@ evalsuite/text/report.py,sha256=nUAgt24rcQsnPLiNTVn3PlzZO3Y7fW_ch_lSelgfEc8,6241
 evalsuite/text/retrieval.py,sha256=otQvOTbbR79kYtMum6o2HOT9UoDlSj0-JYLsFddkV1k,8969
 evalsuite/text/semantic.py,sha256=V1Wc4bvcli0_6ZlA58guTGA65zGoCFMthbPeCh_IDjc,21408
 evalsuite/text/spice.py,sha256=rdPCeilqKkXLMiSAE-rYOn0Zt2nKFum5Tf1oJdE6k-U,6847
-evalsuite/text/structured.py,sha256=mNpTGKJUWXewV-fiQ3TuCnLXjss9MeduTXpyCNbRivk,31773
+evalsuite/text/structured.py,sha256=x08yYf2rdlmiA_D_dI1L-fc9quHiBPMKgAGQbG3kJr8,31981
 evalsuite/vision/__init__.py,sha256=9gpOBTPveM_srE6BM-SgCssdQurya2LPHhoMLGOh2A0,991
 evalsuite/vision/detection.py,sha256=eUnt5iFUVLEoUl1RB51q-vfr5juDCD3YH8ttLFrqxFg,27781
-evalsuite/vision/segmentation.py,sha256=7h0AirBU0G6XMWzdA7MQNO52hXz8oboNAX9s55y6SVQ,38647
-evalsuite_python-0.5.0.dist-info/METADATA,sha256=YKZIpS-l_rI2YePmzDH2LwRq5kK9tzQtfBm1JeVUE94,24620
-evalsuite_python-0.5.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-evalsuite_python-0.5.0.dist-info/entry_points.txt,sha256=YYIwisn4L4sQt3ntzbM0_Z4yZE2KCu3GJ6KfWL1kIwc,54
-evalsuite_python-0.5.0.dist-info/licenses/LICENSE,sha256=cRRQ6Kk8w2qCqtDSSUJQGy-0rqjmjJG09cZ6iIiMvdc,1095
-evalsuite_python-0.5.0.dist-info/RECORD,,
+evalsuite/vision/segmentation.py,sha256=k8uGUsIrJByVygguFsiY3CsrLNhL5MZJ4xDEy6jcgjE,38756
+evalsuite_python-0.5.2.dist-info/METADATA,sha256=5bTTH1ciXStnesbV5utt8-XpEPBikZn7kQB190hyXKM,25181
+evalsuite_python-0.5.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+evalsuite_python-0.5.2.dist-info/entry_points.txt,sha256=YYIwisn4L4sQt3ntzbM0_Z4yZE2KCu3GJ6KfWL1kIwc,54
+evalsuite_python-0.5.2.dist-info/licenses/LICENSE,sha256=cRRQ6Kk8w2qCqtDSSUJQGy-0rqjmjJG09cZ6iIiMvdc,1095
+evalsuite_python-0.5.2.dist-info/RECORD,,

{evalsuite_python-0.5.0.dist-info → evalsuite_python-0.5.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.32.4
+Generator: hatchling 1.33.0
 Root-Is-Purelib: true
 Tag: py3-none-any