evalsuite-python 0.5.1__tar.gz → 0.5.2__tar.gz

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/CHANGELOG.md

@@ -6,6 +6,25 @@ All notable changes to this project are documented here. The format follows
 
 ## [Unreleased]
 
+## [0.5.2] - 2026-11-07
+
+Security and performance hardening. No API changes.
+
+### Security
+- The built-in PII e-mail detector no longer backtracks quadratically: `detect_pii` / `pii_leakage_rate`
+  on 60 kB of adversarial text went from ~6 s to ~10 ms.
+- `codebleu`, `code_complexity` and `syntax_validity_rate` handle source nested too deeply for CPython's
+  parser (MemoryError / RecursionError) instead of crashing: invalid for syntax validity, an
+  `InputValidationError` for complexity, no syntax match for CodeBLEU.
+- Internal `assert` type checks replaced by explicit errors, so behaviour is the same under `python -O`.
+- CI and release workflows pin every action to a commit SHA; Dependabot keeps them and the dependencies current.
+- Documented that user-supplied regular expressions (`format_compliance`) run on a backtracking engine.
+
+### Performance
+- `import evalsuite` no longer imports SciPy; it is loaded on first use (import time ~0.6 s → ~0.07 s).
+- CodeBLEU syntax match uses interned subtree ids instead of serialised strings: linear in the size of the
+  program rather than quadratic.
+
 ## [0.5.1] - 2026-10-10
 
 ### Added

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalsuite-python
-Version: 0.5.1
+Version: 0.5.2
 Summary: A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation.
 Project-URL: Homepage, https://evalsuite-nine.vercel.app
 Project-URL: Documentation, https://evalsuite-nine.vercel.app/docs
@@ -83,7 +83,7 @@ Description-Content-Type: text/markdown
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.5.1).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
+> **Status: stable (0.5.2).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
 > **LLM systems arrive in 0.5.0**: safety, robustness, calibration and uncertainty, agents and tool use,
 > multilingual, code generation, long context and summarization, and inference efficiency and cost.
 

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/README.md

@@ -10,7 +10,7 @@
 EvalSuite brings classification, regression, clinical, statistical, segmentation and object-detection
 evaluation into one consistent, validated, documented framework.
 
-> **Status: stable (0.5.1).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
+> **Status: stable (0.5.2).** Every item on the 0.1.0–0.5.0 roadmaps is implemented and verified, including SPICE.
 > **LLM systems arrive in 0.5.0**: safety, robustness, calibration and uncertainty, agents and tool use,
 > multilingual, code generation, long context and summarization, and inference efficiency and cost.
 

evalsuite_python-0.5.2/src/evalsuite/_lazy.py

@@ -0,0 +1,32 @@
+"""Deferred imports for heavy optional-at-import-time dependencies (SciPy submodules).
+
+``import evalsuite`` should not pay for SciPy until a function that needs it runs.
+"""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+from typing import Any
+
+
+class LazyModule(ModuleType):
+    """Module proxy that imports the real module on first attribute access."""
+
+    def __init__(self, name: str) -> None:
+        super().__init__(name)
+        self.__dict__["_mod"] = None
+
+    def _load(self) -> ModuleType:
+        mod = self.__dict__["_mod"]
+        if mod is None:
+            mod = importlib.import_module(self.__name__)
+            self.__dict__["_mod"] = mod
+        return mod  # type: ignore[no-any-return]
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self._load(), attr)
+
+
+def lazy_module(name: str) -> Any:
+    return LazyModule(name)

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/api.py

@@ -63,7 +63,8 @@ def _classification_registry(
 
 def _hamming(ctx: ClassificationContext) -> MetricResult:
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         wrong = (ctx.y_true != ctx.y_pred).mean(axis=1)
     else:
         wrong = (ctx.true_idx != ctx.pred_idx).astype(float)

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/calibration.py

@@ -14,7 +14,13 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from ._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from .classification.metrics import (
     _binary_target,

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/classification/metrics.py

@@ -109,7 +109,8 @@ def accuracy(y_true: ArrayLike, y_pred: ArrayLike, *, sample_weight: Optional[Ar
 
 def _accuracy(ctx: ClassificationContext) -> MetricResult:
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101 - ensured by context for label metrics
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         correct = (ctx.y_true == ctx.y_pred).all(axis=1)
     else:
         correct = ctx.true_idx == ctx.pred_idx
@@ -491,7 +492,8 @@ def hamming_loss(
     """Hamming loss."""
     ctx = _ctx(y_true, y_pred, sample_weight=sample_weight)
     if ctx.target_type == "multilabel":
-        assert ctx.y_pred is not None  # noqa: S101
+        if ctx.y_pred is None:
+            raise InputValidationError("y_pred is required for this metric.")
         wrong = (ctx.y_true != ctx.y_pred).mean(axis=1)
     else:
         wrong = (ctx.true_idx != ctx.pred_idx).astype(float)

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/clinical/report.py

@@ -10,7 +10,13 @@ from typing import TYPE_CHECKING, Any, Optional, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special
+
+if TYPE_CHECKING:
+    from scipy import special
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
 
 from ..core.exceptions import InputValidationError
 from ..core.export import PathLike, csv_text, html_document, html_table, save_as

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/llmsys/code.py

@@ -40,6 +40,10 @@ __all__ = [
 ]
 
 _C = "code"
+
+# Errors CPython raises for source it cannot parse: syntax errors, null bytes (ValueError), and input nested
+# too deeply for the parser (MemoryError "parser stack overflowed") or for the compiler (RecursionError).
+_PARSE_ERRORS = (SyntaxError, ValueError, MemoryError, RecursionError)
 _REF_HUMANEVAL = (
     "Chen M, Tworek J, Jun H, et al. Evaluating large language models trained on code. arXiv:2107.03374. 2021."
 )
@@ -141,7 +145,7 @@ def syntax_validity_rate(
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore", SyntaxWarning)
                     compile(src, "<generated>", "exec", dont_inherit=True)  # parses only; nothing runs
-            except (SyntaxError, ValueError):
+            except _PARSE_ERRORS:
                 return False
             return True
 
@@ -562,12 +566,16 @@ def code_complexity(code: Any) -> MetricResult:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", SyntaxWarning)
                 tree = ast.parse(src)
+            visitor = _Complexity.run(tree)
+            v = _halstead_volume(tree)
         except SyntaxError as exc:
             raise InputValidationError(f"code[{i}] is not valid Python: {exc.msg} (line {exc.lineno}).") from exc
-        visitor = _Complexity.run(tree)
+        except _PARSE_ERRORS as exc:
+            raise InputValidationError(
+                f"code[{i}] is too deeply nested to analyse ({type(exc).__name__})."
+            ) from exc
         blocks = visitor.blocks
         funcs += [c for _, c in blocks]
-        v = _halstead_volume(tree)
         g = visitor.total
         lloc, sloc, comments = _raw(src)
         c_pct = comments / sloc * 100 if sloc else 0.0
@@ -720,38 +728,38 @@ def _parse(src: str) -> Optional[ast.AST]:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", SyntaxWarning)
             return ast.parse(_strip(src))
-    except SyntaxError:
+    except _PARSE_ERRORS:
         return None
 
 
-def _sexp(node: ast.AST, memo: dict[int, str]) -> str:
-    key = id(node)
-    if key in memo:
-        return memo[key]
-    kids = [
-        c
-        for c in ast.iter_child_nodes(node)
-        if not isinstance(c, (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop))
-    ]
-    tag = type(node).__name__
-    if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.AugAssign)):
-        tag += ":" + type(node.op).__name__
-    s = f"({tag}{''.join(' ' + _sexp(c, memo) for c in kids)})"
-    memo[key] = s
-    return s
-
-
-def _subtrees(tree: ast.AST) -> list[str]:
-    memo: dict[int, str] = {}
-    out = []
-    for node in ast.walk(tree):
-        kids = [
-            c
-            for c in ast.iter_child_nodes(node)
-            if not isinstance(c, (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop))
-        ]
+_SKIP_NODES = (ast.expr_context, ast.operator, ast.unaryop, ast.boolop, ast.cmpop)
+
+
+def _kids(node: ast.AST) -> list[ast.AST]:
+    return [c for c in ast.iter_child_nodes(node) if not isinstance(c, _SKIP_NODES)]
+
+
+def _subtrees(tree: ast.AST, table: dict[tuple[Any, ...], int]) -> list[int]:
+    """Every subtree with children (and the root), as ids that are equal exactly when the subtrees are
+    structurally equal. Built bottom-up without recursion: ``table`` interns (node type, child ids), so the
+    cost is linear in the tree size (comparing serialised subtrees would be quadratic)."""
+    ids: dict[int, int] = {}
+    out: list[int] = []
+    stack: list[tuple[ast.AST, bool]] = [(tree, False)]
+    while stack:
+        node, ready = stack.pop()
+        kids = _kids(node)
+        if not ready:
+            stack.append((node, True))
+            stack.extend((k, False) for k in kids)
+            continue
+        tag = type(node).__name__
+        if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.AugAssign)):
+            tag += ":" + type(node.op).__name__
+        sid = table.setdefault((tag, tuple(ids[id(k)] for k in kids)), len(table))
+        ids[id(node)] = sid
         if kids or node is tree:
-            out.append(_sexp(node, memo))
+            out.append(sid)
     return out
 
 
@@ -878,15 +886,16 @@ def codebleu(
     ngram = _corpus_bleu(t_refs, t_hyps, None)
     weighted = _corpus_bleu(t_refs, t_hyps, kw)
     match = total = df_match = df_total = 0
+    table: dict[tuple[Any, ...], int] = {}
     for rs, p in zip(refs, preds):
         cand = _parse(p)
-        cand_sub = set(_subtrees(cand)) if cand is not None else set()
+        cand_sub = set(_subtrees(cand, table)) if cand is not None else set()
         cand_flow = _normalize_flow(_dataflow(cand)) if cand is not None else []
         for r in rs:
             ref = _parse(r)
             if ref is None:
                 continue
-            ref_sub = _subtrees(ref)
+            ref_sub = _subtrees(ref, table)
             match += sum(s in cand_sub for s in ref_sub)
             total += len(ref_sub)
             ref_flow = _normalize_flow(_dataflow(ref))

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/llmsys/safety.py

@@ -362,7 +362,7 @@ def _luhn(digits: str) -> bool:
 
 
 PII_PATTERNS: dict[str, str] = {
-    "email": r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
+    "email": r"(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
     "phone": r"(?<!\d[\s.-])(?<!\w)(?:\+?\d{1,3}[\s.-]?)?(?:\(\d{2,4}\)[\s.-]?)?\d{3,4}[\s.-]\d{3,4}"
     r"(?:[\s.-]\d{2,4})?(?![\s.-]?\d)(?!\w)",
     "credit_card": r"(?<!\d)\d(?:[ -]?\d){12,18}(?!\d)",

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/stats/effect.py

@@ -2,11 +2,17 @@
 
 from __future__ import annotations
 
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special
+
+if TYPE_CHECKING:
+    from scipy import special
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
 
 from ..core.exceptions import InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/stats/hypothesis.py

@@ -8,11 +8,17 @@ implementations are the reference; EvalSuite adds input validation, effect sizes
 from __future__ import annotations
 
 import math
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from .._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/stats/intervals.py

@@ -3,11 +3,18 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any, Literal, Optional
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import special, stats
+
+if TYPE_CHECKING:
+    from scipy import special, stats
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import EvalSuiteError, InputValidationError, StatisticalTestError
 from ..core.types import ArrayLike

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/stats/paired.py

@@ -2,10 +2,17 @@
 
 from __future__ import annotations
 
-from typing import Any, Literal, Optional
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 import numpy as np
-from scipy import special, stats
+
+if TYPE_CHECKING:
+    from scipy import special, stats
+else:
+    from .._lazy import lazy_module
+
+    special = lazy_module("scipy.special")
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError
 from ..core.types import ArrayLike

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/text/judge.py

@@ -9,10 +9,16 @@ from __future__ import annotations
 
 import math
 from collections.abc import Sequence
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import numpy as np
-from scipy import stats
+
+if TYPE_CHECKING:
+    from scipy import stats
+else:
+    from .._lazy import lazy_module
+
+    stats = lazy_module("scipy.stats")
 
 from ..core.exceptions import InputValidationError
 from ..core.registry import register

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/text/structured.py

@@ -712,7 +712,11 @@ def constraint_satisfaction_rate(checks: Any) -> MetricResult:
 def format_compliance(
     outputs: Any, pattern: str, *, flags: int = re.S, average: Optional[str] = "mean"
 ) -> MetricResult:
-    """``re.fullmatch(pattern, output.strip())`` per output."""
+    """``re.fullmatch(pattern, output.strip())`` per output.
+
+    The pattern is compiled with Python's backtracking ``re`` engine; patterns with nested quantifiers such as
+    ``(a+)+$`` can take exponential time on some outputs, so only pass patterns you trust.
+    """
     outs = _outputs(outputs)
     try:
         rx = re.compile(pattern, flags)

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/version.py

@@ -1,3 +1,3 @@
 """Package version (single source of truth, read by the build backend)."""
 
-__version__ = "0.5.1"
+__version__ = "0.5.2"

{evalsuite_python-0.5.1 → evalsuite_python-0.5.2}/src/evalsuite/vision/segmentation.py

@@ -27,7 +27,13 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
 
 import numpy as np
 from numpy.typing import NDArray
-from scipy import ndimage
+
+if TYPE_CHECKING:
+    from scipy import ndimage
+else:
+    from .._lazy import lazy_module
+
+    ndimage = lazy_module("scipy.ndimage")
 
 from ..core.exceptions import InputValidationError, UndefinedMetricWarning
 from ..core.export import PathLike, csv_text, html_document, html_table, save_as

evalsuite_python-0.5.2/tests/llmsys/test_hardening.py

@@ -0,0 +1,52 @@
+"""Regression tests for adversarial inputs (parser exhaustion, regex backtracking) and lazy imports."""
+
+import subprocess
+import sys
+import time
+
+import pytest
+
+import evalsuite as es
+from evalsuite.core.exceptions import InputValidationError
+
+DEEP = "x=" + "-" * 100_000 + "1"
+
+
+def test_syntax_validity_survives_parser_stack_overflow():
+    assert float(es.syntax_validity_rate([DEEP, "x = 1"])) == 0.5
+
+
+@pytest.mark.parametrize("src", [DEEP, "x=" + "-" * 900 + "1"])
+def test_code_complexity_rejects_too_deep_code_cleanly(src):
+    with pytest.raises(InputValidationError, match=r"too deeply nested|not valid Python"):
+        es.code_complexity([src])
+
+
+def test_codebleu_deep_input_does_not_crash():
+    r = es.codebleu([DEEP], [DEEP])
+    assert 0.0 <= float(r) <= 1.0
+
+
+def test_codebleu_large_program_is_fast():
+    src = "\n".join(f"def f{i}(a, b):\n    return [x * b for x in range(a + {i})]\n" for i in range(2000))
+    start = time.perf_counter()
+    es.codebleu([src], [src])
+    assert time.perf_counter() - start < 5
+
+
+@pytest.mark.parametrize("text", ["1-" * 30_000, "a" * 30_000 + "@" + "b." * 10_000, "a." * 30_000, "a@" * 30_000])
+def test_detect_pii_is_linear_on_adversarial_text(text):
+    start = time.perf_counter()
+    es.detect_pii(text)
+    assert time.perf_counter() - start < 1
+
+
+def test_detect_pii_still_finds_emails():
+    found = es.detect_pii("write to a.b@x.org or c@d.co.uk")
+    assert found["email"] == ["a.b@x.org", "c@d.co.uk"]
+
+
+def test_import_does_not_load_scipy():
+    code = "import sys, evalsuite; print('scipy' in sys.modules)"
+    out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)  # noqa: S603
+    assert out.stdout.strip() == "False"