evalaudit 0.0.1__py3-none-any.whl

evalaudit/__init__.py

@@ -0,0 +1,7 @@
+"""evalaudit: statistical validity checks for AI evaluations."""
+
+__version__ = "0.1.0"
+
+from .scores import score_ci
+
+__all__ = ["score_ci"]

evalaudit/_types.py

@@ -0,0 +1,66 @@
+"""Return types for evalaudit.
+
+Every public function returns one of these. Each carries the numbers and a
+``summary()`` that states, in plain English, what the numbers do and do not
+support. The summary is the point of the library: scipy already computes
+McNemar, but nothing tells a product manager their result does not hold.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+def _pct(x: float) -> str:
+    return f"{x * 100:.1f}%"
+
+
+@dataclass(frozen=True)
+class ScoreCI:
+    """Confidence interval on a single eval score."""
+
+    estimate: float
+    ci_low: float
+    ci_high: float
+    n: int
+    method: str
+    binary: bool
+    confidence: float = 0.95
+
+    @property
+    def margin(self) -> float:
+        """Half-width of the interval."""
+        return (self.ci_high - self.ci_low) / 2
+
+    @property
+    def width(self) -> float:
+        return self.ci_high - self.ci_low
+
+    def summary(self) -> str:
+        conf = f"{self.confidence * 100:.0f}%"
+        if self.binary:
+            head = (
+                f"{_pct(self.estimate)} pass rate "
+                f"({conf} CI: {_pct(self.ci_low)}-{_pct(self.ci_high)}, n={self.n})."
+            )
+            span = f"{self.width * 100:.0f} points"
+        else:
+            head = (
+                f"Mean score {self.estimate:.3f} "
+                f"({conf} CI: {self.ci_low:.3f}-{self.ci_high:.3f}, n={self.n})."
+            )
+            span = f"{self.width:.3f}"
+
+        tail = (
+            f" The interval spans {span}; treat differences smaller than that "
+            f"as unresolved."
+        )
+        if self.n < 30:
+            tail += (
+                f" With only {self.n} observations this estimate is weak "
+                f"regardless of the point value."
+            )
+        return head + tail
+
+    def __str__(self) -> str:  # pragma: no cover
+        return self.summary()

evalaudit/scores.py

@@ -0,0 +1,136 @@
+"""Confidence intervals on a single eval score.
+
+The most common missing piece in published eval results: a pass rate with no
+interval around it. At n=50 the interval is roughly +/- 10 points, which is
+usually wider than the difference being claimed.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Sequence
+
+import numpy as np
+from scipy import stats as _stats
+
+from ._types import ScoreCI
+
+__all__ = ["score_ci"]
+
+_VALID_METHODS = {"auto", "wilson", "bootstrap", "t"}
+
+
+def score_ci(
+    scores: Sequence[float],
+    method: str = "auto",
+    confidence: float = 0.95,
+    n_boot: int = 10000,
+    seed: Optional[int] = None,
+) -> ScoreCI:
+    """Confidence interval on an eval score.
+
+    Parameters
+    ----------
+    scores
+        Binary (0/1) or continuous values, one per evaluated item.
+    method
+        ``"auto"`` picks Wilson for binary data and bootstrap for continuous.
+        ``"wilson"`` is the score interval for a proportion. Use it rather than
+        the normal approximation, which is visibly wrong at small n and near
+        the boundaries, and can return bounds outside [0, 1].
+        ``"bootstrap"`` is the percentile bootstrap of the mean.
+        ``"t"`` is the Student-t interval on the mean.
+    confidence
+        Nominal coverage, default 0.95.
+    n_boot
+        Bootstrap resamples. 1000 is enough for a consulting report;
+        the default is generous.
+    seed
+        Seeds the bootstrap so results are reproducible. Always set this in
+        anything you publish.
+
+    Returns
+    -------
+    ScoreCI
+
+    Examples
+    --------
+    >>> score_ci([1] * 42 + [0] * 8).summary()
+    '84.0% pass rate (95% CI: 71.5%-91.7%, n=50). ...'
+    """
+    x = np.asarray(scores, dtype=float)
+    n = len(x)
+    if n == 0:
+        raise ValueError("scores must not be empty")
+    if method not in _VALID_METHODS:
+        raise ValueError(f"method must be one of {_VALID_METHODS}, got {method!r}")
+
+    binary = _is_binary(x)
+
+    if method == "auto":
+        method = "wilson" if binary else "bootstrap"
+
+    estimate = float(np.mean(x))
+
+    if method == "wilson":
+        k = int(np.sum(x))
+        ci_low, ci_high = _wilson(k, n, confidence)
+    elif method == "bootstrap":
+        ci_low, ci_high = _bootstrap_mean(x, confidence, n_boot, seed)
+    else:  # "t"
+        alpha = 1 - confidence
+        sem = float(np.std(x, ddof=1) / np.sqrt(n))
+        t_crit = _stats.t.ppf(1 - alpha / 2, df=n - 1)
+        ci_low = estimate - t_crit * sem
+        ci_high = estimate + t_crit * sem
+
+    return ScoreCI(
+        estimate=estimate,
+        ci_low=ci_low,
+        ci_high=ci_high,
+        n=n,
+        method=method,
+        binary=binary,
+        confidence=confidence,
+    )
+
+
+def _is_binary(x: np.ndarray) -> bool:
+    """True when every value is 0 or 1."""
+    return bool(np.all((x == 0) | (x == 1)))
+
+
+def _wilson(k: int, n: int, confidence: float) -> tuple[float, float]:
+    """Wilson score interval for a proportion.
+
+        (p + z^2/2n  +/-  z * sqrt( p(1-p)/n + z^2/4n^2 )) / (1 + z^2/n)
+
+    Clip to [0, 1] at the end.
+    """
+    p = k / n
+    z = _stats.norm.ppf(1 - (1 - confidence) / 2)
+    z2 = z * z
+    denom = 1 + z2 / n
+    centre = (p + z2 / (2 * n)) / denom
+    margin = (z / denom) * np.sqrt(p * (1 - p) / n + z2 / (4 * n * n))
+    lo = float(np.clip(centre - margin, 0.0, 1.0))
+    hi = float(np.clip(centre + margin, 0.0, 1.0))
+    return lo, hi
+
+
+def _bootstrap_mean(
+    x: np.ndarray, confidence: float, n_boot: int, seed: Optional[int]
+) -> tuple[float, float]:
+    """Percentile bootstrap of the mean.
+
+    Vectorise this. Draw an (n_boot, n) index matrix with
+    ``rng.integers(0, n, size=(n_boot, n))`` and take row means in one pass.
+    A Python loop here is the difference between instant and slow on a laptop.
+    """
+    rng = np.random.default_rng(seed)
+    n = len(x)
+    idx = rng.integers(0, n, size=(n_boot, n))
+    means = x[idx].mean(axis=1)
+    alpha = 1 - confidence
+    lo = float(np.percentile(means, 100 * alpha / 2))
+    hi = float(np.percentile(means, 100 * (1 - alpha / 2)))
+    return lo, hi

evalaudit-0.0.1.dist-info/METADATA

@@ -0,0 +1,134 @@
+Metadata-Version: 2.5
+Name: evalaudit
+Version: 0.0.1
+Summary: Statistical validity checks for human-graded AI evaluations. Rater agreement, judge validation, and whether your result holds up.
+Project-URL: Homepage, https://github.com/ondibahezron-glitch/evalaudit
+Project-URL: Issues, https://github.com/ondibahezron-glitch/evalaudit/issues
+Author-email: Hezron Mokaya <ondibahezron@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: annotation-quality,bradley-terry,evaluation,inter-rater-reliability,krippendorff,llm,llm-as-judge,statistics
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.22
+Requires-Dist: pandas>=1.5
+Requires-Dist: scipy>=1.9
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# evalaudit
+
+[![Tests](https://github.com/ondibahezron-glitch/evalaudit/actions/workflows/tests.yml/badge.svg)](https://github.com/ondibahezron-glitch/evalaudit/actions/workflows/tests.yml)
+
+**Statistical validity checks for human-graded AI evaluations.**
+
+Three experts graded 200 items. One of them disagreed with the other two on
+almost everything. Nobody noticed, because nobody computed agreement — and
+half the items were only graded once, so it could not have been computed.
+
+That evaluation produced a number. The number is now in a slide deck.
+
+`evalaudit` is the set of checks that should have run first.
+
+---
+
+## Start here
+
+```python
+from evalaudit import score_ci
+
+wins = [1] * 58 + [0] * 42
+print(score_ci(wins).summary())
+```
+
+```
+58.0% pass rate (95% CI: 48.2%-67.2%, n=100). The interval spans
+19 points; treat differences smaller than that as unresolved.
+```
+
+The interval crosses 50%. On this data the new model might be worse than the
+old one. That evaluation cannot support the decision being made with it.
+
+---
+
+## Why this exists
+
+`scipy` computes McNemar's test. `sklearn` computes Cohen's kappa. Several
+libraries now put bootstrapped intervals on automated eval scores.
+
+What is missing is anything built for the **human** side of evaluation: the
+graders, the rubric, the LLM judge standing in for both. That is where the
+expensive failures happen, and it is where almost no tooling exists.
+
+Every function returns an object with a `.summary()`. The numbers are the
+input to the audit. The sentence is the output.
+
+Two rules the library follows throughout:
+
+- **Never report a bare p-value.** Always an effect size and an interval.
+- **State what the data cannot support**, not only what it can.
+
+---
+
+## What it checks
+
+| Module | Question |
+|---|---|
+| `agreement` | Do your graders agree? Which grader is the outlier? Which items are ambiguous rather than hard? |
+| `judge` | Does your LLM judge track your humans — and on which slices does it stop? Is it biased by position or length? |
+| `pairwise` | In a blind pairwise comparison, which models are actually separable? |
+| `power` | Could this many comparisons ever have detected the effect you care about? |
+| `scores` | Is there an interval around your headline number, and how wide? |
+| `compare` | Does the margin between two systems survive a paired test? |
+| `audit` | All of the above, as a report ranked by what changes the conclusion. |
+
+The first four are the point. The last three are the foundation they stand on.
+
+---
+
+## Status
+
+Early. `scores` is implemented. `agreement`, `judge` and `pairwise` are
+specified and landing over the coming weeks.
+
+---
+
+## Coverage
+
+A statistics package earns trust by showing its intervals cover at the rate
+they claim. Every interval here is tested by simulation: generate 1,000
+datasets with a known truth, confirm the 95% interval contains it about 95% of
+the time. Those tests run in CI and you can read them in `tests/`.
+
+---
+
+## Related
+
+[`promptstats`](https://pypi.org/project/promptstats/) covers the automated
+side well — prompt sensitivity, model comparison across prompt variations,
+bootstrapped intervals on benchmark scores. If your evaluation has no human
+graders in it, start there.
+
+---
+
+## Install
+
+```bash
+git clone https://github.com/ondibahezron-glitch/evalaudit.git
+cd evalaudit
+pip install -e ".[dev]"
+```
+
+Python 3.9+, numpy, scipy, pandas. Nothing else. PyPI package coming soon.
+
+---
+
+## Licence
+
+MIT.

evalaudit-0.0.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+evalaudit/__init__.py,sha256=xDC1rHAi-QtzJNx-hZ2zZXA8hrajKAKoT20w9lOo42g,142
+evalaudit/_types.py,sha256=_G2Q5_UZsIWKXKrEF8ZNQI3WpmQ5Sx0RojZTOHw48Ic,1890
+evalaudit/scores.py,sha256=diNHQpVWOgNQC1CXI-w_b7XDyl0T_w8Uq7cBx18IpV0,4099
+evalaudit-0.0.1.dist-info/METADATA,sha256=3CLJxaioUBFr_RsdiYjzoZj1vFnGik8gSAnr72jLVYM,4524
+evalaudit-0.0.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+evalaudit-0.0.1.dist-info/licenses/LICENSE,sha256=JI27FMe4Ywi4XKsJPsHc4eYV0M71wU9_8FypQkia0XA,1070
+evalaudit-0.0.1.dist-info/RECORD,,

evalaudit-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

evalaudit-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hezron Mokaya
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.