measly 0.1.0__py3-none-any.whl

measly/__init__.py

@@ -0,0 +1,43 @@
+"""Estimate whether a model is data-limited, and whether more capacity would pay off."""
+
+from measly.analysis import (
+    DEFAULT_FRACTIONS,
+    VALIDATION_WARN,
+    Analysis,
+    Projection,
+    analyse,
+    mean_squared_error,
+)
+from measly.fit import (
+    POW3,
+    POW4,
+    CurveEnsemble,
+    ScalingLaw,
+    fit_scaling_law,
+    fit_scaling_laws,
+    pow3,
+)
+from measly.interfaces import Model, Score
+from measly.plot import plot
+from measly.sweep import sweep, train_test_split
+
+__all__ = [
+    "analyse",
+    "Analysis",
+    "Projection",
+    "mean_squared_error",
+    "plot",
+    "DEFAULT_FRACTIONS",
+    "VALIDATION_WARN",
+    "Model",
+    "Score",
+    "train_test_split",
+    "sweep",
+    "pow3",
+    "ScalingLaw",
+    "POW3",
+    "POW4",
+    "CurveEnsemble",
+    "fit_scaling_law",
+    "fit_scaling_laws",
+]

measly/analysis.py

@@ -0,0 +1,287 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+
+import numpy as np
+import xarray as xr
+from numpy.typing import NDArray
+
+from measly.fit import POW4, CurveEnsemble, ScalingLaw, fit_scaling_law, fit_scaling_laws
+from measly.interfaces import Model, Score
+from measly.sweep import sweep
+
+__all__ = [
+    "mean_squared_error",
+    "DEFAULT_FRACTIONS",
+    "VALIDATION_WARN",
+    "Projection",
+    "Analysis",
+    "analyse",
+]
+
+DEFAULT_FRACTIONS: tuple[float, ...] = tuple(
+    round(f, 4) for f in np.geomspace(0.1, 1.0, 8)
+)
+
+
+VALIDATION_WARN = 0.15
+"""Above this held-back error, a projection MUST NOT be relied upon."""
+
+
+def mean_squared_error(predicted, true) -> float:
+    """The default `Score`. Regression only. Supply your own otherwise."""
+    return float(np.mean((np.asarray(predicted) - np.asarray(true)) ** 2))
+
+
+def _score_name(score: Score) -> str:
+    """A label for the report. Falls back to the type for a callable object."""
+    return getattr(score, "__name__", None) or type(score).__name__
+
+
+def _unwrap(values: NDArray, scalar: bool) -> float | NDArray:
+    """Give back a float for a scalar `factor`, an array for several."""
+    return float(values[0]) if scalar else values
+
+
+@dataclass(frozen=True)
+class Projection:
+    """What a model is expected to score on a larger training set.
+
+    Every field is a float when `project` is given one factor, and an array
+    aligned with the factors when it is given several.
+    """
+
+    loss: float | NDArray
+    low: float | NDArray
+    high: float | NDArray
+    gain: float | NDArray
+    gain_low: float | NDArray
+    gain_high: float | NDArray
+
+
+@dataclass(frozen=True)
+class Analysis:
+    """The measured curve, the fitted curves, and whether to believe them."""
+
+    results: xr.DataArray
+    curves: dict[str, CurveEnsemble]
+    validation: dict[str, float]
+    law: ScalingLaw
+    score: Score
+    n_train: int
+
+    @property
+    def models(self) -> list[str]:
+        return list(self.curves)
+
+    def measured(self) -> dict[str, float]:
+        """Mean score at the full training pool, straight from the sweep."""
+        largest = float(self.results.fraction.max())
+        return {
+            name: float(self.results.sel(model=name, fraction=largest).mean("draw"))
+            for name in self.models
+        }
+
+    def project(self, factor: float = 2.0, interval: float = 0.9
+                ) -> dict[str, Projection]:
+        """Loss and gain at `factor` times the current training pool.
+
+        `factor` MAY be a sequence. One call then gives the whole
+        extrapolation curve, which is what a plot needs.
+
+        The interval on `loss` is a percentile range across the fitted curves.
+        It is dominated by test-set noise: every point in a draw shares one
+        test set, so that draw's curve is shifted bodily. It is conservative,
+        covering 98% or more at a nominal 90% against simulated ground truth.
+
+        The interval on `gain` is far tighter, because each member is paired
+        with itself and the shared offset cancels. `gain` is therefore not
+        `measured()` minus `loss`; it is what the curve itself says it climbs.
+        """
+        tail = (1.0 - interval) / 2.0
+        quantiles = [tail, 1.0 - tail]
+
+        scalar = np.ndim(factor) == 0
+        factors = np.atleast_1d(np.asarray(factor, dtype=float))
+
+        projections = {}
+        for name, ensemble in self.curves.items():
+            # (len(factors), n_members). Quantiles MUST reduce the member axis
+            # only; flattening both would mix factors into one distribution.
+            future = ensemble.predict(factors)
+            # Pair each member against its own value at the measured size.
+            # Both carry that draw's test-set offset, so it cancels; comparing
+            # against a mean over draws leaves it in and swamps the gain.
+            gains = ensemble.predict(np.ones(1)) - future
+            lo, hi = np.quantile(future, quantiles, axis=-1)
+            gain_lo, gain_hi = np.quantile(gains, quantiles, axis=-1)
+            projections[name] = Projection(
+                loss=_unwrap(np.median(future, axis=-1), scalar),
+                low=_unwrap(lo, scalar),
+                high=_unwrap(hi, scalar),
+                gain=_unwrap(np.median(gains, axis=-1), scalar),
+                gain_low=_unwrap(gain_lo, scalar),
+                gain_high=_unwrap(gain_hi, scalar),
+            )
+        return projections
+
+    def summary(self, factor: float = 2.0) -> str:
+        now = self.measured()
+        projected = self.project(factor)
+        target = round(self.n_train * factor)
+
+        lines = [
+            f"measly: {len(self.models)} model(s), {self.n_train} training examples, "
+            f"{self.results.sizes['draw']} draws",
+            f"        {self.law!r}, {_score_name(self.score)} (lower is better)",
+            "",
+        ]
+        for name in self.models:
+            p = projected[name]
+
+            error = self.validation.get(name, float("nan"))
+            if np.isnan(error):
+                check = "not run"
+            elif error > VALIDATION_WARN:
+                check = (f"{error:.1%} error — this curve does not predict its own "
+                         "measured sizes, so do not rely on the projection")
+            else:
+                check = f"{error:.1%} error"
+
+            rows = [
+                (f"now ({self.n_train} examples)", f"{now[name]:.4f}"),
+                (f"at {target} examples",
+                 f"{p.loss:.4f} [{p.low:.4f}, {p.high:.4f}]"),
+                ("gain from getting there",
+                 f"{p.gain:+.4f} [{p.gain_low:+.4f}, {p.gain_high:+.4f}]"),
+                ("held-back check", check),
+            ]
+            width = max(len(label) for label, _ in rows)
+
+            lines.append(f"  {name}")
+            lines += [f"    {label:<{width}}  {value}" for label, value in rows]
+            lines.append("")
+
+        return "\n".join(lines).rstrip()
+
+    def __repr__(self) -> str:
+        worst = max((v for v in self.validation.values() if not np.isnan(v)),
+                    default=float("nan"))
+        checked = "unchecked" if np.isnan(worst) else f"worst held-back {worst:.1%}"
+        return (f"<Analysis {len(self.models)} model(s), n_train={self.n_train}, "
+                f"{self.law!r}, {checked}>")
+
+
+def _validate(results: xr.DataArray, fractions: Sequence[float], hold_back: int,
+              law: ScalingLaw, floor: float) -> dict[str, float]:
+    """Refit on the smaller sizes, then predict the largest measured ones.
+
+    Returns the worst relative error per model. Returns nan when too few points
+    remain to fit the law once some are held back.
+    """
+    names = [str(n) for n in np.atleast_1d(results.model.values)]
+    kept, held = list(fractions[:-hold_back]), list(fractions[-hold_back:])
+
+    n_params = law.func.__code__.co_argcount - 1
+    if hold_back <= 0 or len(kept) <= n_params:
+        return {name: float("nan") for name in names}
+
+    errors = {}
+    for name in names:
+        ensemble = fit_scaling_law(
+            results.sel(model=name, fraction=kept), law=law, floor=floor
+        )
+        if len(ensemble) == 0:
+            errors[name] = float("nan")
+            continue
+
+        worst = 0.0
+        for fraction in held:
+            actual = float(results.sel(model=name, fraction=fraction).mean("draw"))
+            predicted = float(np.median(np.ravel(ensemble.predict(fraction))))
+            if actual != 0.0:
+                worst = max(worst, abs(predicted - actual) / abs(actual))
+        errors[name] = worst
+
+    return errors
+
+
+def analyse(
+    models: Mapping[str, Model] | Sequence[Model],
+    X,
+    y,
+    score: Score = mean_squared_error,
+    *,
+    fractions: Sequence[float] = DEFAULT_FRACTIONS,
+    n_draws: int = 100,
+    test_fraction: float = 0.25,
+    law: ScalingLaw = POW4,
+    floor: float = 0.0,
+    rng: int | np.random.Generator | None = None,
+    hold_back: int = 2,
+    n_jobs: int = 1,
+) -> Analysis:
+    """Measure a learning curve for each model and project it forward.
+
+    `rng` accepts a seed, a `Generator`, or nothing. Pass a seed for a
+    reproducible run. The split and the subsets both depend on it.
+
+    `hold_back` sets how many of the largest measured sizes are withheld from a
+    second fit, used to check the projection. Set it to 0 to skip the check.
+    You then have no evidence the form suits your data.
+
+    `models` MAY be a `{name: model}` mapping. Name them whenever `repr` is
+    unhelpful, as it is for an sklearn `Pipeline`.
+
+    `n_jobs` parallelises the sweep without changing results. See `sweep`.
+    """
+    generator = np.random.default_rng(rng)
+    fractions = list(fractions)
+
+    results = sweep(
+        models=models,
+        X=X,
+        y=y,
+        fractions=fractions,
+        rng=generator,
+        score=score,
+        n_draws=n_draws,
+        test_fraction=test_fraction,
+        n_jobs=n_jobs,
+    )
+
+    curves = fit_scaling_laws(results, law=law, floor=floor)
+    validation = _validate(results, fractions, hold_back, law, floor)
+    n_train = int(round(len(y) * (1.0 - test_fraction) * max(fractions)))
+
+    return Analysis(results=results, curves=curves, validation=validation,
+                    law=law, score=score, n_train=n_train)
+
+
+if __name__ == "__main__":
+    # uv run python src/measly/analysis.py
+    # The `-m` form warns: __init__ imports this module, so runpy loads it twice.
+    class _Ridge:
+        """A model in the fit/predict convention. Penalty sets its capacity."""
+
+        def __init__(self, penalty: float) -> None:
+            self.penalty, self.weights = penalty, None
+
+        def __repr__(self) -> str:
+            return f"ridge({self.penalty:g})"
+
+        def fit(self, X, y) -> None:
+            gram = X.T @ X + self.penalty * np.eye(X.shape[1])
+            self.weights = np.linalg.solve(gram, X.T @ y)
+
+        def predict(self, X):
+            return X @ self.weights
+
+    _rng = np.random.default_rng(0)
+    _X = _rng.normal(size=(2000, 12))
+    _y = _X @ _rng.normal(size=12) + _rng.normal(0.0, 1.0, 2000)
+    _models = [_Ridge(1.0), _Ridge(200.0)]
+
+    # One call does the sweep, the fits and the hold-back check.
+    _result = analyse(_models, _X, _y, rng=0, score=mean_squared_error)
+    print(_result.summary())
+

measly/fit.py

@@ -0,0 +1,145 @@
+"""Fit a scaling law to a sweep's measured losses.
+
+One curve per draw. Draws are independent, so a single curve means nothing.
+Read percentiles across the ensemble.
+"""
+
+from dataclasses import dataclass, field
+from typing import Callable
+
+import numpy as np
+import xarray as xr
+from numpy.typing import ArrayLike, NDArray
+from scipy.optimize import curve_fit
+
+
+__all__ = [
+    "pow3",
+    "ScalingLaw",
+    "POW3",
+    "POW4",
+    "CurveEnsemble",
+    "fit_scaling_law",
+    "fit_scaling_laws",
+]
+
+# Below 0, loss would rise with data. Above 5 is not physical.
+_ALPHA_BOUNDS = (0.01, 5.0)
+
+
+def pow3(n, lower_bound, A, alpha):
+    """L(n) = L_inf + A * n^(-alpha). Called `pow3` in the literature.
+
+    `n` MAY be counts or fractions of the pool. `lower_bound` and `alpha` are
+    identical either way. Rescaling n is absorbed entirely by `A`.
+    """
+    return lower_bound + A * np.asarray(n, dtype=float) ** (-alpha)
+
+
+@dataclass(frozen=True)
+class ScalingLaw:
+    """A functional form plus the starting point and bounds it needs.
+
+    These MUST travel together. Forms differ in parameter count, and in which
+    parameter is the asymptote that `floor` constrains. A generic guess would
+    constrain the wrong one. `guess(y, floor)` returns `(p0, (lower, upper))`.
+    """
+
+    name: str
+    func: Callable[..., NDArray]
+    guess: Callable[[NDArray, float], tuple[list[float], tuple]]
+
+    def __call__(self, x, *params) -> NDArray:
+        return self.func(x, *params)
+
+    def __repr__(self) -> str:
+        return self.name
+
+
+def _pow4(n, lower_bound, A, alpha, shift):
+    """L(n) = L_inf + A * (n + shift)^(-alpha). Called `pow4`.
+
+    `pow3` is this with `shift` at zero. So `pow4` never fits worse.
+
+    `shift` separates where the curve bends from how low it ends up. Under
+    `pow3` one parameter sets both. A curve still descending at the largest
+    measured size can then only be fitted by driving `L_inf` to zero.
+    """
+    return lower_bound + A * (np.asarray(n, dtype=float) + shift) ** (-alpha)
+
+
+def _pow3_guess(y: NDArray, floor: float) -> tuple[list[float], tuple]:
+    start = max(float(y.min()) * 0.9, floor)
+    p0 = [start, max(float(y.max() - y.min()), 1e-6), 1.0]
+    return p0, ((floor, 0.0, _ALPHA_BOUNDS[0]), (np.inf, np.inf, _ALPHA_BOUNDS[1]))
+
+
+def _pow4_guess(y: NDArray, floor: float) -> tuple[list[float], tuple]:
+    p0, (lower, upper) = _pow3_guess(y, floor)
+    return p0 + [0.01], (lower + (0.0,), upper + (10.0,))
+
+
+POW3 = ScalingLaw("pow3", pow3, _pow3_guess)
+POW4 = ScalingLaw("pow4", _pow4, _pow4_guess)
+
+
+@dataclass
+class CurveEnsemble:
+    """Fitted curves from one model's draws. Read them by evaluating all."""
+
+    curves: list[Callable] = field(default_factory=list)
+    n_failed: int = 0  # draws whose fit did not converge
+
+    def __len__(self) -> int:
+        return len(self.curves)
+
+    def predict(self, x: ArrayLike) -> NDArray:
+        """Evaluate every member at every x.
+
+        Returns `(len(x), n_members)`. Take quantiles across members for an
+        interval. A single member MUST NOT be read on its own.
+        """
+        return np.stack([curve(x) for curve in self.curves], axis=-1)
+
+
+def fit_scaling_law(
+    model_results: xr.DataArray, law: ScalingLaw = POW4, floor: float = 0.0
+) -> CurveEnsemble:
+    """Fit one curve per draw for a single model.
+
+    `model_results` has dims `(fraction, draw)`. `floor` raises the lower bound
+    on the asymptote. Pass a measured noise floor if you have one.
+
+    `law` picks the form. `POW4` is the default. On curves that have not yet
+    bent it extrapolates far better than `POW3`. `POW3` in that regime loses to
+    assuming no further improvement at all. `POW3` is what the literature
+    usually quotes.
+    """
+    ensemble = CurveEnsemble()
+
+    x = np.asarray(model_results.fraction.values, dtype=float)
+
+    for y in model_results.transpose("draw", "fraction").values:
+        p0, bounds = law.guess(y, floor)
+
+        try:
+            popt, _ = curve_fit(law.func, x, y, p0=p0, bounds=bounds, maxfev=10_000)
+        except (RuntimeError, ValueError):
+            ensemble.n_failed += 1
+            continue
+
+        # popt MUST be bound as a default argument. A closure over the loop
+        # variable leaves every curve holding the last draw's parameters.
+        ensemble.curves.append(lambda t, p=popt: law.func(t, *p))
+
+    return ensemble
+
+
+def fit_scaling_laws(
+    results: xr.DataArray, law: ScalingLaw = POW4, floor: float = 0.0
+) -> dict[str, CurveEnsemble]:
+    """Fit every model in a sweep, keyed by its label on the `model` axis."""
+    return {
+        str(name): fit_scaling_law(results.sel(model=name), law=law, floor=floor)
+        for name in results.model.values
+    }

measly/interfaces.py

@@ -0,0 +1,31 @@
+"""The contracts a user implements to run measly on their own problem."""
+
+from typing import Any, Callable, Protocol, Sequence
+
+__all__ = ["Score", "Model"]
+
+
+type Score = Callable[[Sequence[float], Sequence[float]], float]
+"""Reduce predictions and true targets to one number. Lower MUST be better.
+
+It SHOULD be a mean over per-point terms: MSE, MAE, error rate. Set-level
+statistics MUST NOT be used. F1, AUC and Spearman do not decompose per point,
+so the power-law form does not describe them. The fit still returns a number.
+"""
+
+
+class Model(Protocol):
+    """Anything with scikit-learn's fit/predict convention.
+
+    `fit` MUST leave the model fully retrained. `sweep` refits one instance at
+    every sample size. A model that carries state between calls, such as an
+    sklearn estimator with `warm_start=True`, reports the same score at every
+    size. That plots as a flat curve and raises nothing.
+
+    `predict` MUST return one number per row of `X`, in order. Returns are
+    `Any`: sklearn's `fit` gives back `self`, and `predict` an ndarray.
+    """
+
+    def fit(self, X, y) -> Any: ...
+
+    def predict(self, X) -> Any: ...

measly/plot.py

@@ -0,0 +1,81 @@
+"""One figure for an `Analysis`: what was measured, and what follows from it."""
+
+import matplotlib.pyplot as plt
+import numpy as np
+from scipy.ndimage import gaussian_filter1d
+from matplotlib.ticker import FuncFormatter, LogLocator, NullLocator
+
+from measly.analysis import Analysis, _score_name
+
+__all__ = ["plot"]
+
+PALETTE = ("#2563eb", "#ea580c", "#059669", "#7c3aed", "#db2777")
+GUIDE = "#94a3b8"
+
+
+def plot(analysis: Analysis, factor: float = 2.0, interval: float = 0.9, ax=None):
+    """Measured points, the fitted band, and the projection out to `factor`.
+
+    Points are the mean over draws, with bars at one standard deviation. The
+    band is the percentile range across fitted curves, so it widens right of
+    the dashed line, where nothing was measured.
+
+    Returns the `Axes`, so several analyses MAY share a figure.
+    """
+    if ax is None:
+        _, ax = plt.subplots(figsize=(7.5, 4.5))
+
+    measured = np.asarray(analysis.results.fraction.values, dtype=float)
+    grid = np.geomspace(measured.min(), max(factor, measured.max()), 200)
+    projected = analysis.project(grid, interval)
+
+    for index, name in enumerate(analysis.models):
+        scores = analysis.results.sel(model=name)
+        band = projected[name]
+        colour = PALETTE[index % len(PALETTE)]
+
+        error = analysis.validation.get(name, float("nan"))
+        label = name if np.isnan(error) else f"{name}   held-back {error:.1%}"
+
+        ax.fill_between(grid * analysis.n_train, band.low, band.high,
+                        color=colour, alpha=0.12, lw=0)
+        for edge in (band.low, band.high):
+            ax.plot(grid * analysis.n_train, edge, color=colour, alpha=0.35, lw=0.8)
+        # The pointwise median of a finite ensemble kinks where members cross.
+        # Smoothing is cosmetic. `project` and `summary` report the raw median.
+        centre = gaussian_filter1d(band.loss, sigma=4, mode="nearest")
+        ax.plot(grid * analysis.n_train, centre, color=colour, lw=2,
+                label=label, zorder=4)
+        ax.errorbar(measured * analysis.n_train, scores.mean("draw"),
+                    yerr=scores.std("draw", ddof=1), fmt="o", ms=5,
+                    color=colour, elinewidth=1.1, capsize=0, zorder=5)
+
+    ax.axvline(analysis.n_train, color=GUIDE, ls=(0, (2, 3)), lw=1, zorder=1)
+    for text, offset, side in (("measured", -5, "right"), ("projected", 5, "left")):
+        ax.annotate(text, xy=(analysis.n_train, 0.0),
+                    xycoords=("data", "axes fraction"), xytext=(offset, 7),
+                    textcoords="offset points", ha=side, va="bottom",
+                    color=GUIDE, fontsize=8)
+
+    ax.set_xscale("log")
+    ax.set_xlim(min(grid) * analysis.n_train / 1.05, max(grid) * analysis.n_train * 1.05)
+    ax.xaxis.set_major_locator(LogLocator(subs=(1, 2, 3, 5)))
+    ax.xaxis.set_minor_locator(NullLocator())
+    ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:,.0f}"))
+
+    ax.grid(axis="y", color="#e5e7eb", lw=0.8)
+    ax.set_axisbelow(True)
+    for side in ("top", "right"):
+        ax.spines[side].set_visible(False)
+    for side in ("left", "bottom"):
+        ax.spines[side].set_color(GUIDE)
+    ax.tick_params(colors="#475569", length=4)
+
+    ax.set_xlabel("training examples", color="#334155")
+    ax.set_ylabel(_score_name(analysis.score), color="#334155")
+    ax.set_title(f"{analysis.law!r} fitted to {analysis.n_train} examples, "
+                 f"projected to {round(analysis.n_train * factor):,}",
+                 loc="left", color="#1e293b", fontsize=11, pad=14)
+    ax.legend(frameon=False, fontsize=9, loc="upper right",
+              handlelength=1.6, borderaxespad=0)
+    return ax

measly/sweep.py

@@ -0,0 +1,97 @@
+"""Measure performance across a grid of training-set sizes."""
+
+import os
+from collections.abc import Mapping, Sequence
+from concurrent.futures import ProcessPoolExecutor
+
+import numpy as np
+import xarray as xr
+
+from measly.interfaces import Score, Model
+
+__all__ = ["train_test_split", "sweep"]
+
+
+def train_test_split(X, y, test_fraction: float, rng: np.random.Generator) -> tuple:
+
+    order = rng.permutation(len(X))
+    n_test = int(round(len(y) * test_fraction))
+    test, train = order[:n_test], order[n_test:]
+
+    return X[train], X[test], y[train], y[test]
+
+
+def _score_draw(task) -> np.ndarray:
+    """One draw's scores, `(fraction, model)`. Top level so a pool can pickle it."""
+    models, score, train_X, train_y, test_X, test_y, subsets = task
+    scores = np.empty((len(subsets), len(models)))
+    for m, model in enumerate(models):
+        for f, indices in enumerate(subsets):
+            model.fit(train_X[indices], train_y[indices])
+            scores[f, m] = score(model.predict(test_X), test_y)
+    return scores
+
+
+def sweep(
+    models: Mapping[str, Model] | Sequence[Model],
+    X,
+    y,
+    fractions: list[float],
+    rng: np.random.Generator,
+    score: Score,
+    n_draws: int = 1000,
+    test_fraction: float = 0.25,
+    n_jobs: int = 1,
+) -> xr.DataArray:
+    """Score every model at every fraction of the training pool, `n_draws` times.
+
+    Only the training pool is downsampled. The test set is fixed within a draw
+    and redrawn between draws. Models MUST be scored on held-out rows. Scoring
+    on the training rows makes the curve rise with n rather than fall.
+
+    Subsets are drawn without replacement. Fraction 1.0 still varies across
+    draws, because each draw resplits the pool. A bootstrap is not needed for
+    that spread and biases the level up by repeating rows.
+
+    `models` MAY be a `{name: model}` mapping, which then keys the `model`
+    axis. Otherwise `repr` does, and an sklearn `Pipeline`'s embeds an address.
+    Those keys MUST be distinct, or one model would overwrite another.
+
+    `n_jobs` > 1 scores draws in that many processes, -1 in all cores. Results
+    do not change. `score` and the models MUST be picklable, so no lambdas.
+    Set `OMP_NUM_THREADS=1` or the workers' BLAS threads will contend.
+    """
+
+    named = (dict(models) if isinstance(models, Mapping)
+             else {repr(model): model for model in models})
+    if len(named) != len(models):
+        raise ValueError(
+            "models must have distinct repr(). The results array keys its model "
+            "axis by repr, so duplicates would overwrite each other. Pass a "
+            "{name: model} mapping to name them."
+        )
+
+    def tasks():
+        for _ in range(n_draws):
+            train_X, test_X, train_y, test_y = train_test_split(X, y, test_fraction, rng)
+            subsets = [
+                rng.choice(len(train_X), size=round(len(train_X) * fraction), replace=False)
+                for fraction in fractions
+            ]
+            yield list(named.values()), score, train_X, train_y, test_X, test_y, subsets
+
+    if n_jobs == 1:
+        per_draw = map(_score_draw, tasks())
+    else:
+        # ponytail: holds all draws' data at once. Ship indices if the pool is large.
+        workers = (os.cpu_count() or 1) if n_jobs == -1 else n_jobs
+        with ProcessPoolExecutor(workers) as pool:
+            per_draw = list(pool.map(_score_draw, tasks(),
+                                     chunksize=max(1, n_draws // (4 * workers))))
+    scores = np.stack(list(per_draw), axis=-1)
+
+    return xr.DataArray(
+        scores,
+        dims=("fraction", "model", "draw"),
+        coords={"fraction": fractions, "model": list(named), "draw": np.arange(n_draws)},
+    )

measly-0.1.0.dist-info/METADATA

@@ -0,0 +1,122 @@
+Metadata-Version: 2.5
+Name: measly
+Version: 0.1.0
+Summary: Estimate whether a model is data-limited, and whether more capacity would pay off.
+Project-URL: Homepage, https://github.com/tremgan/measly
+Project-URL: Source, https://github.com/tremgan/measly
+Project-URL: Issues, https://github.com/tremgan/measly/issues
+Author-email: Remi Tregan <tregan.remi@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: active-learning,learning-curve,sample-size,scaling-law
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Requires-Dist: matplotlib>=3.8
+Requires-Dist: numpy>=1.26
+Requires-Dist: scipy>=1.17.1
+Requires-Dist: xarray>=2026.9.0
+Description-Content-Type: text/markdown
+
+# measly🍽️
+
+More data helps. measly estimates how much.
+
+measly estimates whether a model is data-limited, meaning whether measuring more samples would improve its performance. It is built for fields where each measurement is slow and expensive, such as the life sciences.
+
+Note that measly's sweep trains `'n_draws' * 'n_models' * 'n_fractions'` models, which can be extremely computationally expensive. The premise is that this is (should be) still cheaper than collecting new data. If this is not the case, measly isn't appropriate for your use case.
+
+## Problem Definition 
+
+measly fits a model to many random subsets of the dataset at different fractions of its size. Each draw gives one learning curve, and each curve is fitted with a scaling law. The default is `pow4`:
+
+$$L(n) = L_\infty + A\,(n + d)^{-\alpha}$$
+
+`pow3`, the three-parameter form most of the literature quotes, is `pow4`
+with $d$ fixed at zero:
+
+$$L(n) = L_\infty + A\,n^{-\alpha}$$
+
+
+![a learning curve for two model capacities](docs/learning-curve.png)
+
+At 75 examples the 75-feature model is more than three times worse. At 750 it
+is 1% better. The curves cross near 400.
+
+A pilot study hides that crossover. Measured only at 75 examples, you would
+have picked the small model and been wrong about every later round. A model's
+rank at one sample size says little about its rank at the next.
+
+Both gains are small but positive, and the larger model gains about 40% more.
+So collecting more data helps a little, and mostly the high-capacity model.
+
+The gain interval is far tighter than the loss interval. Every size
+in a draw is scored on the same test set, so that draw's whole curve shifts up
+or down together. The shift dominates the loss band and cancels in the gain,
+because each curve is compared against its own level, not an average over
+draws.
+
+## Does it work?
+
+The example data is synthetic, so the projection can be checked against the
+truth. [`examples/capacity.py`](examples/capacity.py) draws extra rows from the
+same process, refits at four sizes past the measured range, and scores on
+20,000 held-out rows. These are the open diamonds in the figure. The larger
+sets extend the pilot, as collecting more data would. Only the new rows are
+redrawn across the ten repeats.
+
+```
+               model  examples            actual  projected           interval   error
+  ridge(20 features)       938   1.1592 +-0.0027     1.0795   [0.9534, 1.2388]    6.9%
+  ridge(20 features)      1125   1.1539 +-0.0047     1.0762   [0.9357, 1.2321]    6.7%
+  ridge(20 features)      1312   1.1519 +-0.0047     1.0737   [0.9322, 1.2308]    6.8%
+  ridge(20 features)      1500   1.1499 +-0.0042     1.0717   [0.9312, 1.2301]    6.8%
+  ridge(75 features)       938   1.0736 +-0.0091     1.1035   [0.9201, 1.3113]    2.8%
+  ridge(75 features)      1125   1.0594 +-0.0090     1.1006   [0.9139, 1.3095]    3.9%
+  ridge(75 features)      1312   1.0515 +-0.0091     1.0963   [0.9079, 1.3087]    4.3%
+  ridge(75 features)      1500   1.0466 +-0.0057     1.0961   [0.9040, 1.3084]    4.7%
+```
+
+All eight land inside their intervals, never more than 6.9% from the median.
+
+The projected loss shows no consistent bias. Over eight pilots the projection
+at 2x sat above what actually happened in 3 of 8 for the nearly flat model and
+5 of 8 for the steeply descending one. The mean error was +0.4% and +2.4%, and
+all sixteen intervals held the truth. The scatter comes from scoring on a slice
+of your own pilot.
+
+The audit stops at twice the measured pool. The held-back check fits on the
+smaller sizes and predicts the largest two, a stretch of about 1.9x, so that
+is as far as the method carries evidence. Projecting ten times out would
+return a number and support none of it.
+
+The error bars on the diamonds are smaller than the markers. Ten independent
+collections at one size vary by 0.0027 to 0.0091, while the band spans about
+0.3, so the band is 30 to 100 times wider. That is the safe direction to err
+in. Most of the width is test-set noise, not uncertainty about the curve.
+
+Subsets are drawn without replacement. Each draw resplits the pool, so even the
+full-size fit varies across draws. A bootstrap would add little to that spread
+and would bias the level up, because a resample repeats rows.
+
+`analyse` always checks itself. It refits the curves on the smaller sample
+sizes, predicts the largest ones you already measured, and reports the error.
+Above 15% the projection MUST NOT be relied on: the shape fits your data but
+does not predict it.
+
+`analyse(...).results` is an `xarray.DataArray` over `(fraction, model, draw)`
+if you want the raw measurements.
+
+## Previous work in this area/Inspiration
+
+- [The Shape of Learning Curves: a Review](https://arxiv.org/abs/2103.10948). Viering & Loog, 2021.
+- [Deep Learning Scaling is Predictable, Empirically](https://arxiv.org/abs/1712.00409). Hestness et al., 2017.
+- [Scaling Laws for Neural Language Models](https://arxiv.org/abs/2001.08361). Kaplan et al., 2020.
+- [Training Compute-Optimal Large Language Models](https://arxiv.org/abs/2203.15556). Hoffmann et al., 2022.
+- [Revisiting Neural Scaling Laws in Language and Vision](https://arxiv.org/abs/2209.06640). Alabdulmohsin et al., 2022.
+- [Broken Neural Scaling Laws](https://github.com/ethancaballero/broken_neural_scaling_laws). Caballero et al., 2022.
+- [How Much More Data Do I Need?](https://research.nvidia.com/labs/toronto-ai/estimatingrequirements/). Mahmood et al., 2022.
+- [Estimation of Predictive Performance in High-Dimensional Data Settings using Learning Curves](https://arxiv.org/abs/2206.03825). Goedhart et al., 2022.

measly-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+measly/__init__.py,sha256=YqjYyhpvPHzh_IQHABbyxb-MGSiKAAiMOXSiFVuHgFk,824
+measly/analysis.py,sha256=vA2gtltqK4Wuz9PCF7inMfruClgFZWkMGSV9kycv5EI,10327
+measly/fit.py,sha256=SZ362l8Td5Bw_qj-2VFcGmcpy_xVcdZfOePwFFNpN_c,4722
+measly/interfaces.py,sha256=S65JCnXymA8mzoHPxQZrNwquiC3hq1WnoZiVIu-XwBI,1143
+measly/plot.py,sha256=8-JZKUEecyQhjpqj557TuyiVVre2NOVe2iHnZZyvO_c,3670
+measly/sweep.py,sha256=1UjTRsZrzMFqIEPfr94vVCqa6xOuTc0IKl9Y2NnpCv4,3726
+measly-0.1.0.dist-info/METADATA,sha256=Oz2yDTJ-Y0BFJLhPoPgR2KUKUmNGZPyQewDXFnc9NfI,6521
+measly-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+measly-0.1.0.dist-info/licenses/LICENSE,sha256=IQO1FLCH5dypLlWDpWMdoQe4U2xcERhaZAOeeF6fKGE,1068
+measly-0.1.0.dist-info/RECORD,,

measly-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

measly-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Remi Tregan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.