measly 0.1.0__tar.gz

measly-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,22 @@
+name: publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read    # a permissions block zeroes every scope not listed
+      id-token: write   # mints the OIDC token PyPI trades for upload rights
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      # The release tag and pyproject.toml are two sources of truth for the
+      # version. PyPI rejects a duplicate upload, so catch a mismatch here.
+      - if: github.event_name == 'release'   # on dispatch the ref is a branch
+        run: grep -q "^version = \"${GITHUB_REF_NAME#v}\"$" pyproject.toml
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1

measly-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: test
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - run: uv sync --group dev --locked
+      - run: uv run pytest tests -q

measly-0.1.0/.gitignore

@@ -0,0 +1,14 @@
+.vibe-wise
+data/
+__pycache__/
+*.py[cod]
+.venv/
+*.egg-info/
+playground/
+.ruff_cache
+
+demo/
+.pytest_cache/
+dist/
+
+CLAUDE.md

measly-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Remi Tregan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

measly-0.1.0/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.5
+Name: measly
+Version: 0.1.0
+Summary: Estimate whether a model is data-limited, and whether more capacity would pay off.
+Project-URL: Homepage, https://github.com/tremgan/measly
+Project-URL: Source, https://github.com/tremgan/measly
+Project-URL: Issues, https://github.com/tremgan/measly/issues
+Author-email: Remi Tregan <tregan.remi@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: active-learning,learning-curve,sample-size,scaling-law
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Requires-Dist: matplotlib>=3.8
+Requires-Dist: numpy>=1.26
+Requires-Dist: scipy>=1.17.1
+Requires-Dist: xarray>=2026.9.0
+Description-Content-Type: text/markdown
+
+# measly🍽️
+
+More data helps. measly estimates how much.
+
+measly estimates whether a model is data-limited, meaning whether measuring more samples would improve its performance. It is built for fields where each measurement is slow and expensive, such as the life sciences.
+
+Note that measly's sweep trains `'n_draws' * 'n_models' * 'n_fractions'` models, which can be extremely computationally expensive. The premise is that this is (should be) still cheaper than collecting new data. If this is not the case, measly isn't appropriate for your use case.
+
+## Problem Definition 
+
+measly fits a model to many random subsets of the dataset at different fractions of its size. Each draw gives one learning curve, and each curve is fitted with a scaling law. The default is `pow4`:
+
+$$L(n) = L_\infty + A\,(n + d)^{-\alpha}$$
+
+`pow3`, the three-parameter form most of the literature quotes, is `pow4`
+with $d$ fixed at zero:
+
+$$L(n) = L_\infty + A\,n^{-\alpha}$$
+
+
+![a learning curve for two model capacities](docs/learning-curve.png)
+
+At 75 examples the 75-feature model is more than three times worse. At 750 it
+is 1% better. The curves cross near 400.
+
+A pilot study hides that crossover. Measured only at 75 examples, you would
+have picked the small model and been wrong about every later round. A model's
+rank at one sample size says little about its rank at the next.
+
+Both gains are small but positive, and the larger model gains about 40% more.
+So collecting more data helps a little, and mostly the high-capacity model.
+
+The gain interval is far tighter than the loss interval. Every size
+in a draw is scored on the same test set, so that draw's whole curve shifts up
+or down together. The shift dominates the loss band and cancels in the gain,
+because each curve is compared against its own level, not an average over
+draws.
+
+## Does it work?
+
+The example data is synthetic, so the projection can be checked against the
+truth. [`examples/capacity.py`](examples/capacity.py) draws extra rows from the
+same process, refits at four sizes past the measured range, and scores on
+20,000 held-out rows. These are the open diamonds in the figure. The larger
+sets extend the pilot, as collecting more data would. Only the new rows are
+redrawn across the ten repeats.
+
+```
+               model  examples            actual  projected           interval   error
+  ridge(20 features)       938   1.1592 +-0.0027     1.0795   [0.9534, 1.2388]    6.9%
+  ridge(20 features)      1125   1.1539 +-0.0047     1.0762   [0.9357, 1.2321]    6.7%
+  ridge(20 features)      1312   1.1519 +-0.0047     1.0737   [0.9322, 1.2308]    6.8%
+  ridge(20 features)      1500   1.1499 +-0.0042     1.0717   [0.9312, 1.2301]    6.8%
+  ridge(75 features)       938   1.0736 +-0.0091     1.1035   [0.9201, 1.3113]    2.8%
+  ridge(75 features)      1125   1.0594 +-0.0090     1.1006   [0.9139, 1.3095]    3.9%
+  ridge(75 features)      1312   1.0515 +-0.0091     1.0963   [0.9079, 1.3087]    4.3%
+  ridge(75 features)      1500   1.0466 +-0.0057     1.0961   [0.9040, 1.3084]    4.7%
+```
+
+All eight land inside their intervals, never more than 6.9% from the median.
+
+The projected loss shows no consistent bias. Over eight pilots the projection
+at 2x sat above what actually happened in 3 of 8 for the nearly flat model and
+5 of 8 for the steeply descending one. The mean error was +0.4% and +2.4%, and
+all sixteen intervals held the truth. The scatter comes from scoring on a slice
+of your own pilot.
+
+The audit stops at twice the measured pool. The held-back check fits on the
+smaller sizes and predicts the largest two, a stretch of about 1.9x, so that
+is as far as the method carries evidence. Projecting ten times out would
+return a number and support none of it.
+
+The error bars on the diamonds are smaller than the markers. Ten independent
+collections at one size vary by 0.0027 to 0.0091, while the band spans about
+0.3, so the band is 30 to 100 times wider. That is the safe direction to err
+in. Most of the width is test-set noise, not uncertainty about the curve.
+
+Subsets are drawn without replacement. Each draw resplits the pool, so even the
+full-size fit varies across draws. A bootstrap would add little to that spread
+and would bias the level up, because a resample repeats rows.
+
+`analyse` always checks itself. It refits the curves on the smaller sample
+sizes, predicts the largest ones you already measured, and reports the error.
+Above 15% the projection MUST NOT be relied on: the shape fits your data but
+does not predict it.
+
+`analyse(...).results` is an `xarray.DataArray` over `(fraction, model, draw)`
+if you want the raw measurements.
+
+## Previous work in this area/Inspiration
+
+- [The Shape of Learning Curves: a Review](https://arxiv.org/abs/2103.10948). Viering & Loog, 2021.
+- [Deep Learning Scaling is Predictable, Empirically](https://arxiv.org/abs/1712.00409). Hestness et al., 2017.
+- [Scaling Laws for Neural Language Models](https://arxiv.org/abs/2001.08361). Kaplan et al., 2020.
+- [Training Compute-Optimal Large Language Models](https://arxiv.org/abs/2203.15556). Hoffmann et al., 2022.
+- [Revisiting Neural Scaling Laws in Language and Vision](https://arxiv.org/abs/2209.06640). Alabdulmohsin et al., 2022.
+- [Broken Neural Scaling Laws](https://github.com/ethancaballero/broken_neural_scaling_laws). Caballero et al., 2022.
+- [How Much More Data Do I Need?](https://research.nvidia.com/labs/toronto-ai/estimatingrequirements/). Mahmood et al., 2022.
+- [Estimation of Predictive Performance in High-Dimensional Data Settings using Learning Curves](https://arxiv.org/abs/2206.03825). Goedhart et al., 2022.

measly-0.1.0/README.md

@@ -0,0 +1,99 @@
+# measly🍽️
+
+More data helps. measly estimates how much.
+
+measly estimates whether a model is data-limited, meaning whether measuring more samples would improve its performance. It is built for fields where each measurement is slow and expensive, such as the life sciences.
+
+Note that measly's sweep trains `'n_draws' * 'n_models' * 'n_fractions'` models, which can be extremely computationally expensive. The premise is that this is (should be) still cheaper than collecting new data. If this is not the case, measly isn't appropriate for your use case.
+
+## Problem Definition 
+
+measly fits a model to many random subsets of the dataset at different fractions of its size. Each draw gives one learning curve, and each curve is fitted with a scaling law. The default is `pow4`:
+
+$$L(n) = L_\infty + A\,(n + d)^{-\alpha}$$
+
+`pow3`, the three-parameter form most of the literature quotes, is `pow4`
+with $d$ fixed at zero:
+
+$$L(n) = L_\infty + A\,n^{-\alpha}$$
+
+
+![a learning curve for two model capacities](docs/learning-curve.png)
+
+At 75 examples the 75-feature model is more than three times worse. At 750 it
+is 1% better. The curves cross near 400.
+
+A pilot study hides that crossover. Measured only at 75 examples, you would
+have picked the small model and been wrong about every later round. A model's
+rank at one sample size says little about its rank at the next.
+
+Both gains are small but positive, and the larger model gains about 40% more.
+So collecting more data helps a little, and mostly the high-capacity model.
+
+The gain interval is far tighter than the loss interval. Every size
+in a draw is scored on the same test set, so that draw's whole curve shifts up
+or down together. The shift dominates the loss band and cancels in the gain,
+because each curve is compared against its own level, not an average over
+draws.
+
+## Does it work?
+
+The example data is synthetic, so the projection can be checked against the
+truth. [`examples/capacity.py`](examples/capacity.py) draws extra rows from the
+same process, refits at four sizes past the measured range, and scores on
+20,000 held-out rows. These are the open diamonds in the figure. The larger
+sets extend the pilot, as collecting more data would. Only the new rows are
+redrawn across the ten repeats.
+
+```
+               model  examples            actual  projected           interval   error
+  ridge(20 features)       938   1.1592 +-0.0027     1.0795   [0.9534, 1.2388]    6.9%
+  ridge(20 features)      1125   1.1539 +-0.0047     1.0762   [0.9357, 1.2321]    6.7%
+  ridge(20 features)      1312   1.1519 +-0.0047     1.0737   [0.9322, 1.2308]    6.8%
+  ridge(20 features)      1500   1.1499 +-0.0042     1.0717   [0.9312, 1.2301]    6.8%
+  ridge(75 features)       938   1.0736 +-0.0091     1.1035   [0.9201, 1.3113]    2.8%
+  ridge(75 features)      1125   1.0594 +-0.0090     1.1006   [0.9139, 1.3095]    3.9%
+  ridge(75 features)      1312   1.0515 +-0.0091     1.0963   [0.9079, 1.3087]    4.3%
+  ridge(75 features)      1500   1.0466 +-0.0057     1.0961   [0.9040, 1.3084]    4.7%
+```
+
+All eight land inside their intervals, never more than 6.9% from the median.
+
+The projected loss shows no consistent bias. Over eight pilots the projection
+at 2x sat above what actually happened in 3 of 8 for the nearly flat model and
+5 of 8 for the steeply descending one. The mean error was +0.4% and +2.4%, and
+all sixteen intervals held the truth. The scatter comes from scoring on a slice
+of your own pilot.
+
+The audit stops at twice the measured pool. The held-back check fits on the
+smaller sizes and predicts the largest two, a stretch of about 1.9x, so that
+is as far as the method carries evidence. Projecting ten times out would
+return a number and support none of it.
+
+The error bars on the diamonds are smaller than the markers. Ten independent
+collections at one size vary by 0.0027 to 0.0091, while the band spans about
+0.3, so the band is 30 to 100 times wider. That is the safe direction to err
+in. Most of the width is test-set noise, not uncertainty about the curve.
+
+Subsets are drawn without replacement. Each draw resplits the pool, so even the
+full-size fit varies across draws. A bootstrap would add little to that spread
+and would bias the level up, because a resample repeats rows.
+
+`analyse` always checks itself. It refits the curves on the smaller sample
+sizes, predicts the largest ones you already measured, and reports the error.
+Above 15% the projection MUST NOT be relied on: the shape fits your data but
+does not predict it.
+
+`analyse(...).results` is an `xarray.DataArray` over `(fraction, model, draw)`
+if you want the raw measurements.
+
+## Previous work in this area/Inspiration
+
+- [The Shape of Learning Curves: a Review](https://arxiv.org/abs/2103.10948). Viering & Loog, 2021.
+- [Deep Learning Scaling is Predictable, Empirically](https://arxiv.org/abs/1712.00409). Hestness et al., 2017.
+- [Scaling Laws for Neural Language Models](https://arxiv.org/abs/2001.08361). Kaplan et al., 2020.
+- [Training Compute-Optimal Large Language Models](https://arxiv.org/abs/2203.15556). Hoffmann et al., 2022.
+- [Revisiting Neural Scaling Laws in Language and Vision](https://arxiv.org/abs/2209.06640). Alabdulmohsin et al., 2022.
+- [Broken Neural Scaling Laws](https://github.com/ethancaballero/broken_neural_scaling_laws). Caballero et al., 2022.
+- [How Much More Data Do I Need?](https://research.nvidia.com/labs/toronto-ai/estimatingrequirements/). Mahmood et al., 2022.
+- [Estimation of Predictive Performance in High-Dimensional Data Settings using Learning Curves](https://arxiv.org/abs/2206.03825). Goedhart et al., 2022.

measly-0.1.0/examples/capacity.py

@@ -0,0 +1,127 @@
+"""More capacity is worse on a pilot and better once the data arrives.
+
+The small model is misspecified: it sees a subset of the features, so it has a
+floor above the noise. The large model is unbiased but pays variance early.
+The curves therefore cross, and which model is "better" depends entirely on
+how much data you have.
+
+Because the data is synthetic the projection can be audited rather than
+trusted. The extra rows are drawn from the same process and the models
+refitted on them, so the figure carries both what measly predicted and what
+actually happened.
+
+    uv run --group demo python examples/capacity.py
+
+Writes docs/learning-curve.png.
+"""
+
+import numpy as np
+from matplotlib.lines import Line2D
+from sklearn.compose import ColumnTransformer
+from sklearn.linear_model import Ridge
+from sklearn.pipeline import Pipeline
+
+from measly import analyse, mean_squared_error, plot
+from measly.plot import GUIDE, PALETTE
+
+TOTAL, CAPACITY = 75, 20
+PILOT, NOISE = 3000, 1.0
+TAIL = 0.05
+"""Weight on every feature the small model omits.
+
+It alone sets where the curves cross, at `NOISE**2 / TAIL**2` examples. The
+feature counts set how steep the large model's descent is and how wide the
+final gap is, without moving the crossing.
+"""
+
+FACTORS = (1.25, 1.5, 1.75, 2.0)
+"""Multiples of the measured pool to audit the projection at.
+
+Capped at 2x on purpose. The held-back check fits on the smaller sizes and
+predicts the largest two, a stretch of `1.0 / 0.5179`, so roughly 1.9x is as
+far as the method carries evidence. Auditing past that measures nothing the
+tool claims.
+"""
+
+REPEATS = 10
+"""Independent collections at each size, so the audit carries its own spread."""
+
+
+def ridge(capacity):
+    """Ridge that sees only the first `capacity` features."""
+    return Pipeline([
+        ("capacity",
+         ColumnTransformer([("keep", "passthrough", slice(0, capacity))])),
+        ("ridge", Ridge()),
+    ])
+
+
+rng = np.random.default_rng(0)
+weights = np.concatenate([rng.normal(size=4), np.full(TOTAL - 4, TAIL)])
+
+
+def sample(n):
+    X = rng.normal(size=(n, TOTAL))
+    return X, X @ weights + rng.normal(0.0, NOISE, n)
+
+
+# Named, because a Pipeline's repr embeds a memory address.
+models = {f"ridge({capacity} features)": ridge(capacity)
+          for capacity in (CAPACITY, TOTAL)}
+
+X_pilot, y_pilot = sample(PILOT)
+result = analyse(models, X_pilot, y_pilot, rng=0)
+print(result.summary())
+print(f"\ncurves cross near {round(NOISE ** 2 / TAIL ** 2)} examples\n")
+
+# Go and collect the data measly was asked about, then check it was right.
+# A lab keeps its pilot and adds to it, so the larger sets contain the smaller
+# ones. Only the new rows are redrawn across repeats, because the pilot is not
+# something that gets collected again.
+X_check, y_check = sample(20_000)
+sizes = [round(result.n_train * factor) for factor in FACTORS]
+X_base, y_base = X_pilot[:result.n_train], y_pilot[:result.n_train]
+
+scores = {name: np.empty((REPEATS, len(sizes))) for name in models}
+for repeat in range(REPEATS):
+    X_new, y_new = sample(max(sizes) - result.n_train)
+    X_more = np.vstack([X_base, X_new])
+    y_more = np.concatenate([y_base, y_new])
+    for name, model in models.items():
+        for column, n in enumerate(sizes):
+            model.fit(X_more[:n], y_more[:n])
+            scores[name][repeat, column] = mean_squared_error(
+                model.predict(X_check), y_check)
+
+actual = {name: value.mean(axis=0) for name, value in scores.items()}
+spread = {name: value.std(axis=0, ddof=1) for name, value in scores.items()}
+
+print(f"{'model':>20} {'examples':>9} {'actual':>17} {'projected':>10}"
+      f" {'interval':>18} {'error':>7}")
+for name in models:
+    for factor, n, measured, sd in zip(FACTORS, sizes, actual[name], spread[name]):
+        projected = result.project(factor)[name]
+        print(f"{name:>20} {n:>9} {f'{measured:.4f} +-{sd:.4f}':>17}"
+              f" {projected.loss:>10.4f}"
+              f" {f'[{projected.low:.4f}, {projected.high:.4f}]':>18}"
+              f" {abs(projected.loss - measured) / measured:>7.1%}"
+              f"{'' if projected.low <= measured <= projected.high else '  OUT'}")
+
+ax = plot(result, factor=max(FACTORS))
+for index, name in enumerate(result.models):
+    colour = PALETTE[index % len(PALETTE)]
+    ax.errorbar(sizes, actual[name], yerr=spread[name], fmt="D", ms=5.5,
+                mfc="white", mec=colour, mew=1.5, color=colour,
+                elinewidth=1.1, capsize=0, zorder=6)
+
+legend = ax.get_legend_handles_labels()[0] + [
+    Line2D([], [], ls="none", marker="o", color=GUIDE, ms=5,
+           label="measured by measly"),
+    Line2D([], [], ls="none", marker="D", mfc="white", mec=GUIDE, mew=1.5,
+           ms=5.5, label="actual, after collecting more"),
+]
+ax.legend(handles=legend, frameon=False, fontsize=9, loc="upper right",
+          handlelength=1.6, borderaxespad=0)
+ax.figure.tight_layout()
+ax.figure.savefig("docs/learning-curve.png", dpi=120)
+print("\nwrote docs/learning-curve.png")

measly-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "measly"
+version = "0.1.0"
+description = "Estimate whether a model is data-limited, and whether more capacity would pay off."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+authors = [{ name = "Remi Tregan", email = "tregan.remi@gmail.com" }]
+keywords = ["learning-curve", "scaling-law", "sample-size", "active-learning"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "matplotlib>=3.8",
+    "numpy>=1.26",
+    "scipy>=1.17.1",
+    "xarray>=2026.9.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/tremgan/measly"
+Source = "https://github.com/tremgan/measly"
+Issues = "https://github.com/tremgan/measly/issues"
+
+[dependency-groups]
+dev = ["pytest>=8"]
+demo = ["scikit-learn>=1.4"]
+
+[tool.pytest.ini_options]
+pythonpath = ["tests"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/measly"]

measly-0.1.0/src/measly/__init__.py

@@ -0,0 +1,43 @@
+"""Estimate whether a model is data-limited, and whether more capacity would pay off."""
+
+from measly.analysis import (
+    DEFAULT_FRACTIONS,
+    VALIDATION_WARN,
+    Analysis,
+    Projection,
+    analyse,
+    mean_squared_error,
+)
+from measly.fit import (
+    POW3,
+    POW4,
+    CurveEnsemble,
+    ScalingLaw,
+    fit_scaling_law,
+    fit_scaling_laws,
+    pow3,
+)
+from measly.interfaces import Model, Score
+from measly.plot import plot
+from measly.sweep import sweep, train_test_split
+
+__all__ = [
+    "analyse",
+    "Analysis",
+    "Projection",
+    "mean_squared_error",
+    "plot",
+    "DEFAULT_FRACTIONS",
+    "VALIDATION_WARN",
+    "Model",
+    "Score",
+    "train_test_split",
+    "sweep",
+    "pow3",
+    "ScalingLaw",
+    "POW3",
+    "POW4",
+    "CurveEnsemble",
+    "fit_scaling_law",
+    "fit_scaling_laws",
+]