anchorboosting 0.2.0__tar.gz → 0.2.1__tar.gz

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/CHANGELOG.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+0.2.1 (2025-07-01)
+------------------
+
+- `AnchorBooster.refit` no longer raises an error if the model stopped boosting early during initial training.
+
 0.2.0 (2025-06-12)
 ------------------
 

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anchorboosting
-Version: 0.2.0
+Version: 0.2.1
 Summary: Efficient tree-based nonlinear anchor regression and classification
 Author-email: Malte Londschien <malte@londschien.ch>
 Project-URL: homepage, https://github.com/mlondschien/anchorboosting/

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/anchorboosting/models.py

@@ -161,9 +161,6 @@ class AnchorBooster:
         if self.objective != "regression" and not np.isin(y, [0, 1]).all():
             raise ValueError("For binary classification, y values must be in {0, 1}.")
 
-        if Z is None:
-            Z = np.zeros(shape=(len(y), 0))
-
         y = y.flatten()
 
         if self.objective == "regression":
@@ -198,7 +195,7 @@ class AnchorBooster:
         f = np.ones(len(y), dtype=np.float64) * self.init_score_
         booster_preds = f.copy()
 
-        Q, _ = np.linalg.qr(Z, mode="reduced")  # P_Z f = Q @ (Q^T @ f)
+        proj = Proj(Z)
 
         if self.objective == "binary":
             y_tilde = np.where(y == 1, 1, -1).astype(np.float64)
@@ -231,7 +228,7 @@ class AnchorBooster:
                 dr = -f * r + r**2  # d^2/df^2 loss(f, y)
                 ddr = (f**2 - 1) * r - 3 * f * r**2 + 2 * r**3  # d^3/df^3 loss
 
-            r_proj = Q @ (Q.T @ r)
+            r_proj = proj(r)
             grad = r + (self.gamma - 1) * r_proj * dr
 
             # We wish to fit one additional tree. Intuitively, one would use
@@ -348,19 +345,8 @@ class AnchorBooster:
             counts += self.params.get("lambda_l2", 0)
             counts[counts == 0] = 1  # counts==0 should never happen.
 
-            # Mdr^T P_Z Mdr = (Mdr^T Q) @ (Mdr^T Q)^T
-            # One could also compute this using bincount, but it appears this
-            # version using a sparse matrix is faster.
-            Mdr = scipy.sparse.csr_matrix(
-                (
-                    dr,
-                    (np.arange(len(leaves)), leaves),
-                ),
-                shape=(len(leaves), num_leaves),
-                dtype=np.float64,
-            )
-            B = Mdr.T.dot(Q)
-            H = (self.gamma - 1) * B @ B.T
+            # (dr * M)^T P_Z (dr * M)
+            H = (self.gamma - 1) * proj.sandwich(leaves, num_leaves, weights=dr)
             H += np.diag(counts)
 
             # Compute the 2nd order update
@@ -449,7 +435,8 @@ class AnchorBooster:
 
         f = np.full(len(y), self.init_score_, dtype="float64")
 
-        for idx in range(self.num_boost_round):
+        # The model might have stopped boosting prematurely.
+        for idx in range(leaves.shape[1]):
             n_obs = np.bincount(leaves[:, idx], minlength=num_leaves[idx])
 
             if self.objective == "regression":
@@ -505,3 +492,101 @@ def _suppress_stderr():
         os.dup2(old_stderr_fd, stderr_fd)  # put the old stderr back
         os.close(old_stderr_fd)
         os.close(devnull_fd)
+
+
+class Proj:
+    """
+    Cache the projection onto the subspace spanned by Z.
+
+    Parameters
+    ----------
+    Z: np.ndarray of dimension (n, d_Z) or (n,), optional, default=None
+        The `Z` matrix or 1d array of integers.
+    """
+
+    def __init__(self, Z):
+        if Z is None:
+            self._type = "none"
+
+        elif len(Z.shape) == 1 and np.issubdtype(Z.dtype, np.integer):
+            self._type = "categorical"
+            self._Z = Z
+            self._n_categories = np.max(Z) + 1
+            counts = np.bincount(Z, minlength=self._n_categories)
+            counts[counts == 0] = 1.0  # Avoid division by zero
+            self._one_over_counts = 1 / counts
+
+        elif len(Z.shape) == 2 and np.issubdtype(Z.dtype, np.floating):
+            self._type = "linear"
+            self._Q, _ = np.linalg.qr(Z, mode="reduced")
+        else:
+            raise ValueError(
+                "Z should be either a 1d array of integers or a 2d array of floats. "
+                f"Got shape {Z.shape} and dtype {Z.dtype}."
+            )
+
+    def __call__(self, f):
+        """
+        Project the input array `f` onto the subspace spanned by `Z`.
+
+        Parameters
+        ----------
+        f: np.ndarray of shape (n,)
+            The input array to project.
+
+        Returns
+        -------
+        np.ndarray of shape (n,)
+            The projected array.
+        """
+        if self._type == "none":
+            return np.zeros_like(f)
+
+        elif self._type == "categorical":
+            sums = np.bincount(self._Z, weights=f, minlength=self._n_categories)
+            means = sums * self._one_over_counts
+            return means[self._Z]
+
+        elif self._type == "linear":
+            return self._Q @ (self._Q.T @ f)
+
+    def sandwich(self, leaves, num_leaves, weights):
+        """
+        For M = weights * one_hot(leaves), return proj(Z, M).T @ proj(Z, M).
+
+        Parameters
+        ----------
+        leaves: np.ndarray of shape (n,)
+            The leaf indices for each sample in `f`. Integers in [0, num_leaves).
+        num_leaves: int
+            The number of leaves in the decision tree.
+        weights: np.ndarray of shape (n,)
+            The input array to project.
+
+        Returns
+        -------
+        np.ndarray of shape (d, d)
+            The sandwich product.
+        """
+        if self._type == "none":
+            return np.zeros((num_leaves, num_leaves), dtype=weights.dtype)
+
+        elif self._type == "categorical":
+            S = np.zeros((self._n_categories, num_leaves), dtype=weights.dtype)
+            np.add.at(S, (self._Z, leaves), weights)
+            return (S * self._one_over_counts[:, np.newaxis]).T @ S
+
+        elif self._type == "linear":
+            # M^T P_Z M = (M^T Q) @ (M^T Q)^T
+            # One could also compute this using bincount, but it appears this
+            # version using a sparse matrix is faster.
+            M = scipy.sparse.csr_matrix(
+                (
+                    weights,
+                    (np.arange(len(leaves)), leaves),
+                ),
+                shape=(len(leaves), num_leaves),
+                dtype=weights.dtype,
+            )
+            B = M.T @ self._Q
+            return B @ B.T

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/anchorboosting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anchorboosting
-Version: 0.2.0
+Version: 0.2.1
 Summary: Efficient tree-based nonlinear anchor regression and classification
 Author-email: Malte Londschien <malte@londschien.ch>
 Project-URL: homepage, https://github.com/mlondschien/anchorboosting/

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/anchorboosting.egg-info/SOURCES.txt

@@ -11,7 +11,6 @@ readthedocs.yaml
 anchorboosting/__init__.py
 anchorboosting/models.py
 anchorboosting/simulate.py
-anchorboosting/utils.py
 anchorboosting.egg-info/PKG-INFO
 anchorboosting.egg-info/SOURCES.txt
 anchorboosting.egg-info/dependency_links.txt
@@ -24,5 +23,5 @@ docs/conf.py
 docs/index.rst
 tests/test_anchor_boosting.py
 tests/test_derivatives.py
-tests/test_refit.py
-tests/test_utils.py
+tests/test_proj.py
+tests/test_refit.py

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/environment.yml

@@ -9,7 +9,6 @@ dependencies:
   - lightgbm >=4.0.0
   - setuptools  # build
   - pip  # build
-  - scikit-learn  # tests
   - pytest-cov  # tests
   - pytest  # tests
   - polars  # tests

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/pyproject.toml

@@ -5,7 +5,7 @@ requires = ['setuptools', 'setuptools-scm', 'wheel']
 name = "anchorboosting"
 description = "Efficient tree-based nonlinear anchor regression and classification"
 readme = "README.md"
-version = "0.2.0"
+version = "0.2.1"
 requires-python = ">=3.9"
 authors = [
     { name = "Malte Londschien", email = "malte@londschien.ch" }

{anchorboosting-0.2.0 → anchorboosting-0.2.1}/tests/test_anchor_boosting.py

@@ -3,19 +3,23 @@ import numpy as np
 import pytest
 import scipy
 
-from anchorboosting import AnchorBooster
+from anchorboosting.models import AnchorBooster, Proj
 from anchorboosting.simulate import f1, simulate
 
 
 @pytest.mark.parametrize("gamma", [1.0, 2.0, 100])
 @pytest.mark.parametrize("objective", ["regression", "binary"])
-def test_anchor_boosting_second_order(gamma, objective):
+@pytest.mark.parametrize("categorical_z", [True, False])
+def test_anchor_boosting_second_order(gamma, objective, categorical_z):
     learning_rate = 0.1
     num_leaves = 5
     n = 200
     num_boost_round = 10
 
     x, y, a = simulate(f1, n=n, shift=0, seed=0)
+    if categorical_z:
+        a = np.digitize(a[:, 0], bins=[-1, 0, 1, 2, 3])
+        assert np.issubdtype(a.dtype, np.integer)
 
     if objective == "binary":
         y = (y > 0).astype(int)
@@ -38,9 +42,11 @@ def test_anchor_boosting_second_order(gamma, objective):
         num_iteration=1,
     ).flatten()
 
+    proj = Proj(a)
+
     def regression_loss(leaf_values):
         residuals = y - f - leaf_values[leaves]
-        Pa_residuals = a @ np.linalg.solve(a.T @ a, a.T @ residuals)
+        Pa_residuals = proj(residuals)
         return np.sum(np.square(residuals)) + (gamma - 1) * residuals.T @ Pa_residuals
 
     def probit_loss(leaf_values):
@@ -49,7 +55,7 @@ def test_anchor_boosting_second_order(gamma, objective):
         dp = scipy.stats.norm.pdf(scores)
         losses = -np.log(np.where(y == 1, p, 1 - p))
         dl = np.where(y == 1, -dp / p, dp / (1 - p))
-        Pa_dl = a @ np.linalg.solve(a.T @ a, a.T @ dl)
+        Pa_dl = proj(dl)
         return np.sum(losses) + (gamma - 1) / 2 * dl.T @ Pa_dl
 
     if objective == "regression":
@@ -176,18 +182,32 @@ def test_compare_anchor_boosting_to_lgbm(parameters):
 
 
 @pytest.mark.parametrize("objective", ["regression", "binary"])
-def test_anchor_booster_init_score(objective):
+def test_categorical_and_onehot_is_the_same(objective):
     X, y, a = simulate(f1, shift=0, seed=0)
 
+    a = np.digitize(a[:, 0], bins=[-1, 0, 1, 2, 3])
+
     if objective == "binary":
         y = (y > 0).astype(int)
 
-    anchor_booster = AnchorBooster(
+    anchor_booster1 = AnchorBooster(
         gamma=1,
-        num_boost_round=0,
+        num_boost_round=10,
         objective=objective,
         learning_rate=0.1,
     ).fit(X, y, Z=a)
 
-    predictions = anchor_booster.predict(X)
-    assert np.allclose(predictions, np.ones_like(y) * y.mean())
+    predictions1 = anchor_booster1.predict(X)
+
+    M = scipy.sparse.csr_matrix(
+        (np.ones(len(a), dtype=np.float64), (np.arange(len(a)), a)),
+        shape=(len(a), len(np.unique(a))),
+    )
+    anchor_booster2 = AnchorBooster(
+        gamma=1,
+        num_boost_round=10,
+        objective=objective,
+        learning_rate=0.1,
+    ).fit(X, y, Z=M.toarray())
+    predictions2 = anchor_booster2.predict(X)
+    np.testing.assert_allclose(predictions1, predictions2, rtol=1e-5)

anchorboosting-0.2.1/tests/test_proj.py

@@ -0,0 +1,67 @@
+import numpy as np
+import pytest
+import scipy.sparse
+
+from anchorboosting.models import Proj
+
+cases = [
+    (
+        np.array([1, 1, 3]),
+        np.array([1.0, 2.0, 3.0]),
+        np.array([1.5, 1.5, 3.0]),
+    ),
+    (
+        np.array([[0.0], [0.0], [1.0]]),
+        np.array([1.0, 2.0, 3.0]),
+        np.array([0.0, 0.0, 3.0]),
+    ),
+    (
+        np.array([[0.0, 1.0], [0.0, 1.0], [1.0, 0.0]]),
+        np.array([1.0, 2.0, 3.0]),
+        np.array([1.5, 1.5, 3]),
+    ),
+    (
+        np.array([2, 2, 1, 7]),
+        np.array([1.0, 2.0, 3.0, 1.0]),
+        np.array([1.5, 1.5, 3.0, 1.0]),
+    ),
+    (
+        np.array([1, 1, 0]),
+        np.array([1.0, 0.0, 3.0]),
+        np.array([0.5, 0.5, 3]),
+    ),
+    (np.array([0]), np.array([1.0]), np.array([1.0])),
+    (np.array([[0.0]]), np.array([0.0]), np.array([0.0])),
+]
+
+
+@pytest.mark.parametrize("Z, f, result", cases)
+def test_cached_proj_result(Z, f, result):
+    np.testing.assert_almost_equal(Proj(Z)(f), result)
+
+
+@pytest.mark.parametrize("Z, f, _", cases)
+def test_cached_proj_dot_product(Z, f, _):
+    np.testing.assert_almost_equal(
+        np.dot(Proj(Z)(f).T, f),
+        np.dot(Proj(Z)(f).T, Proj(Z)(f)),
+    )
+
+
+def test_proj_sandwich():
+    n = 100
+    num_categories = 10
+    num_leaves = 5
+    rng = np.random.RandomState(0)
+
+    Z = rng.choice(np.arange(num_categories), size=n, replace=True)
+    leaves = rng.choice(np.arange(num_leaves), size=n, replace=True)
+    f = rng.normal(size=n)
+
+    M = scipy.sparse.csr_matrix((f, (np.arange(n), leaves)), (n, num_leaves)).toarray()
+    proj = Proj(Z)
+
+    sandwich1 = proj.sandwich(leaves, num_leaves, f)
+    for ldx in range(num_leaves):
+        M[:, ldx] = proj(M[:, ldx])
+    np.testing.assert_almost_equal(sandwich1, M.T @ M)

anchorboosting-0.2.0/anchorboosting/utils.py

@@ -1,84 +0,0 @@
-# copy-pasted from https://github.com/mlondschien/ivmodels/blob/09f8bc5a78b3793ae5\
-# 1f0bb7f0331b70a0971ebc/ivmodels/utils.py
-import numpy as np
-import scipy
-
-
-def proj(Z, f, n_categories=None, copy=False):
-    """
-    Project f onto the subspace spanned by Z.
-
-    If `Z` is a matrix of floats, uses a copy of `ivmodel`'s `proj:
-    https://github.com/mlondschien/ivmodels/blob/main/ivmodels/utils.py#L12
-    If `Z` is a 1d array of unsigned integers, assumes that `Z` is categorical with
-    values 0, 1, ..., n_categories-1. In this case, the `n_categories` argument must be
-    be supplied. The projection is done by averaging the values of f within each
-    category. If `Z` is None, returns np.zeros_like(f).
-
-    Parameters
-    ----------
-    Z: np.ndarray of dimension (n, d_Z) or (n,)
-        The `Z` matrix or 1d array of integers. If None, returns np.zeros_like(f).
-    *args: np.ndarrays of dimension (n, d_f) or (n,)
-        Vectors or matrices to project.
-    n_categories: int, optional, default=None
-        If not None, then Z is assumed to be categorical with categories 0, 1, ...,
-        n_categories-1. The projection is done by averaging the values of f within each
-        category.
-    copy: bool, default=False
-        If False, overwrites *args.
-
-    Returns
-    -------
-    np.ndarray of dimension (n, d_f) or (n,)
-        Projection of args onto the subspace spanned by Z. Same number of
-        outputs as args. Same dimension as args
-    """
-    if Z is None:
-        return np.zeros_like(f)
-
-    if len(f.shape) > 2:
-        raise ValueError(f"*args should have shapes (n, d_f) or (n,). Got {f.shape}.")
-    if f.shape[0] != Z.shape[0]:
-        raise ValueError(f"Shape mismatch: Z.shape={Z.shape}, f.shape={f.shape}.")
-
-    if n_categories is not None:
-        f = f.copy() if copy else f
-
-        if len(Z.shape) != 1 or not np.issubdtype(Z.dtype, np.integer):
-            raise ValueError(
-                "If n_categories is not None, then Z should be a single column of "
-                f"integers. Got shape {Z.shape} and dtype {Z.dtype}."
-            )
-
-        if len(f.shape) == 1:
-            means = np.zeros((n_categories,))
-            counts = np.zeros(n_categories)
-
-            # np.add.at(a, indices, b) is equivalent to a[indices] += b
-            np.add.at(means, Z, f)
-            np.add.at(counts, Z, 1)
-
-            means[counts > 0] = means[counts > 0] / counts[counts > 0]
-
-            f[:] = means[Z]
-        else:
-            means = np.zeros((n_categories, f.shape[1]))
-            counts = np.zeros(n_categories)
-
-            # np.add.at(a, indices, b) is equivalent to a[indices] += b
-            np.add.at(means, Z, f)
-            np.add.at(counts, Z, 1)
-
-            means[counts > 0, :] = means[counts > 0, :] / counts[counts > 0, None]
-
-            f[:] = means[Z, :]
-
-        return f
-
-    # The gelsy driver raises in this case - we handle it separately
-    if len(f.shape) == 2 and f.shape[1] == 0:
-        return np.zeros_like(f)
-
-    # return np.dot(Z, scipy.linalg.pinv(Z.T @ Z) @ Z.T @ args[0])
-    return np.dot(Z, scipy.linalg.lstsq(Z, f, cond=None, lapack_driver="gelsy")[0])

anchorboosting-0.2.0/tests/test_utils.py

@@ -1,74 +0,0 @@
-import numpy as np
-import pytest
-from scipy.linalg import norm
-from sklearn.preprocessing import OneHotEncoder
-
-from anchorboosting.utils import proj
-
-cases = [
-    (
-        np.array([[0], [0], [1]]),
-        np.array([[1.0], [2.0], [3.0]]),
-        np.array([[1.5], [1.5], [3]]),
-    ),
-    (
-        np.array([[2], [2], [1], [3]]),
-        np.array([[1.0], [2.0], [3.0], [1.0]]),
-        np.array([[1.5], [1.5], [3], [1]]),
-    ),
-    (
-        np.array([[1], [1], [0]]),
-        np.array([[1.0, 2.0], [0.0, 2.0], [3.0, 4.0]]),
-        np.array([[0.5, 2.0], [0.5, 2.0], [3.0, 4.0]]),
-    ),
-    (
-        np.array([[1], [1], [0]]),
-        np.array([1.0, 0.0, 3.0]),
-        np.array([0.5, 0.5, 3]),
-    ),
-    (
-        np.array([[1], [1], [0]]),
-        np.array([[1.0, 2.0], [0.0, 2.0], [3.0, 4.0]]),
-        np.array([[0.5, 2.0], [0.5, 2.0], [3.0, 4.0]]),
-    ),
-    (
-        np.array([[2], [2], [1]]),
-        np.array([[1.0, 2.0], [0.0, 2.0], [3.0, 4.0]]),
-        np.array([[0.5, 2.0], [0.5, 2.0], [3.0, 4.0]]),
-    ),
-    (np.array([[0]]), np.array([[1]]), np.array([[1]])),
-]
-
-
-@pytest.mark.parametrize("anchor, residuals, result", cases)
-@pytest.mark.parametrize("n_categories", [4, None])
-def test_proj_result(n_categories, anchor, residuals, result):
-    if n_categories is None:
-        if anchor.shape[1] != 1:
-            pytest.skip("one_hot only makes sense for 1D anchors")
-        encoder = OneHotEncoder(sparse_output=False, drop=None)
-        anchor = encoder.fit_transform(anchor).astype(float)
-    else:
-        anchor = anchor.flatten()
-
-    np.testing.assert_almost_equal(
-        proj(anchor, residuals.copy(), n_categories=n_categories), result
-    )
-
-
-@pytest.mark.parametrize("anchor, residuals, _", cases)
-def test_proj_dot_product(anchor, residuals, _):
-    np.testing.assert_almost_equal(
-        np.dot(proj(anchor, residuals).T, residuals),
-        np.dot(proj(anchor, residuals).T, proj(anchor, residuals)),
-    )
-
-
-@pytest.mark.parametrize("anchor, residuals, result", cases)
-@pytest.mark.parametrize("gamma", [0.1, 1, 2, 10, 100, 1000])
-def test_proj_orthogonal(anchor, residuals, result, gamma):
-    np.testing.assert_almost_equal(
-        norm(residuals - proj(anchor, residuals)) ** 2
-        + gamma * norm(proj(anchor, residuals)) ** 2,
-        norm(residuals) ** 2 + (gamma - 1) * norm(proj(anchor, residuals)) ** 2,
-    )