anchorboosting 0.0.1__py3-none-any.whl

anchorboosting/__init__.py

@@ -0,0 +1,4 @@
+from .models import AnchorBooster
+from .objectives.anchor_objectives import AnchorRegressionObjective
+
+__all__ = ["AnchorRegressionObjective", "AnchorBooster"]

anchorboosting/models.py

@@ -0,0 +1,368 @@
+import copy
+
+import lightgbm as lgb
+import numpy as np
+import scipy
+
+try:
+    import polars as pl
+
+    _POLARS_INSTALLED = True
+except ImportError:
+    _POLARS_INSTALLED = False
+
+
+class AnchorBooster:
+    """
+    Boost the anchor loss.
+
+    Parameters
+    ----------
+    gamma: float
+        The gamma parameter for the anchor regression objective function. Must be non-
+        negative. If 1, the objective is equivalent to a standard regression objective.
+    dataset_params: dict or None
+        The parameters for the LightGBM dataset. See LightGBM documentation for details.
+    num_boost_round: int
+        The number of boosting iterations. Default is 100.
+    objective: str, optional, default="regression"
+        The objective function to use. Can be "regression" or "binary" for probit
+        regression. If "binary", the outcome values must be 0 or 1.
+    **kwargs: dict
+        Additional parameters for the LightGBM model. See LightGBM documentation for
+        details.
+    """
+
+    def __init__(
+        self,
+        gamma,
+        dataset_params=None,
+        num_boost_round=100,
+        objective="regression",
+        **kwargs,
+    ):
+        self.gamma = gamma
+        self.params = kwargs
+        self.dataset_params = dataset_params or {}
+        self.num_boost_round = num_boost_round
+        self.booster = None
+        self.init_score_ = None
+        self.objective = objective
+
+    def fit(
+        self,
+        X,
+        y,
+        Z,
+        categorical_feature=None,
+    ):
+        """
+        Fit the model.
+
+        Parameters
+        ----------
+        X : polars.DataFrame
+            The input data.
+        y : np.ndarray
+            The outcome.
+        Z : np.ndarray
+            Anchors.
+        categorical_feature : list of str or int
+            List of categorical feature names or indices. If None, all features are
+            assumed to be numerical.
+        """
+        if self.objective != "regression" and not np.isin(y, [0, 1]).all():
+            raise ValueError("For binary classification, y values must be in {0, 1}.")
+
+        if self.objective == "regression":
+            self.init_score_ = np.mean(y)
+        elif self.objective == "binary":
+            self.init_score_ = scipy.stats.norm.ppf(np.mean(y))
+        else:
+            raise ValueError(
+                f"Objective must be 'regression' or 'binary'. Got {self.objective}."
+            )
+
+        if _POLARS_INSTALLED and isinstance(X, pl.DataFrame):
+            feature_name = X.columns
+            X = X.to_arrow()
+        else:
+            feature_name = None
+
+        self._dtype = np.result_type(Z, y)
+
+        dataset_params = {
+            "data": X,
+            "label": y,
+            "categorical_feature": categorical_feature,
+            "feature_name": feature_name,
+            "init_score": np.ones(len(y), dtype=self._dtype) * self.init_score_,
+            **self.dataset_params,
+        }
+
+        data = lgb.Dataset(**dataset_params)
+
+        self.booster = lgb.Booster(params=self.params, train_set=data)
+
+        return self.update(
+            X,
+            y,
+            Z=Z,
+            num_iteration=self.num_boost_round,
+        )
+
+    def update(self, X, y, Z, num_iteration=1):
+        if self.booster is None or self.init_score_ is None:
+            raise ValueError("AnchorBoost has not yet been fitted.")
+
+        y = y.flatten()
+
+        current_iteration = self.booster.current_iteration()
+        if current_iteration == 0:
+            f = np.ones(len(y), dtype=self._dtype) * self.init_score_
+        else:
+            f = self.booster.predict(X, raw_score=True) + self.init_score_
+
+        Q, _ = np.linalg.qr(Z, mode="reduced")  # P_Z f = Q @ (Q^T @ f)
+
+        for idx in range(current_iteration, current_iteration + num_iteration):
+            # For regression, the loss (without anchor) is
+            # loss(f, y) = 0.5 * || y - f ||^2
+            if self.objective == "regression":
+                r = f - y  # d/df loss(f, y)
+                dr = np.ones(len(y), dtype=self._dtype)  # d^2/df^2 loss(f, y)
+                ddr = np.zeros(len(y), dtype=self._dtype)  # d^3/df^3 loss(f, y)
+            # For probit regression, the loss (without anchor) is
+            # loss(f, y) = - sum_i (y_i log(p_i) + (1 - y_i) log(1 - p_i))
+            # where p_i = scipy.stats.cdf(f_i)
+            else:
+                # We wish to compute the following:
+                # p = scipy.stats.norm.cdf(f)
+                # dp = scipy.stats.norm.pdf(f)  # d/df p(f)
+                # r = np.where(y == 1, -dp / p, dp / (1 - p))  # d/df loss(f, y)
+                # The equation for r is numerically unstable. Instead, we use
+                # scipy.special.log_ndtr, with log_ndtr(f) = log(norm.cdf(f)).
+                y_tilde = np.where(y == 1, 1, -1)
+                log_phi = -0.5 * f**2 - 0.5 * np.log(2 * np.pi)  # log(norm.pdf(f))
+                r = -y_tilde * np.exp(log_phi - scipy.special.log_ndtr(y_tilde * f))
+                dr = -f * r + r**2  # d^2/df^2 loss(f, y)
+                ddr = (f**2 - 1) * r - 3 * f * r**2 + 2 * r**3  # d^3/df^3 loss
+
+            r_proj = Q @ (Q.T @ r)
+            grad = r + (self.gamma - 1) * r_proj * dr
+
+            # We wish to fit one additional tree. Intuitively, one would use
+            # is_finished = self.booster.update(fobj=self.objective.objective)
+            # for this. This makes a call to self.__inner_predict(0) to get the current
+            # predictions for all existing trees. See:
+            # https://github.com/microsoft/LightGBM/blob/18c11f861118aa889b9d4579c2888d\
+            # 5c908fd250/python-package/lightgbm/basic.py#L4165
+            # To avoid passing data through all trees each time, this uses a cache.
+            # However, this cache is based on the "original" tree values, not the one
+            # we set below. We thus use "our own" predictions and skip __inner_predict.
+            # No idea what the set_objective_to_none does, but lgbm raises if we don't.
+            self.booster._Booster__inner_predict_buffer = None
+            if not self.booster._Booster__set_objective_to_none:
+                self.booster.reset_parameter(
+                    {"objective": "none"}
+                )._Booster__set_objective_to_none = True
+
+            # is_finished is True if there we no splits satisfying the splitting
+            # criteria. c.f. https://github.com/microsoft/LightGBM/pull/6890
+            # The hessian is used only for the `min_hessian_in_leaf` parameter to
+            # avoid numerical instabilities.
+            is_finished = self.booster._Booster__boost(grad, dr)
+
+            if is_finished:
+                print(f"Finished training after {idx} iterations.")
+                break
+
+            leaves = self.booster.predict(
+                X, start_iteration=idx, num_iteration=1, pred_leaf=True
+            ).flatten()
+            num_leaves = np.max(leaves) + 1
+
+            # We wish to do 2nd order updates in the leaves. Since the anchor regression
+            # objective is quadratic, for regression a 2nd order update is equal to the
+            # global minimizer.
+            # Let M be the one-hot encoding of the tree's leaf assignments. That is,
+            # M[i, j] = 1 if leaves[i] == j else 0.
+            # We have
+            # r = d/df loss(f, y)
+            # dr = d^2/df^2 loss(f, y)
+            # ddr = d^3/df^3 loss(f, y)
+            # The anchor loss is
+            # L = loss(f, y) + (gamma - 1) / 2 * || P_Z r ||^2
+            # d/df L = d/df loss(f, y) + (gamma - 1) P_Z r * dr
+            # d^2/df^2 L = diag(d^2/df^2 loss(f, y)) + (gamma - 1) diag(P_Z r * ddr)
+            #            + (gamma - 1) diag(dr) P_Z diag(dr)
+            #
+            # We do the 2nd order update
+            # beta = - (M^T [d^2/df^2 L] M)^{-1} M^T [d/df L]
+
+            # M^T x = bincount(leaves_masked, weights=x)
+            g = np.bincount(leaves, weights=grad, minlength=num_leaves)
+
+            # M^T diag(x) M = diag(np.bincount(leaves, weights=x))
+            counts = np.bincount(
+                leaves,
+                weights=dr + (self.gamma - 1) * r_proj * ddr,
+                minlength=num_leaves,
+            )
+            counts += self.params.get("lambda_l2", 0)
+
+            # Mdr^T P_Z Mdr = (Mdr^T Q) @ (Mdr^T Q)^T
+            # One could also compute this using bincount, but it appears this
+            # version using a sparse matrix is faster.
+            Mdr = scipy.sparse.csr_matrix(
+                (
+                    dr,
+                    (np.arange(len(leaves)), leaves),
+                ),
+                shape=(len(leaves), num_leaves),
+                dtype=self._dtype,
+            )
+            B = Mdr.T.dot(Q)
+            H = (self.gamma - 1) * B @ B.T
+            H += np.diag(counts)
+
+            # Compute the 2nd order update
+            leaf_values = -np.linalg.solve(H, g) * self.params.get("learning_rate", 0.1)
+
+            for ldx, val in enumerate(leaf_values):
+                self.booster.set_leaf_output(idx, ldx, val)
+
+            # Ensure f == self.init_score_ + self.booster.predict(X)
+            f += leaf_values[leaves]
+
+        return self
+
+    def predict(self, X, raw_score=False, **kwargs):
+        """
+        Predict the outcome.
+
+        Parameters
+        ----------
+        X : numpy.ndarray, polars.DataFrame, or pyarrow.Table
+            The input data.
+        num_iteration : int
+            Number of boosting iterations to use. If -1, all are used. Else, needs to be
+            in [0, num_boost_round].
+        raw_score : bool
+            If True, returns scores. If False, returns predicted probabilities.
+        """
+        if self.booster is None:
+            raise ValueError("AnchorBoost has not yet been fitted.")
+
+        if _POLARS_INSTALLED and isinstance(X, pl.DataFrame):
+            X = X.to_arrow()
+
+        scores = self.booster.predict(X, raw_score=True, **kwargs)
+
+        if self.objective == "binary" and not raw_score:
+            return scipy.stats.norm.cdf(scores + self.init_score_)
+        else:
+            return scores + self.init_score_
+
+    def refit(self, X, y, decay_rate=0):
+        """
+        Refit the model using new data.
+
+        Set :math:`f^0_\\mathrm{refit} =` ``init_score_``.
+        Starting from :math:`f^j_\\mathrm{refit}`, we drop the new data :math:`(X, y)`
+        down the tree :math:`\\hat t^{j+1}`.
+        Let :math:`\\hat \\beta_\\mathrm{new}^{j+1}` be the second order optimization
+        of the loss :math:`\\ell(\\hat f^j_\\mathrm{refit} + \\hat t^{j+1}(X), y)`
+        with respect to the leaf node values :math:`\\beta^{j+1}``of
+        :math:`\\hat t^{j+1}(X)`.
+        We set
+        :math:`\\hat \\beta^{j+1}_\\mathrm{refit} = \\mathrm{decay rate} \\hat \\beta^{j+1}_\\mathrm{old} + (1 - \\mathrm{decay rate}) \\hat \\beta^{j+1}_\\mathrm{new}`.
+        Refitting updates the tree's leaf values, but not their structure.
+        ``AnchorBooster.refit`` differs from ``lgbm.Booster.refit`` in that it supports
+        probit regression and leaf nodes with no samples from the new data are not,
+        updated, instead of being shrunk towards zero (as in LightGBM).
+
+        Refit is not in-place, but returns a new instance of ``AnchorBooster``.
+
+        Parameters
+        ----------
+        X : numpy.ndarray, polars.DataFrame, or pyarrow.Table
+            The new data.
+        y : np.ndarray
+            The new outcomes.
+        decay_rate : float
+            The decay rate for the leaf values. Must be in [0, 1]. Default is 0. If 0,
+            the leaf values are set to the new values. If 1, the leaf values are not
+            updated.
+
+        Returns
+        -------
+        AnchorBooster
+            A new instance of AnchorBooster with the updated leaf values.
+        """  # noqa: E501
+        self_copied = copy.deepcopy(self)
+
+        # For some reason, the model params are not copied over.
+        # https://github.com/microsoft/LightGBM/issues/6821
+        self_copied.booster.params = self.booster.params
+
+        if self.objective == "binary" and not np.isin(y, [0, 1]).all():
+            raise ValueError("For binary classification, y values must be in {0, 1}.")
+
+        if self.objective == "binary":
+            y_tilde = np.where(y == 1, 1, -1)
+
+        leaves = self_copied.booster.predict(X, pred_leaf=True)
+        f = np.full(len(y), self.init_score_, dtype="float64")
+
+        for idx in range(self.num_boost_round):
+            if self.objective == "regression":
+                grad_hess_ones = np.empty((len(y), 3), dtype="float64")
+                grad_hess_ones[:, 0] = f - y
+                grad_hess_ones[:, 1:] = 1.0
+                # g = f - y
+                # h = 1.0
+            elif self.objective == "binary":
+                log_phi = -0.5 * f**2 - 0.5 * np.log(2 * np.pi)  # log(norm.pdf(f))
+                grad_hess_ones = np.empty((len(y), 3), dtype="float64")
+                grad_hess_ones[:, 0] = -y_tilde * np.exp(
+                    log_phi - scipy.special.log_ndtr(y_tilde * f)
+                )
+                grad_hess_ones[:, 1] = (
+                    -f * grad_hess_ones[:, 0] + grad_hess_ones[:, 0] ** 2
+                )
+                grad_hess_ones[:, 2] = 1.0
+                # g = -y_tilde * np.exp(log_phi - scipy.special.log_ndtr(y_tilde * f))
+                # h = -f * g + g**2
+            else:
+                raise ValueError("Objective must be 'regression' or 'binary'.")
+
+            num_leaves = np.max(leaves[:, idx]) + 1
+
+            # We aggregate gradients, hessians, and counts over the leaves.
+            # A bincount is faster than a for loop. Still this passes 3x through the
+            # data. We use np.add.at to do this in a single pass and avoid a copy.
+            # n_obs = np.bincount(leaves[:, idx], minlength=num_leaves)
+            # sum_grad = np.bincount(leaves[:, idx], weights=g, minlength=num_leaves)
+            # sum_hess = np.bincount(leaves[:, idx], weights=h, minlength=num_leaves)
+            arr = np.zeros((num_leaves, 3), dtype=np.float64)
+            np.add.at(arr, leaves[:, idx], grad_hess_ones)
+
+            sum_grad = arr[:, 0]
+            sum_hess = arr[:, 1]
+            n_obs = arr[:, 2]
+
+            values = -sum_grad / sum_hess * self_copied.params.get("learning_rate", 0.1)
+
+            new_values = np.zeros(num_leaves, dtype="float64")
+
+            for ldx in np.where(n_obs > 0)[0]:
+                old_value = self_copied.booster.get_leaf_output(idx, ldx)
+                new_values[ldx] = (
+                    decay_rate * old_value + (1 - decay_rate) * values[ldx]
+                )
+                self_copied.booster.set_leaf_output(idx, ldx, new_values[ldx])
+
+            f += new_values[leaves[:, idx]]
+
+        return self_copied

anchorboosting/objectives/__init__.py

@@ -0,0 +1,11 @@
+from .anchor_objectives import (
+    AnchorKookClassificationObjective,
+    AnchorKookMultiClassificationObjective,
+    AnchorRegressionObjective,
+)
+
+__all__ = [
+    "AnchorKookClassificationObjective",
+    "AnchorKookMultiClassificationObjective",
+    "AnchorRegressionObjective",
+]

anchorboosting/objectives/anchor_objectives.py

@@ -0,0 +1,123 @@
+import numpy as np
+
+from anchorboosting.objectives.mixins import (
+    ClassificationMixin,
+    LGBMMixin,
+    MultiClassificationMixin,
+    ProjMixin,
+    RegressionMixin,
+)
+
+
+class AnchorKookClassificationObjective(ClassificationMixin, LGBMMixin, ProjMixin):
+    def __init__(self, gamma, precompute_proj=True, n_categories=None):
+        super().__init__(precompute_proj=precompute_proj, n_categories=n_categories)
+        self.gamma = gamma
+        self.name = "kook anchor classification"
+
+    def __repr__(self):
+        return f"AnchorKookClassificationObjective(gamma={self.gamma})"
+
+    def residuals(self, f, data):
+        residuals = self.predictions(f) - data.get_label()
+        return residuals
+
+    def loss(self, f, data):
+        return (
+            super().loss(f, data)
+            + (self.gamma - 1)
+            * self.proj(
+                data.anchor,
+                self.residuals(f, data),
+                copy=False,
+            )
+            ** 2
+        )
+
+    def grad(self, f, data):
+        predictions = self.predictions(f)
+        proj_residuals = self.proj(data.anchor, self.residuals(f, data), copy=False)
+
+        return super().grad(f, data) + 2 * (
+            self.gamma - 1
+        ) * proj_residuals * predictions * (1 - predictions)
+
+
+class AnchorKookMultiClassificationObjective(
+    MultiClassificationMixin, LGBMMixin, ProjMixin
+):
+    def __init__(self, gamma, n_classes, precompute_proj=True, n_categories=None):
+        self.gamma = gamma
+        super().__init__(
+            n_classes=n_classes,
+            precompute_proj=precompute_proj,
+            n_categories=n_categories,
+        )
+        self.name = "kook anchor multi-classification"
+
+    def __repr__(self):
+        return f"AnchorKookMultiClassificationObjective(gamma={self.gamma})"
+
+    def residuals(self, f, data):
+        predictions = self.predictions(f)
+        predictions[self._indices(data.get_label())] -= 1
+        return predictions.flatten("F")
+
+    def loss(self, f, data):
+        residuals = self.residuals(f, data).reshape((-1, self.n_classes), order="F")
+        proj_residuals = self.proj(data.anchor, residuals, copy=False)
+        # Multiply with self.factor to align two-class classification with
+        # AnchorKookClassificationObjective
+        return super().loss(f, data) + self.factor * (self.gamma - 1) * np.sum(
+            proj_residuals**2, axis=1
+        )
+
+    def grad(self, f, data):
+        residuals = self.residuals(f, data).reshape((-1, self.n_classes), order="F")
+        proj_residuals = self.proj(data.anchor, residuals, copy=False)
+
+        predictions = self.predictions(f)
+        proj_residuals -= np.sum(proj_residuals * predictions, axis=1, keepdims=True)
+        # Multiply with factor to align two-class classification with
+        # AnchorKookClassificationObjective
+        anchor_grad = 2 * self.factor * (self.gamma - 1) * predictions * proj_residuals
+        return super().grad(f, data) + anchor_grad.flatten("F")
+
+
+class AnchorRegressionObjective(RegressionMixin, LGBMMixin, ProjMixin):
+    def __init__(self, gamma, n_categories=None, precompute_proj=True):
+        super().__init__(precompute_proj=precompute_proj, n_categories=n_categories)
+        self.gamma = gamma
+        self.name = "anchor regression"
+
+    def __repr__(self):
+        return f"AnchorRegressionObjective(gamma={self.gamma})"
+
+    def residuals(self, f, data):
+        return data.get_label() - f
+
+    def loss(self, f, data):
+        if self.gamma == 1:
+            return super().loss(f, data)
+
+        # For gamma <= 1, this is equivalent to kappa := (gamma - 1) / gamma and
+        # loss = (1 - kappa) | y - f |^2 + kappa | P_Z (y - f) |^2
+        return (
+            super().loss(f, data)
+            + 0.5
+            * (self.gamma - 1)
+            * self.proj(
+                data.anchor,
+                self.residuals(f, data),
+                copy=False,
+            )
+            ** 2
+        )
+
+    def grad(self, f, data):
+        residuals = self.residuals(f, data)
+        if self.gamma == 1:
+            return -residuals
+
+        proj_residuals = self.proj(data.anchor, residuals, copy=True)
+        return -residuals - (self.gamma - 1) * proj_residuals

anchorboosting/objectives/mixins.py

@@ -0,0 +1,237 @@
+import numpy as np
+
+from anchorboosting.utils import proj
+
+
+class ProjMixin:
+    def __init__(self, precompute_proj=False, n_categories=None):
+        self.n_categories = n_categories
+        self.precompute_proj = precompute_proj
+        self.pinvZ = None
+
+    def proj(self, Z, *args, copy=False):
+        if self.n_categories is not None:
+            return proj(Z, *args, n_categories=self.n_categories, copy=copy)
+        elif self.precompute_proj:
+            if self.pinvZ is None:
+                self.pinvZ = np.linalg.pinv(Z)
+            if len(args) == 1:
+                return np.dot(Z, self.pinvZ @ args[0])
+            else:
+                return (*(np.dot(Z, self.pinvZ @ f) for f in args),)
+        else:
+            return proj(Z, *args, n_categories=None, copy=copy)
+
+
+class LGBMMixin:
+    higher_is_better = False
+
+    def objective(self, f, data):
+        """Objective function for LGBM."""
+        return self.grad(f, data), self.hess(f, data)
+
+    def score(self, f, data):
+        """Score function for LGBM."""
+        return (
+            f"{self.name} ({self.gamma})",
+            self.loss(f, data).mean(),
+            self.higher_is_better,
+        )
+
+
+class RegressionMixin:
+    def init_score(self, y):
+        return np.tile(y.mean(), len(y))
+
+    def grad(self, f, data):
+        # Replicate LGBM behaviour
+        # https://github.com/microsoft/LightGBM/blob/e9fbd19d7cbaeaea1ca54a091b160868fc\
+        # 5c79ec/src/objective/regression_objective.hpp#L130-L131
+        return -(data.get_label() - f)
+
+    def hess(self, f, data):
+        # Replicate LGBM behaviour
+        # https://github.com/microsoft/LightGBM/blob/e9fbd19d7cbaeaea1ca54a091b160868fc\
+        # 5c79ec/src/objective/regression_objective.hpp#L130-L131
+        return np.ones(len(data.get_label()))
+
+    def loss(self, f, data):
+        return 0.5 * (data.get_label() - f) ** 2
+
+
+class ClassificationMixin:
+    def init_score(self, y):
+        """Initial score for LGBM.
+
+        Parameters
+        ----------
+        y: np.ndarray of dimension (n,)
+            Vector with true labels in (0, 1).
+
+        Returns
+        -------
+        np.ndarray of length n
+            Initial scores for LGBM.
+        """
+        p = np.sum(y) / len(y)
+        return np.ones(len(y)) * np.log(p / (1 - p))
+
+    def loss(self, f, data):
+        """Two-class negative log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, 1).
+
+        Returns
+        -------
+        np.ndarray of dimension (n,)
+            Loss.
+        """
+        return np.log(1 + np.exp((-2 * data.get_label() + 1) * f))
+
+    def predictions(self, f):
+        """Compute probability predictions from scores via softmax.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n,)
+            Vector with scores.
+
+        Returns
+        -------
+        np.ndarray of dimension (n,)
+            Vector with probabilities.
+        """
+        return 1 / (1 + np.exp(-f))
+
+    def grad(self, f, data):
+        """
+        Gradient of the two-class log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n,)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, 1).
+        """
+        return self.predictions(f) - data.get_label()
+
+    def hess(self, f, data):
+        """
+        Diagonal of the Hessian of the multi-class log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n,)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, 1).
+        """
+        predictions = self.predictions(f)
+        return predictions * (1.0 - predictions)
+
+
+class MultiClassificationMixin:
+    def __init__(self, n_classes, **kwargs):
+        super().__init__(**kwargs)
+        self.n_classes = n_classes
+        self.factor = (n_classes - 1) / n_classes
+
+    def init_score(self, y):
+        """Initial score for LGBM.
+
+        Parameters
+        ----------
+        y: np.ndarray of dimension (n,)
+            Vector with true labels in (0, ..., n_classes - 1).
+
+        Returns
+        -------
+        np.ndarray of dimension (n * n_classes,)
+            Initial scores for LGBM. Note that this is flattened.
+        """
+        unique_values, unique_counts = np.unique(y, return_counts=True)
+        assert len(unique_values) == self.n_classes
+        assert (sorted(unique_values) == unique_values).all()
+
+        odds = np.array(unique_counts) / np.sum(unique_counts)
+        return np.log(np.tile(odds, (len(y), 1)).flatten("F"))
+
+    def loss(self, f, data):
+        """Multi-class negative log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n * n_classes,)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, ..., n_classes - 1).
+
+        Returns
+        -------
+        np.ndarray of dimension (n,).
+            Loss.
+        """
+        y = data.get_label()
+        f = f.reshape((-1, self.n_classes), order="F")  # (n, n_classes)
+        f = f - np.max(f, axis=1)[:, np.newaxis]  # normalize f to avoid overflow
+        log_divisor = np.log(np.sum(np.exp(f), axis=1))
+        return -f[self._indices(y)] + log_divisor
+
+    def _indices(self, y):
+        return (np.arange(len(y)), y.astype(int))
+
+    def predictions(self, f):
+        """Compute probability predictions from scores via softmax.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n * n_classes,).]
+            Vector with scores.
+
+        Returns
+        -------
+        np.ndarray of dimension (n, n_classes)
+            Vector with probabilities.
+        """
+        f = f.reshape((-1, self.n_classes), order="F")  # (n, n_classes)
+        f = f - np.max(f, axis=1)[:, np.newaxis]  # normalize f to avoid overflow
+        predictions = np.exp(f)
+        predictions /= np.sum(predictions, axis=1)[:, np.newaxis]
+        return predictions
+
+    def grad(self, f, data):
+        """
+        Gradient of the multi-class log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n * n_classes,)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, ..., n_classes - 1).
+        """
+        y = data.get_label()
+        predictions = self.predictions(f)
+        predictions[self._indices(y)] -= 1
+
+        return predictions.flatten("F")
+
+    def hess(self, f, data):
+        """
+        Diagonal of the Hessian of the multi-class log-likelihood loss.
+
+        Parameters
+        ----------
+        f: np.ndarray of dimension (n * n_classes,)
+            Vector with scores.
+        data: lgbm.Dataset
+            LGBM dataset with labels of dimension (n,) in (0, ..., n_classes - 1).
+        """
+        predictions = self.predictions(f).flatten("F")
+        return 1 / self.factor * predictions * (1.0 - predictions)

anchorboosting/simulate.py

@@ -0,0 +1,25 @@
+import numpy as np
+
+
+def simulate(f, n=100, shift=0, seed=0):
+    rng = np.random.RandomState(seed)
+
+    p = 2
+
+    a = rng.normal(size=(n, 2)) + shift
+    h = rng.normal(size=(n, 1))
+    x_noise = 0.5 * rng.normal(size=(n, p))
+    x = x_noise + np.repeat(a[:, 0] + a[:, 1] + 2 * h[:, 0], p).reshape((n, p))
+
+    y_noise = 0.25 * rng.normal(size=n)
+    y = f(x[:, 0], x[:, 1]) - 2 * a[:, 0] + 3 * h[:, 0] + y_noise
+
+    return x, y, a
+
+
+def f1(x2, x3):
+    return (x2 <= 0) + (x2 <= -0.5) * (x3 <= 1)
+
+
+def f2(x2, x3):
+    return x2 + x3 + (x2 <= 0) + (x2 <= -0.5) * (x3 <= 1)

anchorboosting/utils.py

@@ -0,0 +1,84 @@
+# copy-pasted from https://github.com/mlondschien/ivmodels/blob/09f8bc5a78b3793ae5\
+# 1f0bb7f0331b70a0971ebc/ivmodels/utils.py
+import numpy as np
+import scipy
+
+
+def proj(Z, f, n_categories=None, copy=False):
+    """
+    Project f onto the subspace spanned by Z.
+
+    If `Z` is a matrix of floats, uses a copy of `ivmodel`'s `proj:
+    https://github.com/mlondschien/ivmodels/blob/main/ivmodels/utils.py#L12
+    If `Z` is a 1d array of unsigned integers, assumes that `Z` is categorical with
+    values 0, 1, ..., n_categories-1. In this case, the `n_categories` argument must be
+    be supplied. The projection is done by averaging the values of f within each
+    category. If `Z` is None, returns np.zeros_like(f).
+
+    Parameters
+    ----------
+    Z: np.ndarray of dimension (n, d_Z) or (n,)
+        The `Z` matrix or 1d array of integers. If None, returns np.zeros_like(f).
+    *args: np.ndarrays of dimension (n, d_f) or (n,)
+        Vectors or matrices to project.
+    n_categories: int, optional, default=None
+        If not None, then Z is assumed to be categorical with categories 0, 1, ...,
+        n_categories-1. The projection is done by averaging the values of f within each
+        category.
+    copy: bool, default=False
+        If False, overwrites *args.
+
+    Returns
+    -------
+    np.ndarray of dimension (n, d_f) or (n,)
+        Projection of args onto the subspace spanned by Z. Same number of
+        outputs as args. Same dimension as args
+    """
+    if Z is None:
+        return np.zeros_like(f)
+
+    if len(f.shape) > 2:
+        raise ValueError(f"*args should have shapes (n, d_f) or (n,). Got {f.shape}.")
+    if f.shape[0] != Z.shape[0]:
+        raise ValueError(f"Shape mismatch: Z.shape={Z.shape}, f.shape={f.shape}.")
+
+    if n_categories is not None:
+        f = f.copy() if copy else f
+
+        if len(Z.shape) != 1 or not np.issubdtype(Z.dtype, np.integer):
+            raise ValueError(
+                "If n_categories is not None, then Z should be a single column of "
+                f"integers. Got shape {Z.shape} and dtype {Z.dtype}."
+            )
+
+        if len(f.shape) == 1:
+            means = np.zeros((n_categories,))
+            counts = np.zeros(n_categories)
+
+            # np.add.at(a, indices, b) is equivalent to a[indices] += b
+            np.add.at(means, Z, f)
+            np.add.at(counts, Z, 1)
+
+            means[counts > 0] = means[counts > 0] / counts[counts > 0]
+
+            f[:] = means[Z]
+        else:
+            means = np.zeros((n_categories, f.shape[1]))
+            counts = np.zeros(n_categories)
+
+            # np.add.at(a, indices, b) is equivalent to a[indices] += b
+            np.add.at(means, Z, f)
+            np.add.at(counts, Z, 1)
+
+            means[counts > 0, :] = means[counts > 0, :] / counts[counts > 0, None]
+
+            f[:] = means[Z, :]
+
+        return f
+
+    # The gelsy driver raises in this case - we handle it separately
+    if len(f.shape) == 2 and f.shape[1] == 0:
+        return np.zeros_like(f)
+
+    # return np.dot(Z, scipy.linalg.pinv(Z.T @ Z) @ Z.T @ args[0])
+    return np.dot(Z, scipy.linalg.lstsq(Z, f, cond=None, lapack_driver="gelsy")[0])

anchorboosting-0.0.1.dist-info/METADATA

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: anchorboosting
+Version: 0.0.1
+Summary: Anchor Boosting
+Author-email: Malte Londschien <malte@londschien.ch>
+Project-URL: homepage, https://github.com/mlondschien/anchorboosting/
+Classifier: Programming Language :: Python
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: lightgbm
+Dynamic: license-file
+
+# Anchor Boosting
+
+This repository combines ideas from [1, 2 and 3].
+[1] suggests regularizing linear regression with the correlation between a so-called anchor variable and the regression's residuals.
+The anchor variable is assumed to be exogenous to the system, i.e., not causally affected by covariates, the outcome, or relevant hidden variables.
+[1] show that such regularization induces robustness of the linear regression model to shift interventions with limited magnitude in directions as observed in the data.
+
+[2] proposes to generalize this to nonlinear regression by simply optimizing, e.g., via boosting, the regularized anchor regression loss, replacing linear regression with a more flexible model class.
+
+Lastly, [3] give ideas to generalize anchor regression to classification.
+
+This repository combines these approaches and implements non-linear anchor (multiclass-) classification using LGBM.
+
+## Classification via boosting
+
+We first describe how to explicitly solve a classification problem with boosting.
+This includes calculating the gradient and the diagonal of the hessian of the loss with respect to model parameters.
+In each boosting step, the gradient is fit with a weak learner (CART tree).
+The diagonal of the hessian is used as a stopping criterium only (`min_hessian_in_leaf`).
+
+We separately describe two-class and multi-class classification, even though the former is a special case of the latter.
+$K$-class classification involves learning $K$ scores (parameters), each fitted with a single weak learner.
+I.e., the boosting algorithm will train $K$ trees in each set.
+
+### Two-class classification
+
+Consider a setup with observations $x_i \in \mathbb{R}^p$ with outcomes $y_i \in \{-1, 1\}$ for $i=1, \ldots, n$. 
+We assign raw scores $f_i \in \mathbb{R}$ to each observation and use the expit to obtain probability predictions.
+The estimated probability of observation $x_i$ with raw score $f_i$ to belong to class 1 is $p_i = \frac{e^f}{1 + e^f} = (1 + e^{-f})^{-1}$.
+The log-likelihood of raw scores $`f=(f_i)_{i=1}^n`$ is
+
+$$\ell(f, y) = -\sum_{i=1}^n \log(1 + e^{-yf}).$$
+
+The gradient of the log-likelihood is
+
+$$
+\frac{d}{d f_i} \ell(f, y) = y \frac{e^{-yf}}{1 + e^{-yf}} =
+\begin{cases}
+1 - p & y = 1 \\
+-p & y = 0
+\end{cases}
+$$
+
+The (diagonal of the) Hessian of the log-likelihood is
+
+$$
+\frac{d^2}{d^2f_i} \ell(f, y) = (1 - p_i) p_i.
+$$
+
+### Multiclass classification
+Consider a setup with observations $x_i \in \mathbb{R}^p$ with outcomes $y_i \in \{1, \ldots, K\}$ for $i=1, \ldots, n$.
+We assign raw scores $f_i = (f_{i, 1}, \ldots, f_{i, K}) \in \mathbb{R}^K$ to each observation and use cross-entropy to obtain probability predictions.
+For $k = 1, \ldots, K$ and $i=1, \ldots, n$ the estimated probability of observation $x_i$ with raw score $f_i$ to belong to class $k$ is $p_{i, k} := \exp(f_{i, k}) / \sum_{j=1}^K \exp(f_{i, j})$. The log-likelihood of raw scores $`(f_i)_{i=1}^n`$ is then
+```math
+\ell\left((f_{i, k})_{i=1, \ldots, n}^{k=1, \ldots, K}, (y_i)_{i=1}^n\right) = \sum_{i=1}^n \left(f_{i, y} - \log\left(\sum_{j=1}^K \exp(f_{i,j})\right)\right).
+```
+
+The gradient of the log-likelihood is
+
+$$
+\frac{d}{df_{i, k}} \ell(f, y) = \begin{cases}
+1 - p_{i, k} &  y_i = k \\
+-p_{i, k} &  y_i \neq k \\
+\end{cases}
+$$
+
+The (diagonal of the) Hessian of the log-likelihood is
+```math
+\frac{d^2}{d^2 f_{i, k}} \ell(f, y) = (1 - p_{i, k}) p_{i, k}
+```
+
+## Anchorized classification
+
+[1] suggest adding a regularization term based on an "anchor variable" $A$ to the linear least squares optimization problem to improve distributional robustness.
+
+Say additional to features and outcomes we observe anchor values $a_i \in \mathbb{R}^q, i=1,\ldots K$.
+Write $A = (a_i)_{i=1}^n \in \mathbb{R}^{n \times q}$ and $\pi_A$ for the linear projection onto the column space of $A$.
+[1] show that
+
+$$
+b^\gamma = \underset{b}{\arg\min} \|(\mathrm{Id} - \pi_A)(Y - Xb)\|_2^2 + \gamma\|\pi_A(Y - Xb)\|_2^2
+$$
+
+minimizes the linear model's worst-case risk with respect to certain shift interventions as seen in the data.
+
+[2] suggests applying anchor regression to nonlinear regression by boosting the above anchor loss with a flexible (nonlinear) learner.
+[3] presents an idea to generalize anchor regression to different distributions than the Gaussian distribution, i.e., using different losses than the squared error.
+One interesting case is the logistic loss for (two-class) classification.
+This is mostly based on intelligently defining residuals to be used in the equivalent classification anchor loss.
+
+[4] proposes the usage of a different type of residuals for classification outside of the use with anchor regression.
+We apply their residuals in the context of anchor classification.
+
+### Anchorized two-class classification according to [3]
+
+Note that again $y \in \{-1, 1\}$. 
+[3] suggest defining residuals as
+
+$$r_i = \frac{d}{d f}(\ell(f, y)) = y \frac{e^{-yf}}{1 + e^{-yf}} =
+\begin{cases}
+1 - p & y = 1 \\
+-p & y = 0
+\end{cases}
+$$
+
+For some tuning parameter $\gamma$, we add the regularization term $(\gamma - 1) \| \pi_A r \|_2^2$ to our optimization problem.
+
+$$
+\hat f = \underset{f}{\arg \min} \ \ell(f, Y) + (\gamma - 1) \|\pi_A r\|_2^2
+$$
+
+This encourages uncorrelatedness between the residuals and the anchor and, hopefully, better domain generalization. To optimize this, we also calculate the gradient of the regularization term. First, note that
+
+$$
+\frac{d}{d f_i} p_i = p_i (1 - p_i)
+$$
+
+such that
+
+$$
+\frac{d}{d f_i} \| \pi_A r \|_2^2 = 2 \pi_A r \cdot p (1 - p)
+$$
+
+### Anchorized multiclass classification according to [3]
+
+We combine the notions of [1, 2, 3].
+Motivated by [3], define residuals 
+```math
+r_{i, k} =
+\frac{d}{df_{i, k}} \ell(f, y) =
+ \begin{cases}
+1 - p_{i, k} &  y_i = k \\
+-p_{i, k} &  y_i \neq k
+\end{cases}
+```
+such that for all $i$ we have $\sum_{k} r_{i, k} = 0$.
+
+For some tuning parameter $\gamma$, we add the regularization term $(\gamma - 1) \| \pi_A r \|_2^2$ to our optimization problem.
+
+$$
+\hat f = \underset{f}{\arg \min} \ \ell(f, Y) + (\gamma - 1) \|\pi_A r\|_2^2
+$$
+
+This encourages uncorrelatedness between the residuals and the anchor and, hopefully, better domain generalization. To optimize this, we also calculate the gradient of the regularization term. First, note that
+
+$$
+\frac{d}{d f_{i, k}} p_{i, j} = 
+\frac{d}{d f_{i, k}} \frac{\exp(f_{i, j})}{\sum_l \exp(f_{i, l})} =
+\begin{cases}
+\frac{\exp(f_{i, k})}{\sum_l \exp(f_{i, l})} - \frac{\exp(f_{i, k})^2}{(\sum_l \exp(f_{i, l}))^2} & j = k \\
+-\frac{\exp(f_{i, k})\exp{(f_{i, j})}}{(\sum_l \exp(f_{i, l}))^2} & j \neq k
+\end{cases} =
+\begin{cases}
+p_{i, j} - p_{i, j}p_{i, k} & j = k \\
+-p_{i, j} p_{i, k} & j \neq k
+\end{cases}.
+$$
+
+Then,
+
+```math
+\frac{d}{d f_{i, k}} \| \pi_A r\|_2^2 = 2 \pi_A r \cdot \left(\begin{cases}
+p_{i, j} - p_{i, j}p_{i, k} & l=i, j = k \\
+-p_{i, j} p_{i, k} & l=i, j \neq k \\
+0 & l \neq i
+\end{cases}\right)_{l=1, \ldots, n}^{j=1, \ldots K}
+=(\pi_A r)_{i, k} \ p_{i, k} - \sum_{l=1}^K (\pi_A r)_{i, l} \ p_{i, l}.
+```
+
+<!---
+The diagonal of the hessian is equal to 
+
+$$
+\frac{d^2}{d f_{i, k}^2} \| \pi_A r\|^2 = 2 \pi_A r \cdot
+\left(\begin{cases}
+p_{i, j} - 3 p_{i, j}^2 + 2 p_{i, j}^3 & l=i, j = k \\
+2 p_{i, j} p_{i, k}^2 - p_{i, j} p{i, k} & l=i, j \neq k \\
+0 & l \neq i
+\end{cases}\right)_{l=1, \ldots, n}^{j=1, \ldots K} +
+2 \pi_A \left(\begin{cases}
+p_{i, j} - p_{i, j}p_{i, k} & l=i, j = k \\
+- p_{i, j} p_{i, k} & l=i, j \neq k \\
+0 & l \neq i
+\end{cases}\right)_{l=1, \ldots, n}^{j=1, \ldots K} \cdot \left(\begin{cases}
+p_{i, j} - p_{i, j}p_{i, k} & l=i, j = k \\
+- p_{i, j} p_{i, k} & l=i, j \neq k \\
+0 & l \neq i
+\end{cases}\right)_{l=1, \ldots, n}^{j=1, \ldots K}
+$$
+
+$$
+\frac{d^2}{d f_{i, k} d f_{i, l}} p_{i, j} =
+\begin{cases}
+(1 - 2 p_{i, j}) p_{i, j} (1 - p_{i, j}) & j = k \\
+p_{i, j}^2 p_{i, k} & j \neq k
+\end{cases}
+$$ -->
+
+## Anchorized two-class classification with residuals motivated by [4]
+
+
+
+## References
+
+[1] Rothenhäusler, D., N. Meinshausen, P. Bühlmann, and J. Peters (2021). Anchor regression: Heterogeneous data meet causality. Journal of the Royal Statistical Society Series B (Statistical Methodology) 83(2), 215–246.
+
+[2] Bühlmann, P. (2020). Invariance, causality and robustness. Statistical Science 35(3), 404– 426.
+
+[3] Kook, L., B. Sick, and P. Bühlmann (2022). Distributional anchor regression. Statistics and Computing 32(3), 1–19.

anchorboosting-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+anchorboosting/__init__.py,sha256=MGNvLzDGE19NKIwN6muC6P6c-dznFfn-VoQi_kd08Yc,160
+anchorboosting/models.py,sha256=7MEPp9lvSr0_FEO_Dxgf-GWLz_N5R2GRoU2cerGMULM,14787
+anchorboosting/simulate.py,sha256=eMBwbOsE9U9t8l1uCNg8asLHSV5v6Rff8CHGmltlaKI,577
+anchorboosting/utils.py,sha256=Bh439YcvTYQ6QBIISuL1BPSo_v7nq8VAalSONosxeEg,3152
+anchorboosting/objectives/__init__.py,sha256=hbBiAd6Ng_sfc1aaBc559DaCn1oGacC5XJDeq2Eoiz8,284
+anchorboosting/objectives/anchor_objectives.py,sha256=rloFuorTs-kcGvE2L1zGlJJlPj3r_fukN-PtTwo9nOo,4223
+anchorboosting/objectives/mixins.py,sha256=-ceTSWUFyOBBOrFltHTYFTLoW31qlUOC683nnaLfTk0,7182
+anchorboosting-0.0.1.dist-info/licenses/LICENSE,sha256=DZyvhSIAeCQbdMQXcvAU_5VZo7j6Iqh5wxwNUNIkgXY,1506
+anchorboosting-0.0.1.dist-info/METADATA,sha256=FP5A3K_iLtFRn_jVDPI2IUgEWxmLqK0T2BH7ZvRAc1Y,8811
+anchorboosting-0.0.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+anchorboosting-0.0.1.dist-info/top_level.txt,sha256=MQhJ5Jg216IFXbvNuArVQZPT8NEYMbU1k-PkG_EnBXc,15
+anchorboosting-0.0.1.dist-info/RECORD,,

anchorboosting-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

anchorboosting-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,13 @@
+BSD 3-clause license
+Copyright (c) 2022 Malte Londschien
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

anchorboosting-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+anchorboosting