sklearn-morpho 0.2.3__py3-none-any.whl

sklearn_morpho/stopping/stopping_early.py

@@ -0,0 +1,40 @@
+from . import StoppingMethod
+
+
+class EarlyStoppingMethod(StoppingMethod):
+    """
+    Stopping method that triggers whenever the validation cost starts increasing
+    while the training cost keeps decreasing, this for a set number of
+    consecutive epochs.
+
+    It is assumed that the training cost keeps decreasing as should be the case
+    during functional training.
+    """
+
+    def __init__(self, delay=5):
+        if delay <= 0:
+            raise ValueError(f'invalid delay, expected > 0 but got {delay}')
+
+        self.delay = delay
+        self.count = 0
+
+        self.best_validation_cost: float | None = None
+
+    def requires_validation(self) -> bool:
+        return True
+
+    def should_stop(
+        self, n_epochs: int, train_cost: float, validation_cost: float
+    ) -> bool:
+        if self.best_validation_cost is None:
+            self.best_validation_cost = validation_cost
+        else:
+            if validation_cost >= self.best_validation_cost:
+                self.count += 1
+                if self.count >= self.delay:
+                    return True
+            else:
+                self.count = 0
+                self.best_validation_cost = validation_cost
+
+        return False

sklearn_morpho/stopping/stopping_epoch.py

@@ -0,0 +1,24 @@
+from . import StoppingMethod
+
+
+class EpochStoppingMethod(StoppingMethod):
+    """
+    Stopping method that triggers whenever the number of epochs gets greater or
+    equal to the parameter.
+    """
+
+    def __init__(self, max_epochs=20):
+        if max_epochs <= 0:
+            raise ValueError(
+                f'invalid max_epochs, expected > 0 but got {max_epochs}'
+            )
+
+        self.max_epochs = max_epochs
+
+    def requires_validation(self) -> bool:
+        return False
+
+    def should_stop(
+        self, n_epochs: int, train_cost: float, validation_cost: float
+    ) -> bool:
+        return n_epochs >= self.max_epochs

sklearn_morpho/stopping/stopping_train_stop.py

@@ -0,0 +1,17 @@
+from . import StoppingMethod
+
+
+class TrainStopStoppingMethod(StoppingMethod):
+    """
+    Stopping method that triggers whenever the training cost gets to 0, that is
+    the perceptron found a global minimum and will no longer try to improve,
+    rendering validation-related stopping methods unusable.
+    """
+
+    def requires_validation(self) -> bool:
+        return False
+
+    def should_stop(
+        self, n_epochs: int, train_cost: float, validation_cost: float
+    ) -> bool:
+        return train_cost <= 0

sklearn_morpho/training/__init__.py

@@ -0,0 +1,6 @@
+from .dccp_ldep import LDEPDccpTrainer as LDEPDccpTrainer
+from .dccp_rdep import RDEPDccpTrainer as RDEPDccpTrainer
+from .dccp_simple_perceptron import (
+    SimplePerceptronDccpTrainer as SimplePerceptronDccpTrainer,
+)
+from .dccp_wrapper import DccpTrainer as DccpTrainer

sklearn_morpho/training/dccp_ldep.py

@@ -0,0 +1,270 @@
+from typing import Literal, cast
+import numpy as np
+import cvxpy as cp
+
+from .dccp_wrapper import DccpTrainer
+from ..weighting import SampleWeighting
+from ..stopping import StoppingMethod
+
+
+class LDEPDccpTrainer(DccpTrainer):
+    """
+    l-DEP trainer.
+
+    During training, the final parameters are implicitly embedded into one
+    another.
+    Some of them (like lambda) are only extracted once the training ends.
+    This is not necessary for performance, but is there for readability and
+    interpretability reasons once the estimator is trained.
+    """
+
+    def __init__(
+        self,
+        latent_dims: tuple[int, int],
+        margin: float,
+        penalty: float,
+        validation_ratio: float,
+        weighting_method: SampleWeighting,
+        stopping_methods: list[StoppingMethod],
+        use_dccp_library: bool,
+        verbose: Literal[0, 1, 2],
+        random_state: np.random.RandomState,
+    ) -> None:
+        """
+        Initialize the l-DEP trainer.
+
+        param latent_dims: the latent dimensions for the max and min perceptrons
+        param [others]:    see base class
+        """
+
+        self.latent_dims = latent_dims
+
+        super().__init__(
+            margin,
+            penalty,
+            validation_ratio,
+            weighting_method,
+            stopping_methods,
+            use_dccp_library,
+            verbose,
+            random_state,
+        )
+
+    def at_training_start(self, data_dim: int) -> None:
+        N_max, N_min = self.latent_dims
+
+        self._objective = None
+
+        # Extracted parameters that will be populated during training but need
+        # initial values for linearization
+        self.max_perceptron = self.random_state.randn(N_max)
+        self.min_perceptron = self.random_state.randn(N_min)
+        self.max_matrix = self.random_state.randn(N_max, data_dim)
+        self.min_matrix = self.random_state.randn(N_min, data_dim)
+
+        # Create constraints for linearization derived from real parameters,
+        # used to absorbe non-convex parameters that are restored at the end.
+        # Method inspired by arXiv:2011.06512v1.
+        self._max_training_weights = cp.Variable(N_max)
+        self._min_training_weights = cp.Variable(N_min)
+        self._max_training_matrix = cp.Variable((N_max, data_dim))
+        self._min_training_matrix = cp.Variable((N_min, data_dim))
+
+    def set_objective(self, K: int, cost_weights: np.ndarray) -> None:
+        # the objective and the slack variables do not change, cache them
+        if self._objective is not None:
+            return
+
+        self._slack = cp.Variable(K)
+
+        value = cp.sum(cp.multiply(cp.pos(self._slack), cost_weights))
+        if self.penalty > 0:
+            value += self.penalty * (
+                cp.sum_squares(self._max_training_weights)
+                + cp.sum_squares(self._min_training_weights)
+                + cp.sum_squares(self._max_training_matrix)
+                + cp.sum_squares(self._min_training_matrix)
+            )
+
+        self._objective = cp.Minimize(value)
+
+    def get_problem_linearized(
+        self, X: np.ndarray, y: np.ndarray, cost_weights: np.ndarray
+    ) -> cp.Problem:
+        K = X.shape[0]
+        self.set_objective(K, cost_weights)
+
+        # Constraints: convex constraints are for data points in the first
+        # class, while concave ones are for points in the second class.
+        # Therefore, linearize things when needed to make the problem convex,
+        # sometimes using values from the previous epoch.
+
+        idx_max = np.argmax(self.max_perceptron + X @ self.max_matrix.T, axis=1)
+        idx_min = np.argmin(self.min_perceptron + X @ self.min_matrix.T, axis=1)
+
+        # create arrays to regroup the active indices using numpy, to then apply
+        # constraints all at once and use AST optimizations inside cvxpy
+        M_max = np.zeros((K, self.max_perceptron.size))
+        M_min = np.zeros((K, self.min_perceptron.size))
+        M_max[np.arange(K), idx_max] = 1
+        M_min[np.arange(K), idx_min] = 1
+
+        constraints = []
+        for label in [0, 1]:
+            mask = y == label
+            if not np.any(mask):
+                continue
+
+            X_ = X[mask]
+            K_ = X_.shape[0]
+
+            # start building the perceptron's outputs before the min/max
+            expr_max = X_ @ self._max_training_matrix.T
+            expr_min = X_ @ self._min_training_matrix.T
+
+            # add weights to every row of (X @ matrix.T) using np.ones to create
+            # a matrix safely for cvxpy's cpp backend
+            ones = np.ones((K_, 1))
+            expr_max += ones @ cp.reshape(
+                self._max_training_weights,
+                (1, self.max_perceptron.size),
+                order='C',
+            )
+            expr_min += ones @ cp.reshape(
+                self._min_training_weights,
+                (1, self.min_perceptron.size),
+                order='C',
+            )
+
+            active_max = cp.sum(cp.multiply(M_max[mask], expr_max), axis=1)
+            active_min = cp.sum(cp.multiply(M_min[mask], expr_min), axis=1)
+
+            if label == 0:
+                constraints.append(
+                    self._slack[mask] - active_min
+                    >= self.margin + cp.max(expr_max, axis=1)
+                )
+            else:
+                constraints.append(
+                    self._slack[mask] + active_max
+                    >= self.margin - cp.min(expr_min, axis=1)
+                )
+
+        return cp.Problem(cast(cp.Minimize, self._objective), constraints)
+
+    def get_problem_dccp(
+        self, X: np.ndarray, y: np.ndarray, cost_weights: np.ndarray
+    ) -> cp.Problem:
+        K = X.shape[0]
+        self.set_objective(K, cost_weights)
+
+        # Constraints: convex constraints are for data points in the first
+        # class, while concave ones are for points in the second class.
+
+        constraints = []
+        for label in [0, 1]:
+            mask = y == label
+            if not np.any(mask):
+                continue
+
+            X_ = X[mask]
+            K_ = X_.shape[0]
+
+            # start building the perceptron's outputs before the min/max
+            expr_max = X_ @ self._max_training_matrix.T
+            expr_min = X_ @ self._min_training_matrix.T
+
+            # add weights to every row of (X @ matrix.T) using np.ones to create
+            # a matrix safely for cvxpy's cpp backend
+            ones = np.ones((K_, 1))
+            expr_max += ones @ cp.reshape(
+                self._max_training_weights,
+                (1, self.max_perceptron.size),
+                order='C',
+            )
+            expr_min += ones @ cp.reshape(
+                self._min_training_weights,
+                (1, self.min_perceptron.size),
+                order='C',
+            )
+
+            if label == 0:
+                constraints.append(
+                    self.margin + cp.max(expr_max, axis=1)
+                    <= self._slack[mask] - cp.min(expr_min, axis=1)
+                )
+            else:
+                constraints.append(
+                    self.margin - cp.min(expr_min, axis=1)
+                    <= self._slack[mask] + cp.max(expr_max, axis=1)
+                )
+
+        return cp.Problem(cast(cp.Minimize, self._objective), constraints)
+
+    def get_cost(self, X: np.ndarray, y: np.ndarray) -> float:
+        expr_max = np.max(self.max_perceptron + X @ self.max_matrix.T, axis=1)
+        expr_min = np.min(self.min_perceptron + X @ self.min_matrix.T, axis=1)
+
+        # can use the expressions directly without needing to multiply by lambda
+        # since still in the training phase
+        cost = (expr_max + expr_min) * (1 - 2 * y)
+        return np.maximum(0, cost).sum()
+
+    def after_epoch(self) -> None:
+        # update the perceptrons weights from this epoch's results
+        if (
+            self._max_training_weights.value is None
+            or self._min_training_weights.value is None
+        ):
+            raise ValueError('CvxPy could not optimize a perceptron')
+
+        self.max_perceptron = self._max_training_weights.value
+        self.min_perceptron = self._min_training_weights.value
+
+        # extract the matrices
+        if (
+            self._max_training_matrix.value is None
+            or self._min_training_matrix.value is None
+        ):
+            raise ValueError('CvxPy could not optimize transformation matrices')
+        self.max_matrix = self._max_training_matrix.value
+        self.min_matrix = self._min_training_matrix.value
+
+    def at_training_end(self) -> None:
+        max_matrix_norm = np.linalg.norm(self.max_matrix)
+        min_matrix_norm = np.linalg.norm(self.min_matrix)
+
+        div = max_matrix_norm + min_matrix_norm
+        if np.isclose(div, 0):
+            raise ValueError('Transformation matrices are zero, cannot solve')
+
+        self.lambda_ = max_matrix_norm / div
+
+        # if lambda is close to a number that creates divisions by zero, it is
+        # safe to nullify the affected elements, that will not contribute anyway
+        if np.isclose(self.lambda_, 0):
+            self.max_matrix = np.zeros_like(self.max_matrix)
+            self.max_perceptron = np.zeros_like(self.max_perceptron)
+        else:
+            self.max_matrix /= self.lambda_
+            self.max_perceptron /= self.lambda_
+        if np.isclose(self.lambda_, 1):
+            self.min_matrix = np.zeros_like(self.max_matrix)
+            self.min_perceptron = np.zeros_like(self.min_perceptron)
+        else:
+            self.min_matrix /= 1 - self.lambda_
+            self.min_perceptron /= 1 - self.lambda_
+
+    def save_best(self) -> None:
+        self.saved = {
+            'max_w': np.copy(self.max_perceptron),
+            'min_w': np.copy(self.min_perceptron),
+            'max_m': np.copy(self.max_matrix),
+            'min_m': np.copy(self.min_matrix),
+        }
+
+    def rollback_to_best(self) -> None:
+        self.max_perceptron = self.saved['max_w']
+        self.min_perceptron = self.saved['min_w']
+        self.max_matrix = self.saved['max_m']
+        self.min_matrix = self.saved['min_m']

sklearn_morpho/training/dccp_rdep.py

@@ -0,0 +1,285 @@
+import numpy as np
+import cvxpy as cp
+from typing import Literal, cast
+from warnings import warn
+
+from .dccp_wrapper import DccpTrainer
+from ..weighting import SampleWeighting
+from ..stopping import StoppingMethod
+
+
+class RDEPDccpTrainer(DccpTrainer):
+    """
+    r-DEP trainer.
+
+    During training, the final parameters are implicitly embedded into one
+    another.
+    Some of them (like lambda) are only extracted once the training ends.
+    This is not necessary for performance, but is there for readability and
+    interpretability reasons once the estimator is trained.
+    """
+
+    def __init__(
+        self,
+        lambda_bounds: tuple[float, float],
+        margin: float,
+        penalty: float,
+        validation_ratio: float,
+        weighting_method: SampleWeighting,
+        stopping_methods: list[StoppingMethod],
+        use_dccp_library: bool,
+        verbose: Literal[0, 1, 2],
+        random_state: np.random.RandomState,
+    ) -> None:
+        """
+        Initialize the r-DEP trainer.
+
+        param lambda_bounds: A pair of min and max values for lambda, to avoid
+                             solvers (especially dccp) from failing to optimize.
+                             To keep the constraints at the right convexity,
+                             the bounds must be inside [0, 1].
+        param [others]:      See base class.
+        """
+
+        super().__init__(
+            margin,
+            penalty,
+            validation_ratio,
+            weighting_method,
+            stopping_methods,
+            use_dccp_library,
+            verbose,
+            random_state,
+        )
+
+        if lambda_bounds[0] < 0 or lambda_bounds[1] > 1:
+            raise ValueError(
+                'Invalid lambda_bounds, expected within [0, 1] '
+                f'got {list(lambda_bounds)}'
+            )
+        if (
+            lambda_bounds[0] == 0 or lambda_bounds[1] == 1
+        ) and use_dccp_library:
+            warn(
+                'Warning: lambda_bounds may be inappropriate for dccp solver: '
+                f'{list(lambda_bounds)}'
+            )
+
+        self.lambda_bounds = lambda_bounds
+
+    def at_training_start(self, data_dim: int) -> None:
+        # similar to l-DEP, see corresponding files for implementation comments
+        self._objective = None
+
+        # Extracted parameters that will be populated during training but need
+        # initial values for linearization
+        self.max_perceptron = self.random_state.randn(data_dim)
+        self.min_perceptron = self.random_state.randn(data_dim)
+        self.lambda_ = 0.5
+
+        # to make the problem DCP, the training weights absorb the
+        # multiplications involving lambda_, they are explicited at the end of
+        # epochs
+        self._max_training_weights = cp.Variable(data_dim)
+        self._min_training_weights = cp.Variable(data_dim)
+        self._training_lambda = cp.Variable()
+
+    def set_objective(
+        self, X: np.ndarray, y: np.ndarray, cost_weights: np.ndarray
+    ) -> None:
+        # the objective and the slack variables do not change, cache them
+        if self._objective is not None:
+            return
+
+        n_features = len(np.unique(y))
+        if n_features != 2:
+            raise ValueError(
+                'Detected degenerate dataset, perhaps after '
+                'train/validation split. Expected 2 features, '
+                f'found only {n_features} feature(s)'
+            )
+
+        # figure out whether we should invert the perceptron's output, since
+        # the lower class should be lower in coordinates
+        labels, inv, counts = np.unique(
+            y, return_inverse=True, return_counts=True
+        )
+        sums = np.zeros((len(labels), X.shape[1]))
+        np.add.at(sums, inv, X)
+        centroids = sums / counts[:, np.newaxis]
+
+        # set the objective
+        self.invert_res = centroids[0].sum() > centroids[1].sum()
+
+        self._slack = cp.Variable(len(X))
+
+        value = cp.sum(cp.multiply(cp.pos(self._slack), cost_weights))
+        if self.penalty > 0:
+            value += self.penalty * (
+                cp.sum_squares(self._max_training_weights)
+                + cp.sum_squares(self._min_training_weights)
+            )
+
+        self._objective = cp.Minimize(value)
+
+    def get_problem_linearized(
+        self, X: np.ndarray, y: np.ndarray, cost_weights: np.ndarray
+    ) -> cp.Problem:
+        K = X.shape[0]
+        self.set_objective(X, y, cost_weights)
+
+        # Constraints: convex constraints are for data points in the first
+        # class, while concave ones are for points in the second class.
+        # Therefore, linearize things when needed to make the problem convex,
+        # sometimes using values from the previous epoch.
+
+        idx_max = np.argmax(self.lambda_ * X + self.max_perceptron, axis=1)
+        idx_min = np.argmin(
+            (1 - self.lambda_) * X + self.min_perceptron, axis=1
+        )
+
+        # create arrays to regroup the active indices using numpy, to then apply
+        # constraints all at once and use AST optimizations inside cvxpy
+        M_max = np.zeros((K, self.max_perceptron.size))
+        M_min = np.zeros((K, self.min_perceptron.size))
+        M_max[np.arange(K), idx_max] = 1
+        M_min[np.arange(K), idx_min] = 1
+
+        constraints = []
+        for label in [0, 1]:
+            mask = y == label
+            if not np.any(mask):
+                continue
+
+            X_ = X[mask]
+            K_ = X_.shape[0]
+
+            # add weights to every row of (X @ matrix.T) using np.ones to create
+            # a matrix safely for cvxpy's cpp backend
+            ones = np.ones((K_, 1))
+            expr_max = self._training_lambda * X_
+            expr_min = (1 - self._training_lambda) * X_
+            expr_max += ones @ cp.reshape(
+                self._max_training_weights,
+                (1, self.max_perceptron.size),
+                order='C',
+            )
+            expr_min += ones @ cp.reshape(
+                self._min_training_weights,
+                (1, self.min_perceptron.size),
+                order='C',
+            )
+
+            active_max = cp.sum(cp.multiply(M_max[mask], expr_max), axis=1)
+            active_min = cp.sum(cp.multiply(M_min[mask], expr_min), axis=1)
+
+            if (label == 0) ^ self.invert_res:
+                constraints.append(
+                    self._slack[mask] - active_min
+                    >= self.margin + cp.max(expr_max, axis=1)
+                )
+            else:
+                constraints.append(
+                    self._slack[mask] + active_max
+                    >= self.margin - cp.min(expr_min, axis=1)
+                )
+
+        # avoid lambda making some constraints convex when they should be
+        # concave and vice versa
+        constraints += [
+            self.lambda_bounds[0] <= self._training_lambda,
+            self._training_lambda <= self.lambda_bounds[1],
+        ]
+        return cp.Problem(cast(cp.Minimize, self._objective), constraints)
+
+    def get_problem_dccp(
+        self, X: np.ndarray, y: np.ndarray, cost_weights: np.ndarray
+    ) -> cp.Problem:
+        self.set_objective(X, y, cost_weights)
+
+        constraints = []
+        for label in [0, 1]:
+            mask = y == label
+            if not np.any(mask):
+                continue
+
+            X_ = X[mask]
+            K_ = X_.shape[0]
+
+            ones = np.ones((K_, 1))
+            expr_max = self._training_lambda * X_
+            expr_min = (1 - self._training_lambda) * X_
+            expr_max += ones @ cp.reshape(
+                self._max_training_weights,
+                (1, self.max_perceptron.size),
+                order='C',
+            )
+            expr_min += ones @ cp.reshape(
+                self._min_training_weights,
+                (1, self.min_perceptron.size),
+                order='C',
+            )
+
+            expr_max = cp.max(expr_max, axis=1)
+            expr_min = cp.min(expr_min, axis=1)
+
+            if (label == 0) ^ self.invert_res:
+                constraints.append(
+                    self.margin + expr_max <= self._slack[mask] - expr_min
+                )
+            else:
+                constraints.append(
+                    self.margin - expr_min <= self._slack[mask] + expr_max
+                )
+
+        constraints += [
+            self.lambda_bounds[0] <= self._training_lambda,
+            self._training_lambda <= self.lambda_bounds[1],
+        ]
+        return cp.Problem(cast(cp.Minimize, self._objective), constraints)
+
+    def get_cost(self, X: np.ndarray, y: np.ndarray) -> float:
+        expr_max = np.max(self.lambda_ * X + self.max_perceptron, axis=1)
+        expr_min = np.min((1 - self.lambda_) * X + self.min_perceptron, axis=1)
+
+        cost = (expr_max + expr_min) * (1 - 2 * y) * (1 - 2 * self.invert_res)
+
+        return np.maximum(0, cost).sum()
+
+    def after_epoch(self) -> None:
+        # update the perceptrons weights from this epoch's results
+        if (
+            self._max_training_weights.value is None
+            or self._min_training_weights.value is None
+        ):
+            raise ValueError('CvxPy could not optimize a perceptron')
+        if self._training_lambda.value is None:
+            raise ValueError('CvxPy could not optimize lambda')
+
+        self.max_perceptron = self._max_training_weights.value
+        self.min_perceptron = self._min_training_weights.value
+        self.lambda_ = self._training_lambda.value
+
+        # if lambda is close to a number that creates divisions by zero, it is
+        # safe to nullify the affected elements, that will not contribute anyway
+        if np.isclose(self.lambda_, 0):
+            self.max_perceptron = np.zeros_like(self.max_perceptron)
+        else:
+            self.max_perceptron /= self.lambda_
+
+        if np.isclose(self.lambda_, 1):
+            self.min_perceptron = np.zeros_like(self.min_perceptron)
+        else:
+            self.min_perceptron /= 1 - self.lambda_
+
+    def save_best(self) -> None:
+        self.saved = {
+            'max_w': np.copy(self.max_perceptron),
+            'min_w': np.copy(self.min_perceptron),
+            'lambda': self.lambda_,
+        }
+
+    def rollback_to_best(self) -> None:
+        self.max_perceptron = self.saved['max_w']
+        self.min_perceptron = self.saved['min_w']
+        self.lambda_ = self.saved['lambda']