optiml 1.7__py3-none-any.whl

optiml/ml/neural_network/layers.py

@@ -0,0 +1,183 @@
+from abc import ABC
+
+import autograd.numpy as np
+
+from .activations import Activation, linear
+from .initializers import glorot_uniform
+from .regularizers import l2
+
+
+class Layer(ABC):
+    """
+    Base abstract class for all neural network layers. A layer implements
+    the ``forward`` pass that maps its input to its output and the
+    ``backward`` pass that back-propagates the error signal.
+    """
+
+    def forward(self, X):
+        raise NotImplementedError
+
+    def backward(self, delta):
+        raise NotImplementedError
+
+
+class ParamLayer(Layer, ABC):
+    """
+    Base abstract class for all layers with trainable parameters, i.e., a
+    coefficient (weight) tensor and, optionally, an intercept (bias) tensor,
+    each with its own initializer and regularizer.
+    """
+
+    def __init__(self,
+                 coef_shape,
+                 activation,
+                 coef_init,
+                 inter_init,
+                 coef_reg,
+                 inter_reg,
+                 fit_intercept,
+                 random_state=None):
+        """
+        Parameters
+        ----------
+
+        coef_shape : tuple of int
+            Shape of the coefficient (weight) tensor.
+
+        activation : `Activation` instance
+            The activation function applied by the layer.
+
+        coef_init : callable, array-like or None
+            Initializer for the coefficient tensor. If None, `glorot_uniform`
+            is used; if callable, it is called with ``coef_shape`` and
+            ``random_state``; otherwise it is used as the initial values.
+
+        inter_init : callable, array-like or None
+            Initializer for the intercept tensor. If None, zeros are used;
+            if callable, it is called with the intercept shape; otherwise it
+            is used as the initial values. Only used when ``fit_intercept`` is True.
+
+        coef_reg : `Regularizer` instance or None
+            Regularizer applied to the coefficient tensor. If None, `l2` is used.
+
+        inter_reg : `Regularizer` instance or None
+            Regularizer applied to the intercept tensor. If None, `l2` is used.
+
+        fit_intercept : bool
+            Whether the layer has an intercept (bias) term.
+
+        random_state : int, RandomState instance or None, default=None
+            Controls the pseudo random number generation for the parameters
+            initialization.
+        """
+
+        if isinstance(activation, Activation):
+            self.activation = activation
+        else:
+            raise TypeError(f'{activation} is not an allowed activation function')
+
+        if coef_init is None:
+            self.coef_ = glorot_uniform(coef_shape, random_state=random_state)
+        elif callable(coef_init):
+            self.coef_ = coef_init(coef_shape, random_state=random_state)
+        else:
+            self.coef_ = np.asarray(coef_init, dtype=float).reshape(-1, 1)
+
+        self.fit_intercept = fit_intercept
+        if self.fit_intercept:
+            shape = [1] * len(coef_shape)
+            shape[-1] = coef_shape[-1]
+            if inter_init is None:
+                self.inter_ = np.zeros(shape)
+            elif callable(inter_init):
+                self.inter_ = inter_init(shape)
+            else:
+                self.inter_ = np.asarray(inter_init, dtype=float).reshape(-1, 1)
+
+        if coef_reg is None:
+            self.coef_reg = l2
+        else:
+            self.coef_reg = coef_reg
+
+        if inter_reg is None:
+            self.inter_reg = l2
+        else:
+            self.inter_reg = inter_reg
+
+
+class FullyConnected(ParamLayer):
+    """
+    Fully connected (dense) layer that computes ``activation(X @ W + b)``.
+    """
+
+    def __init__(self,
+                 n_in,
+                 n_out,
+                 activation=linear,
+                 coef_init=glorot_uniform,
+                 inter_init=np.zeros,
+                 coef_reg=l2,
+                 inter_reg=l2,
+                 fit_intercept=True,
+                 random_state=None):
+        """
+        Parameters
+        ----------
+
+        n_in : int
+            Number of input units (fan-in) of the layer.
+
+        n_out : int
+            Number of output units (fan-out) of the layer, i.e., the number
+            of neurons.
+
+        activation : `Activation` instance, default=linear
+            The activation function applied by the layer.
+
+        coef_init : callable or array-like, default=glorot_uniform
+            Initializer for the coefficient (weight) tensor.
+
+        inter_init : callable or array-like, default=np.zeros
+            Initializer for the intercept (bias) tensor. Only used when
+            ``fit_intercept`` is True.
+
+        coef_reg : `Regularizer` instance, default=l2
+            Regularizer applied to the coefficient tensor.
+
+        inter_reg : `Regularizer` instance, default=l2
+            Regularizer applied to the intercept tensor.
+
+        fit_intercept : bool, default=True
+            Whether to add an intercept (bias) term to the layer.
+
+        random_state : int, RandomState instance or None, default=None
+            Controls the pseudo random number generation for the parameters
+            initialization.
+        """
+        super(FullyConnected, self).__init__(coef_shape=(n_in, n_out),
+                                             activation=activation,
+                                             coef_init=coef_init,
+                                             inter_init=inter_init,
+                                             coef_reg=coef_reg,
+                                             inter_reg=inter_reg,
+                                             fit_intercept=fit_intercept,
+                                             random_state=random_state)
+        self.fan_in = n_in
+        self.fan_out = n_out
+
+    def forward(self, X):
+        self._X = X
+        self._WX_b = np.dot(self._X, self.coef_)
+        if self.fit_intercept:
+            self._WX_b += self.inter_
+        return self.activation(self._WX_b)
+
+    def backward(self, delta):
+        # dW, db
+        dZ = delta * self.activation.jacobian(self._WX_b)
+        grads = {'dW': self._X.T.dot(dZ)}
+        if self.fit_intercept:
+            grads['db'] = np.sum(dZ, axis=0, keepdims=True)
+        # dX
+        dX = dZ.dot(self.coef_.T)
+        return dX, grads

optiml/ml/neural_network/losses.py

@@ -0,0 +1,178 @@
+from abc import ABC
+
+import autograd.numpy as np
+from scipy.special import xlogy
+
+from .activations import Linear
+from .layers import ParamLayer
+from .regularizers import L2
+from ...opti import OptimizationFunction
+
+
+class NeuralNetworkLoss(OptimizationFunction, ABC):
+    """
+    Base abstract class for all neural network loss functions. It defines the
+    objective minimized during training, i.e., the data loss averaged over the
+    samples plus the layers regularization terms, together with its jacobian
+    computed via back-propagation.
+
+    Subclasses must implement ``loss`` and, optionally, override ``delta``.
+    """
+
+    def __init__(self, neural_net, X, y):
+        """
+        Parameters
+        ----------
+
+        neural_net : `NeuralNetwork` instance
+            The neural network estimator this loss is attached to. It provides
+            the layers and the forward/backward passes used by the objective.
+
+        X : ndarray of shape (n_samples, n_features)
+            Training data over which the loss is evaluated.
+
+        y : ndarray of shape (n_samples, n_outputs)
+            Target values associated with ``X``.
+        """
+        super(NeuralNetworkLoss, self).__init__(X.shape[1])
+        self.neural_net = neural_net
+        self.X = X
+        self.y = y
+
+    def args(self):
+        return self.X, self.y
+
+    def loss(self, y_pred, y_true):
+        raise NotImplementedError
+
+    def delta(self, y_pred, y_true):
+        return y_pred - y_true
+
+    def function(self, packed_coef_inter, X_batch=None, y_batch=None):
+        if X_batch is None:
+            X_batch = self.X
+        if y_batch is None:
+            y_batch = self.y
+
+        self.neural_net._unpack(packed_coef_inter)
+
+        n_samples = X_batch.shape[0]
+        coef_regs = sum(layer.coef_reg(layer.coef_) for layer in self.neural_net.layers
+                        if isinstance(layer, ParamLayer)) / (2 * n_samples)
+        inter_regs = sum(layer.inter_reg(layer.inter_) for layer in self.neural_net.layers
+                         if isinstance(layer, ParamLayer) and layer.fit_intercept) / (2 * n_samples)
+        return 1 / (2 * n_samples) * self.loss(self.neural_net.forward(X_batch), y_batch) + coef_regs + inter_regs
+
+    def jacobian(self, packed_coef_inter, X_batch=None, y_batch=None):
+        if X_batch is None:
+            X_batch = self.X
+        if y_batch is None:
+            y_batch = self.y
+
+        self.neural_net._unpack(packed_coef_inter)
+
+        n_samples = X_batch.shape[0]
+        delta = 1 / n_samples * self.delta(self.neural_net.forward(X_batch), y_batch)
+        return self.neural_net._pack(*self.neural_net.backward(delta))
+
+
+class MeanSquaredError(NeuralNetworkLoss):
+    r"""
+    Compute the mean squared error loss for regression as:
+
+    .. math::
+
+        L(y_{pred}, y_{true}) = \sum (y_{pred} - y_{true})^2
+    """
+
+    def x_star(self):
+        if (len(self.neural_net.layers) == 1 and
+                isinstance(self.neural_net.layers[-1].activation, Linear) and
+                isinstance(self.neural_net.layers[-1].coef_reg, L2) and
+                not self.neural_net.layers[-1].fit_intercept):
+            if not hasattr(self, 'x_opt'):
+                if self.neural_net.layers[-1].coef_reg.lmbda == 0.:
+                    self.x_opt = np.linalg.inv(self.X.T.dot(self.X)).dot(self.X.T).dot(self.y)
+                else:
+                    self.x_opt = np.linalg.inv(self.X.T.dot(self.X) + np.eye(self.ndim) *
+                                               self.neural_net.layers[-1].coef_reg.lmbda).dot(self.X.T).dot(self.y)
+            return self.x_opt
+        return np.full(fill_value=np.nan, shape=self.ndim)
+
+    def f_star(self):
+        if not np.isnan(self.x_star()).all():
+            return self.function(self.x_star())
+        return np.inf
+
+    def loss(self, y_pred, y_true):
+        return np.sum(np.square(y_pred - y_true))
+
+
+class MeanAbsoluteError(NeuralNetworkLoss):
+    r"""
+    Compute the mean absolute error loss for regression as:
+
+    .. math::
+
+        L(y_{pred}, y_{true}) = \sum \lvert y_{pred} - y_{true} \rvert
+    """
+
+    def loss(self, y_pred, y_true):
+        return np.sum(np.abs(y_pred - y_true))
+
+    def delta(self, y_pred, y_true):
+        return np.sign(y_pred - y_true)
+
+
+class BinaryCrossEntropy(NeuralNetworkLoss):
+    r"""Binary Cross-Entropy aka Sigmoid Cross-Entropy loss
+    function for binary and multi-label classification
+    or regression between 0 and 1 with sigmoid output layer:
+
+    .. math::
+
+        L(y_{pred}, y_{true}) = -\sum \left[ y_{true} \log(y_{pred}) +
+        (1 - y_{true}) \log(1 - y_{pred}) \right]
+    """
+
+    def loss(self, y_pred, y_true):
+        return -np.sum(xlogy(y_true, y_pred) + xlogy(1. - y_true, 1. - y_pred))
+
+
+class CategoricalCrossEntropy(NeuralNetworkLoss):
+    r"""Categorical Cross-Entropy loss function for multi-class (single-label)
+    classification with softmax output layer and one-hot encoded target data:
+
+    .. math::
+
+        L(y_{pred}, y_{true}) = -\sum y_{true} \log(y_{pred})
+    """
+
+    def loss(self, y_pred, y_true):
+        return -np.sum(xlogy(y_true, y_pred))
+
+    def delta(self, y_pred, y_true):
+        # according to: https://deepnotes.io/softmax-crossentropy
+        one_hot_mask = y_true.astype(bool)
+        y_pred[one_hot_mask] -= 1.
+        return y_pred
+
+
+class SparseCategoricalCrossEntropy(NeuralNetworkLoss):
+    """Sparse Categorical Cross-Entropy loss function for multi-class
+    (single-label) classification with softmax output layer"""
+
+    def loss(self, y_pred, y_true):
+        assert y_pred.shape[0] == y_true.shape[0]
+        return -np.sum(np.log(y_pred[np.arange(y_pred.shape[0]), y_true.astype(int).ravel()]))
+
+    def delta(self, y_pred, y_true):
+        y_pred[np.arange(y_pred.shape[0]), y_true.astype(int).ravel()] -= 1.
+        return y_pred
+
+
+mean_squared_error = MeanSquaredError
+mean_absolute_error = MeanAbsoluteError
+binary_cross_entropy = BinaryCrossEntropy
+categorical_cross_entropy = CategoricalCrossEntropy
+sparse_categorical_cross_entropy = SparseCategoricalCrossEntropy

optiml/ml/neural_network/regularizers.py

@@ -0,0 +1,87 @@
+from abc import ABC
+
+import numpy as np
+
+
+class Regularizer(ABC):
+    """
+    Base abstract class for all regularizers. A regularizer penalizes the
+    magnitude of the parameters and exposes both its ``function`` and its
+    ``jacobian``.
+    """
+
+    def __init__(self, lmbda=0.):
+        """
+        Parameters
+        ----------
+
+        lmbda : float, default=0.
+            Regularization strength. The higher the value, the stronger
+            the penalty on the parameters.
+        """
+        self.lmbda = lmbda
+
+    def function(self, theta):
+        raise NotImplementedError
+
+    def jacobian(self, theta):
+        raise NotImplementedError
+
+    def __call__(self, theta):
+        return self.function(theta)
+
+
+class L1(Regularizer):
+    r"""
+    L1 (Lasso) regularizer:
+
+    .. math::
+
+        R(\theta) = \lambda \sum \lvert \theta \rvert
+    """
+
+    def __init__(self, lmbda=0.):
+        """
+        Parameters
+        ----------
+
+        lmbda : float, default=0.
+            Regularization strength.
+        """
+        super(L1, self).__init__(lmbda)
+
+    def function(self, theta):
+        return self.lmbda * np.sum(np.abs(theta))
+
+    def jacobian(self, theta):
+        return self.lmbda * np.sign(theta)
+
+
+class L2(Regularizer):
+    r"""
+    L2 (Ridge) regularizer:
+
+    .. math::
+
+        R(\theta) = \lambda \sum \theta^2
+    """
+
+    def __init__(self, lmbda=0.):
+        """
+        Parameters
+        ----------
+
+        lmbda : float, default=0.
+            Regularization strength.
+        """
+        super(L2, self).__init__(lmbda)
+
+    def function(self, theta):
+        return self.lmbda * np.sum(np.square(theta))
+
+    def jacobian(self, theta):
+        return self.lmbda * theta
+
+
+l1 = L1()
+l2 = L2()

optiml/ml/svm/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ['SVM', 'SVC', 'SVR']
+
+from ._base import SVM, SVC, SVR