scikit-network 0.28.3__cp39-cp39-macosx_12_0_arm64.whl

sknetwork/gnn/layer.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+Created on Thu Apr 21 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Optional, Union
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.gnn.activation import BaseActivation
+from sknetwork.gnn.loss import BaseLoss
+from sknetwork.gnn.base_layer import BaseLayer
+from sknetwork.utils.check import add_self_loops
+from sknetwork.linalg import diag_pinv
+
+
+class Convolution(BaseLayer):
+    """Graph convolutional layer.
+
+    Apply the following function to the embedding :math:`X`:
+
+    :math:`\\sigma(\\bar AXW + b)`,
+
+    where :math:`\\bar A` is the normalized adjacency matrix (possibly with inserted self-embeddings),
+    :math:`W`, :math:`b` are trainable parameters and :math:`\\sigma` is the activation function.
+
+    Parameters
+    ----------
+    layer_type : str
+        Layer type. Can be either ``'Conv'``, convolutional operator as in [1] or ``'Sage'``, as in [2].
+    out_channels: int
+        Dimension of the output.
+    activation: str (default = ``'Relu'``) or custom activation.
+        Activation function.
+        If a string, can be either ``'Identity'``, ``'Relu'``, ``'Sigmoid'`` or ``'Softmax'``.
+    use_bias: bool (default = `True`)
+        If ``True``, add a bias vector.
+    normalization: str (default = ``'both'``)
+        Normalization of the adjacency matrix for message passing.
+        Can be either `'left'`` (left normalization by the degrees), ``'right'`` (right normalization by the degrees),
+        ``'both'`` (symmetric normalization by the square root of degrees, default) or ``None`` (no normalization).
+    self_embeddings: bool (default = `True`)
+        If ``True``, consider self-embedding in addition to neighbors embedding for each node of the graph.
+    sample_size: int (default = 25)
+        Size of neighborhood sampled for each node. Used only for ``'Sage'`` layer.
+
+    Attributes
+    ----------
+    weight: np.ndarray,
+        Trainable weight matrix.
+    bias: np.ndarray
+        Bias vector.
+    embedding: np.ndarray
+        Embedding of the nodes (before activation).
+    output: np.ndarray
+        Output of the layer (after activation).
+
+    References
+    ----------
+    [1] Kipf, T., & Welling, M. (2017).
+    `Semi-supervised Classification with Graph Convolutional Networks.
+    <https://arxiv.org/pdf/1609.02907.pdf>`_
+    5th International Conference on Learning Representations.
+
+    [2] Hamilton, W. Ying, R., & Leskovec, J. (2017)
+    `Inductive Representation Learning on Large Graphs.
+    <https://arxiv.org/pdf/1706.02216.pdf>`_
+    NIPS
+    """
+    def __init__(self, layer_type: str, out_channels: int, activation: Optional[Union[BaseActivation, str]] = 'Relu',
+                 use_bias: bool = True, normalization: str = 'both', self_embeddings: bool = True,
+                 sample_size: int = None, loss: Optional[Union[BaseLoss, str]] = None):
+        super(Convolution, self).__init__(layer_type, out_channels, activation, use_bias, normalization,
+                                          self_embeddings, sample_size, loss)
+
+    def forward(self, adjacency: Union[sparse.csr_matrix, np.ndarray],
+                features: Union[sparse.csr_matrix, np.ndarray]) -> np.ndarray:
+        """Compute graph convolution.
+
+        Parameters
+        ----------
+        adjacency
+            Adjacency matrix of the graph.
+        features : sparse.csr_matrix, np.ndarray
+            Input feature of shape :math:`(n, d)` with :math:`n` the number of nodes in the graph and :math:`d`
+            the size of feature space.
+
+        Returns
+        -------
+        output: np.ndarray
+            Output of the layer.
+        """
+        if not self.weights_initialized:
+            self._initialize_weights(features.shape[1])
+
+        n_row, n_col = adjacency.shape
+
+        weights = adjacency.dot(np.ones(n_col))
+        if self.normalization == 'left':
+            d_inv = diag_pinv(weights)
+            adjacency = d_inv.dot(adjacency)
+        elif self.normalization == 'right':
+            d_inv = diag_pinv(weights)
+            adjacency = adjacency.dot(d_inv)
+        elif self.normalization == 'both':
+            d_inv = diag_pinv(np.sqrt(weights))
+            adjacency = d_inv.dot(adjacency).dot(d_inv)
+
+        if self.self_embeddings:
+            adjacency = add_self_loops(adjacency)
+
+        message = adjacency.dot(features)
+        embedding = message.dot(self.weight)
+
+        if self.use_bias:
+            embedding += self.bias
+
+        output = self.activation.output(embedding)
+
+        self.embedding = embedding
+        self.output = output
+
+        return output
+
+
+def get_layer(layer: Union[BaseLayer, str] = 'conv', **kwargs) -> BaseLayer:
+    """Get layer.
+
+    Parameters
+    ----------
+    layer : str or custom layer
+        If a string, must be either ``'Conv'`` (Convolution) or ``'Sage'`` (GraphSAGE).
+
+    Returns
+    -------
+    Layer object.
+    """
+    if issubclass(type(layer), BaseLayer):
+        return layer
+    elif type(layer) == str:
+        layer = layer.lower()
+        if layer in ['conv', 'gcnconv', 'graphconv']:
+            return Convolution('conv', **kwargs)
+        elif layer in ['sage', 'sageconv', 'graphsage']:
+            kwargs['normalization'] = 'left'
+            kwargs['self_embeddings'] = True
+            return Convolution('sage', **kwargs)
+        else:
+            raise ValueError("Layer name must be \"Conv\" or \"SAGEConv\".")
+    else:
+        raise TypeError("Layer must be a string or a \"BaseLayer\" object.")

sknetwork/gnn/layers.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+Created on Thu Apr 21 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Union
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.gnn.activation import get_activation_function
+from sknetwork.gnn.base_layer import BaseLayer
+from sknetwork.gnn.utils import has_self_loops, add_self_loops
+from sknetwork.linalg import diag_pinv
+
+
+class GCNConv(BaseLayer):
+    """Graph convolutional operator.
+
+    :math:`H^{\derivative}=\sigma(D^{-1/2}\hat{A}D^{-1/2}HW + b)`,
+
+    where :math:`\hat{A} = A + I` denotes the adjacency matrix with inserted self-loops and
+    :math:`D` its diagonal degree matrix. :math:`W` and :math:`b` are trainable parameters and
+    :math:`\\sigma` is the activation function.
+
+    Parameters
+    ----------
+    out_channels: int
+        Size of each output sample.
+    activation: str (default = ``'Relu'``)
+        Activation function.
+        Can be either:
+
+        * ``'relu'``, the rectified linear unit function, returns f(x) = max(0, x)
+        * ``'sigmoid'``, the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).
+        * ``'softmax'``, the softmax function, returns f(x) = exp(x) / sum(exp(x))
+    use_bias: bool (default = `True`)
+        If ``True``, add a bias vector.
+    normalization: str (default = ``'Both'``)
+        Normalization of the adjacency matrix for message passing.
+        Can be either:
+
+        * ``'left'``, left normalization by the vector of degrees
+        * ``'right'``, right normalization by the vector of degrees
+        * ``'both'``,  symmetric normalization by the square root of degrees
+    self_loops: bool (default = `True`)
+        If ``True``, add self-loops to each node in the graph.
+
+    Attributes
+    ----------
+    weight: np.ndarray,
+        Trainable weight matrix.
+    bias: np.ndarray
+        Bias vector.
+    update: np.ndarray
+        Embedding of the nodes before the activation function.
+    embedding: np.ndarray
+        Embedding of the nodes after convolution layer.
+
+    References
+    ----------
+    Kipf, T., & Welling, M. (2017).
+    `Semi-supervised Classification with Graph Convolutional Networks.
+    <https://arxiv.org/pdf/1609.02907.pdf>`_
+    5th International Conference on Learning Representations.
+
+    He, K. & Zhang, X. & Ren, S. & Sun, J. (2015).
+    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification.
+    <https://arxiv.org/pdf/1502.01852.pdf>`_
+    Proceedings of the IEEE International Conference on Computer Vision (ICCV).
+    """
+    def __init__(self, out_channels: int, activation: str = 'Relu', use_bias: bool = True,
+                 normalization: str = 'Both', self_loops: bool = True):
+        super(GCNConv, self).__init__(out_channels, activation, use_bias, normalization, self_loops)
+
+    def forward(self, adjacency: Union[sparse.csr_matrix, np.ndarray],
+                features: Union[sparse.csr_matrix, np.ndarray]) -> np.ndarray:
+        """Compute graph convolution.
+
+        Parameters
+        ----------
+        adjacency
+            Adjacency matrix of the graph.
+        features : sparse.csr_matrix, np.ndarray
+            Input feature of shape :math:`(n, d)` with :math:`n` the number of nodes in the graph and :math:`d`
+            the size of feature space.
+
+        Returns
+        -------
+        embedding: p.ndarray
+            Node embedding.
+        """
+        if not self.weights_initialized:
+            self._initialize_weights(features.shape[1])
+
+        n_row, n_col = adjacency.shape
+
+        if self.self_loops:
+            if not has_self_loops(adjacency):
+                adjacency = add_self_loops(adjacency)
+
+        weights = adjacency.dot(np.ones(n_col))
+        if self.normalization == 'left':
+            d_inv = diag_pinv(weights)
+            adjacency = d_inv.dot(adjacency)
+        elif self.normalization == 'right':
+            d_inv = diag_pinv(weights)
+            adjacency = adjacency.dot(d_inv)
+        elif self.normalization == 'both':
+            d_inv = diag_pinv(np.sqrt(weights))
+            adjacency = d_inv.dot(adjacency).dot(d_inv)
+
+        msg = adjacency.dot(features)
+        update = msg.dot(self.weight)
+
+        if self.use_bias:
+            update += self.bias
+
+        activation_function = get_activation_function(self.activation)
+        embedding = activation_function(update)
+
+        # Keep track of results for backprop
+        self.embedding = embedding
+        self.update = update
+
+        return embedding

sknetwork/gnn/loss.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+Created in April 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+@author: Thomas Bonald <bonald@enst.fr>
+"""
+
+from typing import Union
+
+import numpy as np
+
+from sknetwork.gnn.base_activation import BaseLoss
+from sknetwork.gnn.activation import Sigmoid, Softmax
+
+
+class CrossEntropy(BaseLoss, Softmax):
+    """Cross entropy loss with softmax activation.
+
+    For a single sample with value :math:`x` and true label :math:`y`, the cross-entropy loss
+    is:
+
+     :math:`-\\sum_i 1_{\\{y=i\\}}  \\log (p_i)`
+
+     with
+
+     :math:`p_i = e^{x_i} / \\sum_j e^{x_j}`.
+
+    For :math:`n` samples, return the average loss.
+    """
+    def __init__(self):
+        super(CrossEntropy, self).__init__()
+        self.name = 'Cross entropy'
+
+    @staticmethod
+    def loss(signal: np.ndarray, labels: np.ndarray) -> float:
+        """Get loss value.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+            The number of channels must be at least 2.
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+
+        Returns
+        -------
+        value : float
+            Loss value.
+        """
+        n = len(labels)
+        probs = Softmax.output(signal)
+
+        # for numerical stability
+        eps = 1e-15
+        probs = np.clip(probs, eps, 1 - eps)
+
+        value = -np.log(probs[np.arange(n), labels]).sum()
+
+        return value / n
+
+    @staticmethod
+    def loss_gradient(signal: np.ndarray, labels: np.ndarray) -> np.ndarray:
+        """Get the gradient of the loss function (including activation).
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+        Returns
+        -------
+        gradient: float
+            Gradient of the loss function.
+        """
+        probs = Softmax.output(signal)
+        one_hot_encoding = np.zeros_like(probs)
+        one_hot_encoding[np.arange(len(labels)), labels] = 1
+        gradient = probs - one_hot_encoding
+
+        return gradient
+
+
+class BinaryCrossEntropy(BaseLoss, Sigmoid):
+    """Binary cross entropy loss with sigmoid activation.
+
+    For a single sample with true label :math:`y` and predicted probability :math:`p`, the binary cross-entropy loss
+    is:
+
+     :math:`-y  \\log (p) - (1-y)  \\log (1 - p).`
+
+    For :math:`n` samples, return the average loss.
+    """
+    def __init__(self):
+        super(BinaryCrossEntropy, self).__init__()
+        self.name = 'Binary cross entropy'
+
+    @staticmethod
+    def loss(signal: np.ndarray, labels: np.ndarray) -> float:
+        """Get loss value.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+            The number of channels must be at least 2.
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+
+        Returns
+        -------
+        value : float
+            Loss value.
+        """
+        probs = Sigmoid.output(signal)
+        n = len(labels)
+
+        # for numerical stability
+        eps = 1e-15
+        probs = np.clip(probs, eps, 1 - eps)
+
+        if probs.shape[1] == 1:
+            # binary labels
+            value = -np.log(probs[labels > 0]).sum()
+            value -= np.log((1 - probs)[labels == 0]).sum()
+        else:
+            # general case
+            value = -np.log(1 - probs)
+            value[np.arange(n), labels] = -np.log(probs[np.arange(n), labels])
+            value = value.sum()
+
+        return value / n
+
+    @staticmethod
+    def loss_gradient(signal: np.ndarray, labels: np.ndarray) -> np.ndarray:
+        """Get the gradient of the loss function (including activation).
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+        Returns
+        -------
+        gradient: float
+            Gradient of the loss function.
+        """
+        probs = Sigmoid.output(signal)
+        gradient = (probs.T - labels).T
+
+        return gradient
+
+
+def get_loss(loss: Union[BaseLoss, str] = 'CrossEntropyLoss') -> BaseLoss:
+    """Instantiate loss function according to parameters.
+
+    Parameters
+    ----------
+    loss : str or loss function.
+        Which loss function to use. Can be ``'CrossEntropy'`` or ``'BinaryCrossEntropy'`` or custom loss.
+
+    Returns
+    -------
+    Loss function object.
+    """
+    if issubclass(type(loss), BaseLoss):
+        return loss
+    elif type(loss) == str:
+        loss = loss.lower().replace(' ', '')
+        if loss in ['crossentropy', 'ce']:
+            return CrossEntropy()
+        elif loss in ['binarycrossentropy', 'bce']:
+            return BinaryCrossEntropy()
+        else:
+            raise ValueError("Loss must be either \"CrossEntropy\" or \"BinaryCrossEntropy\".")
+    else:
+        raise TypeError("Loss must be either an \"BaseLoss\" object or a string.")

sknetwork/gnn/neighbor_sampler.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Union
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.utils import get_degrees
+
+
+class UniformNeighborSampler:
+    """Neighbor node sampler.
+
+    Uniformly sample nodes over neighborhood.
+
+    Parameters
+    ----------
+    sample_size : int
+        Size of neighborhood sampled for each node.
+    """
+    def __init__(self, sample_size: int):
+        self.sample_size = sample_size
+
+    def _sample_indexes(self, size: int) -> np.ndarray:
+        """Randomly chose indexes without replacement.
+
+        Parameters
+        ----------
+        size : int
+            Highest index available. This index is used if lower than a threshold.
+
+        Returns
+        -------
+            Array of sampled indexes.
+        """
+        return np.random.choice(size, size=min(size, self.sample_size), replace=False)
+
+    def __call__(self, adjacency: Union[sparse.csr_matrix, np.ndarray]) -> sparse.csr_matrix:
+        """Apply node sampling on each node and return filtered adjacency matrix.
+
+        Parameters
+        ----------
+        adjacency
+            Adjacency matrix of the graph.
+
+        Returns
+        -------
+            Filtered adjacency matrix using node sampling.
+        """
+        n_row, _ = adjacency.shape
+        sampled_adjacency = adjacency.copy()
+
+        degrees = get_degrees(adjacency)
+        neighbor_samples = list(map(self._sample_indexes, degrees))
+
+        for i, neighbors in enumerate(neighbor_samples):
+            sampled_adjacency.data[sampled_adjacency.indptr[i]:sampled_adjacency.indptr[i + 1]] = np.zeros(degrees[i])
+            sampled_adjacency.data[sampled_adjacency.indptr[i]:sampled_adjacency.indptr[i + 1]][neighbors] = 1
+
+        sampled_adjacency.eliminate_zeros()
+
+        return sampled_adjacency

sknetwork/gnn/optimizer.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+"""
+Created on Thu Apr 21 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Union, TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from sknetwork.gnn.base import BaseGNN
+
+
+class BaseOptimizer:
+    """Base class for optimizers.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    """
+
+    def __init__(self, learning_rate):
+        self.learning_rate = learning_rate
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values.
+
+        Parameters
+        ----------
+        gnn: BaseGNNClassifier
+            Model containing parameters to update.
+        """
+
+
+class GD(BaseOptimizer):
+    """Gradient Descent optimizer.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    """
+
+    def __init__(self, learning_rate: float = 0.01):
+        super(GD, self).__init__(learning_rate)
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values.
+
+        Parameters
+        ----------
+        gnn: BaseGNNClassifier
+            Model containing parameters to update.
+        """
+        for idx, layer in enumerate(gnn.layers):
+            layer.weight = layer.weight - self.learning_rate * gnn.derivative_weight[idx]
+            layer.bias = layer.bias - self.learning_rate * gnn.derivative_bias[idx]
+
+
+class ADAM(BaseOptimizer):
+    """Adam optimizer.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    beta1, beta2: float
+        Coefficients used for computing running averages of gradients.
+    eps: float (default = 1e-8)
+        Term added to the denominator to improve stability.
+
+    References
+    ----------
+    Kingma, D. P., & Ba, J. (2014).
+    `Adam: A method for stochastic optimization.
+    <https://arxiv.org/pdf/1412.6980.pdf>`_
+    3rd International Conference for Learning Representation.
+    """
+
+    def __init__(self, learning_rate: float = 0.01, beta1: float = 0.9, beta2: float = 0.999,
+                 eps: float = 1e-8):
+        super(ADAM, self).__init__(learning_rate)
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.eps = eps
+        self.m_derivative_weight, self.v_derivative_weight = [], []
+        self.m_derivative_bias, self.v_derivative_bias = [], []
+        self.t = 0
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values and parameters.
+
+        Parameters
+        ----------
+        gnn: `BaseGNNClassifier`
+            Model containing parameters to update.
+        """
+        if self.t == 0:
+            self.m_derivative_weight, self.v_derivative_weight = \
+                [np.zeros(x.shape) for x in gnn.derivative_weight], [np.zeros(x.shape) for x in gnn.derivative_weight]
+            self.m_derivative_bias, self.v_derivative_bias = \
+                [np.zeros(x.shape) for x in gnn.derivative_bias], [np.zeros(x.shape) for x in gnn.derivative_bias]
+
+        for idx, layer in enumerate(gnn.layers):
+            self.t += 1
+
+            # Moving averages
+            self.m_derivative_weight[idx] = \
+                self.beta1 * self.m_derivative_weight[idx] + (1 - self.beta1) * gnn.derivative_weight[idx]
+            self.m_derivative_bias[idx] = \
+                self.beta1 * self.m_derivative_bias[idx] + (1 - self.beta1) * gnn.derivative_bias[idx]
+
+            self.v_derivative_weight[idx] = \
+                self.beta2 * self.v_derivative_weight[idx] + (1 - self.beta2) * (gnn.derivative_weight[idx] ** 2)
+            self.v_derivative_bias[idx] = \
+                self.beta2 * self.v_derivative_bias[idx] + (1 - self.beta2) * (gnn.derivative_bias[idx] ** 2)
+
+            # Correcting moving averages
+            denom_1 = (1 - self.beta1 ** self.t)
+            denom_2 = (1 - self.beta2 ** self.t)
+
+            m_derivative_weight_corr = self.m_derivative_weight[idx] / denom_1
+            m_derivative_bias_corr = self.m_derivative_bias[idx] / denom_1
+            v_derivative_weight_corr = self.v_derivative_weight[idx] / denom_2
+            v_derivative_bias_corr = self.v_derivative_bias[idx] / denom_2
+
+            # Parameters update
+            layer.weight = \
+                layer.weight - (self.learning_rate * m_derivative_weight_corr) / (np.sqrt(v_derivative_weight_corr)
+                                                                                  + self.eps)
+            layer.bias = \
+                layer.bias - (self.learning_rate * m_derivative_bias_corr) / (np.sqrt(v_derivative_bias_corr)
+                                                                              + self.eps)
+
+
+def get_optimizer(optimizer: Union[BaseOptimizer, str] = 'Adam', learning_rate: float = 0.01) -> BaseOptimizer:
+    """Instantiate optimizer according to parameters.
+
+    Parameters
+    ----------
+    optimizer : str or optimizer
+        Which optimizer to use. Can be ``'Adam'`` or ``'GD'`` or custom optimizer.
+    learning_rate: float
+        Learning rate.
+
+    Returns
+    -------
+    Optimizer object
+    """
+    if issubclass(type(optimizer), BaseOptimizer):
+        return optimizer
+    elif type(optimizer) == str:
+        optimizer = optimizer.lower()
+        if optimizer == 'adam':
+            return ADAM(learning_rate=learning_rate)
+        elif optimizer in ['gd', 'gradient']:
+            return GD(learning_rate=learning_rate)
+        else:
+            raise ValueError("Optimizer must be either \"Adam\" or \"GD\" (Gradient Descent).")
+    else:
+        raise TypeError("Optimizer must be either an \"BaseOptimizer\" object or a string.")

sknetwork/gnn/tests/__init__.py

@@ -0,0 +1 @@
+"""tests for gnn"""