scikit-network 0.33.4__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

sknetwork/gnn/loss.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+Created in April 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+@author: Thomas Bonald <bonald@enst.fr>
+"""
+
+from typing import Union
+
+import numpy as np
+
+from sknetwork.gnn.base_activation import BaseLoss
+from sknetwork.gnn.activation import Sigmoid, Softmax
+
+
+class CrossEntropy(BaseLoss, Softmax):
+    """Cross entropy loss with softmax activation.
+
+    For a single sample with value :math:`x` and true label :math:`y`, the cross-entropy loss
+    is:
+
+     :math:`-\\sum_i 1_{\\{y=i\\}}  \\log (p_i)`
+
+     with
+
+     :math:`p_i = e^{x_i} / \\sum_j e^{x_j}`.
+
+    For :math:`n` samples, return the average loss.
+    """
+    def __init__(self):
+        super(CrossEntropy, self).__init__()
+        self.name = 'Cross entropy'
+
+    @staticmethod
+    def loss(signal: np.ndarray, labels: np.ndarray) -> float:
+        """Get loss value.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+            The number of channels must be at least 2.
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+
+        Returns
+        -------
+        value : float
+            Loss value.
+        """
+        n = len(labels)
+        probs = Softmax.output(signal)
+
+        # for numerical stability
+        eps = 1e-10
+        probs = np.clip(probs, eps, 1 - eps)
+
+        value = -np.log(probs[np.arange(n), labels]).sum()
+
+        return value / n
+
+    @staticmethod
+    def loss_gradient(signal: np.ndarray, labels: np.ndarray) -> np.ndarray:
+        """Get the gradient of the loss function (including activation).
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+        Returns
+        -------
+        gradient: float
+            Gradient of the loss function.
+        """
+        probs = Softmax.output(signal)
+        one_hot_encoding = np.zeros_like(probs)
+        one_hot_encoding[np.arange(len(labels)), labels] = 1
+        gradient = probs - one_hot_encoding
+
+        return gradient
+
+
+class BinaryCrossEntropy(BaseLoss, Sigmoid):
+    """Binary cross entropy loss with sigmoid activation.
+
+    For a single sample with true label :math:`y` and predicted probability :math:`p`, the binary cross-entropy loss
+    is:
+
+     :math:`-y  \\log (p) - (1-y)  \\log (1 - p).`
+
+    For :math:`n` samples, return the average loss.
+    """
+    def __init__(self):
+        super(BinaryCrossEntropy, self).__init__()
+        self.name = 'Binary cross entropy'
+
+    @staticmethod
+    def loss(signal: np.ndarray, labels: np.ndarray) -> float:
+        """Get loss value.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+            The number of channels must be at least 2.
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+
+        Returns
+        -------
+        value : float
+            Loss value.
+        """
+        probs = Sigmoid.output(signal)
+        n = len(labels)
+
+        # for numerical stability
+        eps = 1e-15
+        probs = np.clip(probs, eps, 1 - eps)
+
+        if probs.shape[1] == 1:
+            # binary labels
+            value = -np.log(probs[labels > 0]).sum()
+            value -= np.log((1 - probs)[labels == 0]).sum()
+        else:
+            # general case
+            value = -np.log(1 - probs)
+            value[np.arange(n), labels] = -np.log(probs[np.arange(n), labels])
+            value = value.sum()
+
+        return value / n
+
+    @staticmethod
+    def loss_gradient(signal: np.ndarray, labels: np.ndarray) -> np.ndarray:
+        """Get the gradient of the loss function (including activation).
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+        Returns
+        -------
+        gradient: float
+            Gradient of the loss function.
+        """
+        probs = Sigmoid.output(signal)
+        gradient = (probs.T - labels).T
+
+        return gradient
+
+
+def get_loss(loss: Union[BaseLoss, str] = 'CrossEntropyLoss') -> BaseLoss:
+    """Instantiate loss function according to parameters.
+
+    Parameters
+    ----------
+    loss : str or loss function.
+        Which loss function to use. Can be ``'CrossEntropy'`` or ``'BinaryCrossEntropy'`` or custom loss.
+
+    Returns
+    -------
+    Loss function object.
+    """
+    if issubclass(type(loss), BaseLoss):
+        return loss
+    elif type(loss) == str:
+        loss = loss.lower().replace(' ', '')
+        if loss in ['crossentropy', 'ce']:
+            return CrossEntropy()
+        elif loss in ['binarycrossentropy', 'bce']:
+            return BinaryCrossEntropy()
+        else:
+            raise ValueError("Loss must be either \"CrossEntropy\" or \"BinaryCrossEntropy\".")
+    else:
+        raise TypeError("Loss must be either an \"BaseLoss\" object or a string.")

sknetwork/gnn/neighbor_sampler.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Union
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.utils import get_degrees
+
+
+class UniformNeighborSampler:
+    """Neighbor node sampler.
+
+    Uniformly sample nodes over neighborhood.
+
+    Parameters
+    ----------
+    sample_size : int
+        Size of neighborhood sampled for each node.
+    """
+    def __init__(self, sample_size: int):
+        self.sample_size = sample_size
+
+    def _sample_indexes(self, size: int) -> np.ndarray:
+        """Randomly chose indexes without replacement.
+
+        Parameters
+        ----------
+        size : int
+            Highest index available. This index is used if lower than a threshold.
+
+        Returns
+        -------
+            Array of sampled indexes.
+        """
+        return np.random.choice(size, size=min(size, self.sample_size), replace=False)
+
+    def __call__(self, adjacency: Union[sparse.csr_matrix, np.ndarray]) -> sparse.csr_matrix:
+        """Apply node sampling on each node and return filtered adjacency matrix.
+
+        Parameters
+        ----------
+        adjacency
+            Adjacency matrix of the graph.
+
+        Returns
+        -------
+            Filtered adjacency matrix using node sampling.
+        """
+        n_row, _ = adjacency.shape
+        sampled_adjacency = adjacency.copy()
+
+        degrees = get_degrees(adjacency)
+        neighbor_samples = list(map(self._sample_indexes, degrees))
+
+        for i, neighbors in enumerate(neighbor_samples):
+            sampled_adjacency.data[sampled_adjacency.indptr[i]:sampled_adjacency.indptr[i + 1]] = np.zeros(degrees[i])
+            sampled_adjacency.data[sampled_adjacency.indptr[i]:sampled_adjacency.indptr[i + 1]][neighbors] = 1
+
+        sampled_adjacency.eliminate_zeros()
+
+        return sampled_adjacency

sknetwork/gnn/optimizer.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+"""
+Created on Thu Apr 21 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Union, TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from sknetwork.gnn.base import BaseGNN
+
+
+class BaseOptimizer:
+    """Base class for optimizers.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    """
+
+    def __init__(self, learning_rate):
+        self.learning_rate = learning_rate
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values.
+
+        Parameters
+        ----------
+        gnn: BaseGNNClassifier
+            Model containing parameters to update.
+        """
+
+
+class GD(BaseOptimizer):
+    """Gradient Descent optimizer.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    """
+
+    def __init__(self, learning_rate: float = 0.01):
+        super(GD, self).__init__(learning_rate)
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values.
+
+        Parameters
+        ----------
+        gnn: BaseGNNClassifier
+            Model containing parameters to update.
+        """
+        for idx, layer in enumerate(gnn.layers):
+            layer.weight = layer.weight - self.learning_rate * gnn.derivative_weight[idx]
+            layer.bias = layer.bias - self.learning_rate * gnn.derivative_bias[idx]
+
+
+class ADAM(BaseOptimizer):
+    """Adam optimizer.
+
+    Parameters
+    ----------
+    learning_rate: float (default = 0.01)
+        Learning rate for updating weights.
+    beta1, beta2: float
+        Coefficients used for computing running averages of gradients.
+    eps: float (default = 1e-8)
+        Term added to the denominator to improve stability.
+
+    References
+    ----------
+    Kingma, D. P., & Ba, J. (2014).
+    `Adam: A method for stochastic optimization.
+    <https://arxiv.org/pdf/1412.6980.pdf>`_
+    3rd International Conference for Learning Representation.
+    """
+
+    def __init__(self, learning_rate: float = 0.01, beta1: float = 0.9, beta2: float = 0.999,
+                 eps: float = 1e-8):
+        super(ADAM, self).__init__(learning_rate)
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.eps = eps
+        self.m_derivative_weight, self.v_derivative_weight = [], []
+        self.m_derivative_bias, self.v_derivative_bias = [], []
+        self.t = 0
+
+    def step(self, gnn: BaseGNN):
+        """Update model parameters according to gradient values and parameters.
+
+        Parameters
+        ----------
+        gnn: `BaseGNNClassifier`
+            Model containing parameters to update.
+        """
+        if self.t == 0:
+            self.m_derivative_weight, self.v_derivative_weight = \
+                [np.zeros(x.shape) for x in gnn.derivative_weight], [np.zeros(x.shape) for x in gnn.derivative_weight]
+            self.m_derivative_bias, self.v_derivative_bias = \
+                [np.zeros(x.shape) for x in gnn.derivative_bias], [np.zeros(x.shape) for x in gnn.derivative_bias]
+
+        for idx, layer in enumerate(gnn.layers):
+            self.t += 1
+
+            # Moving averages
+            self.m_derivative_weight[idx] = \
+                self.beta1 * self.m_derivative_weight[idx] + (1 - self.beta1) * gnn.derivative_weight[idx]
+            self.m_derivative_bias[idx] = \
+                self.beta1 * self.m_derivative_bias[idx] + (1 - self.beta1) * gnn.derivative_bias[idx]
+
+            self.v_derivative_weight[idx] = \
+                self.beta2 * self.v_derivative_weight[idx] + (1 - self.beta2) * (gnn.derivative_weight[idx] ** 2)
+            self.v_derivative_bias[idx] = \
+                self.beta2 * self.v_derivative_bias[idx] + (1 - self.beta2) * (gnn.derivative_bias[idx] ** 2)
+
+            # Correcting moving averages
+            denom_1 = (1 - self.beta1 ** self.t)
+            denom_2 = (1 - self.beta2 ** self.t)
+
+            m_derivative_weight_corr = self.m_derivative_weight[idx] / denom_1
+            m_derivative_bias_corr = self.m_derivative_bias[idx] / denom_1
+            v_derivative_weight_corr = self.v_derivative_weight[idx] / denom_2
+            v_derivative_bias_corr = self.v_derivative_bias[idx] / denom_2
+
+            # Parameters update
+            layer.weight = \
+                layer.weight - (self.learning_rate * m_derivative_weight_corr) / (np.sqrt(v_derivative_weight_corr)
+                                                                                  + self.eps)
+            if layer.use_bias:
+                layer.bias = \
+                    layer.bias - (self.learning_rate * m_derivative_bias_corr) / (np.sqrt(v_derivative_bias_corr)
+                                                                                  + self.eps)
+
+
+def get_optimizer(optimizer: Union[BaseOptimizer, str] = 'Adam', learning_rate: float = 0.01) -> BaseOptimizer:
+    """Instantiate optimizer according to parameters.
+
+    Parameters
+    ----------
+    optimizer : str or optimizer
+        Which optimizer to use. Can be ``'Adam'`` or ``'GD'`` or custom optimizer.
+    learning_rate: float
+        Learning rate.
+
+    Returns
+    -------
+    Optimizer object
+    """
+    if issubclass(type(optimizer), BaseOptimizer):
+        return optimizer
+    elif type(optimizer) == str:
+        optimizer = optimizer.lower()
+        if optimizer == 'adam':
+            return ADAM(learning_rate=learning_rate)
+        elif optimizer in ['gd', 'gradient']:
+            return GD(learning_rate=learning_rate)
+        else:
+            raise ValueError("Optimizer must be either \"Adam\" or \"GD\" (Gradient Descent).")
+    else:
+        raise TypeError("Optimizer must be either an \"BaseOptimizer\" object or a string.")

sknetwork/gnn/tests/__init__.py

@@ -0,0 +1 @@
+"""tests for gnn"""

sknetwork/gnn/tests/test_activation.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""tests for activation"""
+
+import unittest
+
+from sknetwork.gnn.activation import *
+
+
+class TestActivation(unittest.TestCase):
+
+    def test_get_activation(self):
+        self.assertTrue(isinstance(get_activation('Identity'), BaseActivation))
+        self.assertTrue(isinstance(get_activation('Relu'), ReLu))
+        self.assertTrue(isinstance(get_activation('Sigmoid'), Sigmoid))
+        self.assertTrue(isinstance(get_activation('Softmax'), Softmax))
+        with self.assertRaises(ValueError):
+            get_activation('foo')
+
+        base_act = BaseActivation()
+        self.assertTrue(base_act == get_activation(base_act))
+        with self.assertRaises(TypeError):
+            get_activation(0)
+
+    def test_activation_identity(self):
+        activation = get_activation('Identity')
+        signal = np.arange(5)
+        self.assertTrue((activation.output(signal) == signal).all())
+        direction = np.arange(5)
+        self.assertTrue((activation.gradient(signal, direction) == direction).all())
+
+    def test_activation_relu(self):
+        activation = get_activation('ReLu')
+        signal = np.linspace(-2, 2, 5)
+        self.assertTrue((activation.output(signal) == [0., 0., 0., 1., 2.]).all())
+        direction = np.arange(5)
+        self.assertTrue((activation.gradient(signal, direction) == direction * (signal > 0)).all())
+
+    def test_activation_sigmoid(self):
+        activation = get_activation('Sigmoid')
+        signal = np.array([-np.inf, -1.5, 0, 1.5, np.inf])
+        self.assertTrue(np.allclose(activation.output(signal), np.array([0., 0.18242552, 0.5, 0.81757448, 1.])))
+        signal = np.array([[-1000, 1000, 1000]])
+        direction = np.arange(3)
+        self.assertTrue(np.allclose(activation.output(signal), np.array([[0., 1., 1.]])))
+        self.assertTrue(np.allclose(activation.gradient(signal, direction), np.array([[0., 0., 0.]])))
+
+    def test_activation_softmax(self):
+        activation = get_activation('Softmax')
+        signal = np.array([[-1, 0, 3, 5]])
+        output = activation.output(signal)
+        self.assertTrue(np.allclose(output, np.array([[0.0021657, 0.00588697, 0.11824302, 0.87370431]])))
+        signal = np.array([[-1000, 1000, 1000]])
+        direction = np.arange(3)
+        self.assertTrue(np.allclose(activation.output(signal), np.array([[0., 0.5, 0.5]])))
+        self.assertTrue(np.allclose(activation.gradient(signal, direction), np.array([[0., -0.25, 0.25]])))

sknetwork/gnn/tests/test_base.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""tests for base gnn"""
+
+import unittest
+
+import numpy as np
+
+from sknetwork.data.test_graphs import test_graph
+from sknetwork.gnn.base import BaseGNN
+from sknetwork.gnn.gnn_classifier import GNNClassifier
+from sknetwork.gnn.layer import Convolution
+from sknetwork.gnn.optimizer import ADAM
+
+
+class TestBaseGNN(unittest.TestCase):
+
+    def setUp(self) -> None:
+        """Test graph for tests."""
+        self.adjacency = test_graph()
+        self.n = self.adjacency.shape[0]
+        self.features = self.adjacency
+        self.labels = np.array(4 * [0, 1] + 2 * [-1])
+
+    def test_base_gnn_fit(self):
+        gnn = BaseGNN()
+        with self.assertRaises(NotImplementedError):
+            gnn.fit(self.adjacency, self.features, self.labels, test_size=0.2)
+
+    def test_gnn_fit_transform(self):
+        gnn = GNNClassifier(dims=2, layer_types='Conv', activations='Relu', optimizer='GD', verbose=False)
+        embedding = gnn.fit_transform(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        self.assertTrue(len(embedding) == self.n)
+        self.assertTrue(embedding.shape == (self.n, 2))
+
+    def test_gnn_custom_optimizer(self):
+        gnn = GNNClassifier(dims=2, layer_types='Conv', activations='Relu', optimizer=ADAM(beta1=0.5), verbose=False)
+        embedding = gnn.fit_transform(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        self.assertTrue(len(embedding) == self.n)
+        self.assertTrue(embedding.shape == (self.n, 2))
+
+    def test_gnn_custom_layers(self):
+        gnn = GNNClassifier(layers=[Convolution('Conv', 2, loss='CrossEntropy')], optimizer=ADAM(beta1=0.5))
+        embedding = gnn.fit_transform(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        self.assertTrue(len(embedding) == self.n)
+        self.assertTrue(embedding.shape == (self.n, 2))
+
+        gnn = GNNClassifier(layers=[Convolution('SAGEConv', 2, sample_size=5, loss='CrossEntropy')],
+                            optimizer=ADAM(beta1=0.5),)
+        embedding = gnn.fit_transform(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        self.assertTrue(len(embedding) == self.n)
+        self.assertTrue(embedding.shape == (self.n, 2))
+
+    def test_gnn_custom(self):
+        gnn = GNNClassifier(dims=[20, 8, 2], layer_types='conv',
+                            activations=['Relu', 'Sigmoid', 'Softmax'], optimizer='Adam')
+        self.assertTrue(isinstance(gnn, GNNClassifier))
+        self.assertTrue(gnn.layers[-1].activation.name == 'Cross entropy')
+        y_pred = gnn.fit_predict(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        self.assertTrue(len(y_pred) == self.n)
+
+    def test_check_fitted(self):
+        gnn = BaseGNN()
+        with self.assertRaises(ValueError):
+            gnn._check_fitted()
+        gnn = GNNClassifier(dims=2, layer_types='conv', activations='Relu', optimizer='GD')
+        gnn.fit_transform(self.adjacency, self.features, labels=self.labels, n_epochs=1)
+        fit_gnn = gnn._check_fitted()
+        self.assertTrue(isinstance(fit_gnn, GNNClassifier))
+        self.assertTrue(fit_gnn.embedding_ is not None)
+
+    def test_base_gnn_repr(self):
+        gnn = GNNClassifier(dims=[8, 2], layer_types='conv', activations=['Relu', 'Softmax'], optimizer='Adam')
+        self.assertTrue(gnn.__repr__().startswith("GNNClassifier"))
+

sknetwork/gnn/tests/test_base_layer.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""tests for base layer gnn"""
+
+import unittest
+
+import numpy as np
+
+from sknetwork.data.test_graphs import test_graph
+from sknetwork.gnn.base_layer import BaseLayer
+
+
+class TestBaseLayer(unittest.TestCase):
+
+    def setUp(self) -> None:
+        """Test graph for tests."""
+        self.adjacency = test_graph()
+        self.n = self.adjacency.shape[0]
+        self.features = self.adjacency
+        self.labels = np.array([0]*5 + [1]*5)
+        self.base_layer = BaseLayer('Conv', len(self.labels))
+
+    def test_base_layer_init(self):
+        with self.assertRaises(NotImplementedError):
+            self.base_layer.forward(self.adjacency, self.features)
+
+    def test_base_layer_initialize_weights(self):
+        self.base_layer._initialize_weights(10)
+        self.assertTrue(self.base_layer.weight.shape == (10, len(self.labels)))
+        self.assertTrue(self.base_layer.bias.shape == (1, len(self.labels)))
+        self.assertTrue(self.base_layer.weights_initialized)
+
+    def test_base_layer_repr(self):
+        self.assertTrue(self.base_layer.__repr__().startswith("  BaseLayer(layer_type: Conv, out_channels: 10"))
+        sage_layer = BaseLayer(layer_type='sageconv', out_channels=len(self.labels))
+        self.assertTrue('sample_size' in sage_layer.__repr__())
+        self.assertTrue('sageconv' in sage_layer.__repr__())