scikit-network 0.33.3__cp313-cp313-win_amd64.whl

sknetwork/gnn/base_activation.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on April 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+import numpy as np
+
+
+class BaseActivation:
+    """Base class for activation functions.
+
+    Parameters
+    ----------
+    name : str
+        Name of the activation function.
+    """
+    def __init__(self, name: str = 'custom'):
+        self.name = name
+
+    @staticmethod
+    def output(signal: np.ndarray) -> np.ndarray:
+        """Output of the activation function.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal.
+
+        Returns
+        -------
+        output : np.ndarray, shape (n_samples, n_channels)
+            Output signal.
+        """
+        output = signal
+        return output
+
+    @staticmethod
+    def gradient(signal: np.ndarray, direction: np.ndarray) -> np.ndarray:
+        """Gradient of the activation function.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal.
+        direction : np.ndarray, shape (n_samples, n_channels)
+            Direction where the gradient is taken.
+
+        Returns
+        -------
+        gradient : np.ndarray, shape (n_samples, n_channels)
+            Gradient.
+        """
+        gradient = direction
+        return gradient
+
+
+class BaseLoss(BaseActivation):
+    """Base class for loss functions."""
+    @staticmethod
+    def loss(signal: np.ndarray, labels: np.ndarray) -> float:
+        """Get the loss value.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal (before activation).
+        labels : np.ndarray, shape (n_samples)
+            True labels.
+        """
+        return 0
+
+    @staticmethod
+    def loss_gradient(signal: np.ndarray, labels: np.ndarray) -> np.ndarray:
+        """Gradient of the loss function.
+
+        Parameters
+        ----------
+        signal : np.ndarray, shape (n_samples, n_channels)
+            Input signal.
+        labels : np.ndarray, shape (n_samples,)
+            True labels.
+
+        Returns
+        -------
+        gradient : np.ndarray, shape (n_samples, n_channels)
+            Gradient.
+        """
+        gradient = np.ones_like(signal)
+        return gradient

sknetwork/gnn/base_layer.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created in July 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Optional, Union
+
+import numpy as np
+
+from sknetwork.gnn.activation import BaseActivation, get_activation
+from sknetwork.gnn.loss import BaseLoss, get_loss
+
+
+class BaseLayer:
+    """Base class for GNN layers.
+
+    Parameters
+    ----------
+    layer_type : str
+        Layer type. Can be either ``'Conv'`` (Convolution) or ``'Sage'`` (GraphSAGE).
+    out_channels: int
+        Dimension of the output.
+    activation: str (default = ``'Relu'``) or custom activation.
+        Activation function.
+        If a string, can be either ``'Identity'``, ``'Relu'``, ``'Sigmoid'`` or ``'Softmax'``.
+    use_bias: bool (default = `True`)
+        If ``True``, add a bias vector.
+    normalization: str (default = ``'both'``)
+        Normalization of the adjacency matrix for message passing.
+        Can be either `'left'`` (left normalization by the degrees), ``'right'`` (right normalization by the degrees),
+        ``'both'`` (symmetric normalization by the square root of degrees, default) or ``None`` (no normalization).
+    self_embeddings: bool (default = `True`)
+        If ``True``, consider self-embedding in addition to neighbors embedding for each node of the graph.
+    sample_size: int (default = 25)
+        Size of neighborhood sampled for each node. Used only for ``'SAGEConv'`` layer.
+
+    Attributes
+    ----------
+    weight : np.ndarray,
+        Trainable weight matrix.
+    bias : np.ndarray
+        Bias vector.
+    embedding : np.ndarray
+        Embedding of the nodes (before activation).
+    output : np.ndarray
+        Output of the layer (after activation).
+    """
+    def __init__(self, layer_type: str, out_channels: int, activation: Optional[Union[BaseActivation, str]] = 'Relu',
+                 use_bias: bool = True, normalization: str = 'both', self_embeddings: bool = True,
+                 sample_size: int = 25, loss: Optional[Union[BaseLoss, str]] = None):
+        self.layer_type = layer_type
+        self.out_channels = out_channels
+        if loss is None:
+            self.activation = get_activation(activation)
+        else:
+            self.activation = get_loss(loss)
+        self.use_bias = use_bias
+        self.normalization = normalization.lower()
+        self.self_embeddings = self_embeddings
+        self.sample_size = sample_size
+        self.weight = None
+        self.bias = None
+        self.embedding = None
+        self.output = None
+        self.weights_initialized = False
+
+    def _initialize_weights(self, in_channels: int):
+        """Initialize weights and bias.
+
+        Parameters
+        ----------
+        in_channels: int
+            Number of input channels.
+        """
+        # He initialization
+        self.weight = np.random.randn(in_channels, self.out_channels) * np.sqrt(2 / self.out_channels)
+        if self.use_bias:
+            self.bias = np.zeros((1, self.out_channels))
+        self.weights_initialized = True
+
+    def forward(self, *args, **kwargs):
+        """Compute forward pass."""
+        raise NotImplementedError
+
+    def __call__(self, *args, **kwargs):
+        return self.forward(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        """ String representation of object
+
+        Returns
+        -------
+        str
+            String representation of object
+        """
+        print_attr = ['out_channels', 'layer_type', 'activation', 'use_bias', 'normalization', 'self_embeddings']
+        if 'sage' in self.layer_type:
+            print_attr.append('sample_size')
+        attributes_dict = {k: v for k, v in self.__dict__.items() if k in print_attr}
+        string = ''
+
+        for k, v in attributes_dict.items():
+            if k == 'activation':
+                string += f'{k}: {v.name}, '
+            else:
+                string += f'{k}: {v}, '
+
+        return f'  {self.__class__.__name__}({string[:-2]})'

sknetwork/gnn/gnn_classifier.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created in April 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Iterable, Optional, Union
+from collections import defaultdict
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.classification.metrics import get_accuracy_score
+from sknetwork.gnn.base import BaseGNN
+from sknetwork.gnn.loss import BaseLoss
+from sknetwork.gnn.layer import get_layer
+from sknetwork.gnn.neighbor_sampler import UniformNeighborSampler
+from sknetwork.gnn.optimizer import BaseOptimizer
+from sknetwork.gnn.utils import check_output, check_early_stopping, check_loss, get_layers
+from sknetwork.utils.check import check_format, check_nonnegative, check_square
+from sknetwork.utils.values import get_values
+
+
+class GNNClassifier(BaseGNN):
+    """Graph Neural Network for node classification.
+
+    Parameters
+    ----------
+    dims : iterable or int
+        Dimension of the output of each layer (in forward direction).
+        If an integer, dimension of the output layer (no hidden layer).
+        Optional if ``layers`` is specified.
+    layer_types : iterable or str
+        Layer types (in forward direction).
+        If a string, the same type is used at each layer.
+        Can be ``'Conv'``, graph convolutional layer (default) or ``'Sage'`` (GraphSage).
+    activations : iterable or str
+        Activation functions (in forward direction).
+        If a string, the same activation function is used at each layer.
+        Can be either ``'Identity'``, ``'Relu'``, ``'Sigmoid'`` or ``'Softmax'`` (default = ``'Relu'``).
+    use_bias : iterable or bool
+        Whether to add a bias term at each layer (in forward direction).
+        If ``True``, use a bias term at each layer.
+    normalizations : iterable or str
+        Normalizations of the adjacency matrix for message passing (in forward direction).
+        If a string, the same type of normalization is used at each layer.
+        Can be either ``'left'`` (left normalization by the degrees), ``'right'`` (right normalization by the degrees),
+        ``'both'`` (symmetric normalization by the square root of degrees, default) or ``None`` (no normalization).
+    self_embeddings : iterable or str
+        Whether to add the embedding to each node for message passing (in forward direction).
+        If ``True``, add a self-embedding at each layer.
+    sample_sizes : iterable or int
+        Sizes of neighborhood sampled for each node (in forward direction).
+        If an integer, the same sampling size is used at each layer.
+        Used only for ``'Sage'`` layer type.
+    loss : str (default = ``'CrossEntropy'``) or BaseLoss
+        Name of loss function or custom loss function.
+    layers : iterable or None
+        Custom layers (in forward directions). If used, previous parameters are ignored.
+    optimizer : str or optimizer
+        * ``'Adam'``, stochastic gradient-based optimizer (default).
+        * ``'GD'``, gradient descent.
+    learning_rate : float
+        Learning rate.
+    early_stopping : bool (default = ``True``)
+        Whether to use early stopping to end training.
+        If ``True``, training terminates when validation score is not improving for `patience` number of epochs.
+    patience : int (default = 10)
+        Number of iterations with no improvement to wait before stopping fitting.
+    verbose : bool
+        Verbose mode.
+
+    Attributes
+    ----------
+    conv2, ..., conv1: :class:'GCNConv'
+        Graph convolutional layers.
+    output_ : np.ndarray
+        Output of the GNN.
+    labels_: np.ndarray
+        Predicted node labels.
+    history_: dict
+        Training history per epoch: {``'embedding'``, ``'loss'``, ``'train_accuracy'``, ``'val_accuracy'``}.
+
+    Example
+    -------
+    >>> from sknetwork.gnn.gnn_classifier import GNNClassifier
+    >>> from sknetwork.data import karate_club
+    >>> from numpy.random import randint
+    >>> graph = karate_club(metadata=True)
+    >>> adjacency = graph.adjacency
+    >>> labels_true = graph.labels
+    >>> labels = {i: labels_true[i] for i in [0, 1, 33]}
+    >>> features = adjacency.copy()
+    >>> gnn = GNNClassifier(dims=1, early_stopping=False)
+    >>> labels_pred = gnn.fit_predict(adjacency, features, labels, random_state=42)
+    >>> float(round(np.mean(labels_pred == labels_true), 2))
+    0.88
+    """
+
+    def __init__(self, dims: Optional[Union[int, Iterable]] = None, layer_types: Union[str, Iterable] = 'Conv',
+                 activations: Union[str, Iterable] = 'ReLu', use_bias: Union[bool, list] = True,
+                 normalizations: Union[str, Iterable] = 'both', self_embeddings: Union[bool, Iterable] = True,
+                 sample_sizes: Union[int, list] = 25, loss: Union[BaseLoss, str] = 'CrossEntropy',
+                 layers: Optional[Iterable] = None, optimizer: Union[BaseOptimizer, str] = 'Adam',
+                 learning_rate: float = 0.01, early_stopping: bool = True, patience: int = 10, verbose: bool = False):
+        super(GNNClassifier, self).__init__(loss, optimizer, learning_rate, verbose)
+        if layers is not None:
+            layers = [get_layer(layer) for layer in layers]
+        else:
+            layers = get_layers(dims, layer_types, activations, use_bias, normalizations, self_embeddings, sample_sizes,
+                                loss)
+        self.loss = check_loss(layers[-1])
+        self.layers = layers
+        self.early_stopping = early_stopping
+        self.patience = patience
+        self.history_ = defaultdict(list)
+
+    def forward(self, adjacency: Union[list, sparse.csr_matrix], features: Union[sparse.csr_matrix, np.ndarray]) \
+            -> np.ndarray:
+        """Perform a forward pass on the graph and return the output.
+
+        Parameters
+        ----------
+        adjacency : Union[list, sparse.csr_matrix]
+            Adjacency matrix or list of sampled adjacency matrices.
+        features : sparse.csr_matrix, np.ndarray
+            Features, array of shape (n_nodes, n_features).
+
+        Returns
+        -------
+        output : np.ndarray
+            Output of the GNN.
+        """
+        h = features.copy()
+        for i, layer in enumerate(self.layers):
+            if isinstance(adjacency, list):
+                h = layer(adjacency[i], h)
+            else:
+                h = layer(adjacency, h)
+        return h
+
+    @staticmethod
+    def _compute_predictions(output: np.ndarray) -> np.ndarray:
+        """Compute predictions from the output of the GNN.
+
+        Parameters
+        ----------
+        output : np.ndarray
+            Output of the GNN.
+
+        Returns
+        -------
+        labels : np.ndarray
+            Predicted labels.
+        """
+        if output.shape[1] == 1:
+            labels = (output.ravel() > 0.5).astype(int)
+        else:
+            labels = output.argmax(axis=1)
+        return labels
+
+    def fit(self, adjacency: Union[sparse.csr_matrix, np.ndarray], features: Union[sparse.csr_matrix, np.ndarray],
+            labels: np.ndarray, n_epochs: int = 100, validation: float = 0, reinit: bool = False,
+            random_state: Optional[int] = None) -> 'GNNClassifier':
+        """ Fit model to data and store trained parameters.
+
+        Parameters
+        ----------
+        adjacency : sparse.csr_matrix
+            Adjacency matrix of the graph.
+        features : sparse.csr_matrix, np.ndarray
+            Input feature of shape :math:`(n, d)` with :math:`n` the number of nodes in the graph and :math:`d`
+            the size of feature space.
+        labels : dict, np.ndarray
+            Known labels. Negative values ignored.
+        n_epochs : int (default = 100)
+            Number of epochs (iterations over the whole graph).
+        validation : float
+            Proportion of the training set used for validation (between 0 and 1).
+        reinit: bool  (default = ``False``)
+            If ``True``, reinit the trainable parameters of the GNN (weights and biases).
+        random_state : int
+            Random seed, used for reproducible results across multiple runs.
+        """
+        if reinit:
+            for layer in self.layers:
+                layer.weights_initialized = False
+            self.history_ = defaultdict(list)
+
+        if random_state is not None:
+            np.random.seed(random_state)
+
+        check_format(adjacency, allow_empty=True)
+        check_format(features, allow_empty=True)
+
+        labels = get_values(adjacency.shape, labels)
+        labels = labels.astype(int)
+        if (labels < 0).all():
+            raise ValueError('At least one node must have a non-negative label.')
+        check_output(self.layers[-1].out_channels, labels)
+
+        self.train_mask = labels >= 0
+        if self.val_mask is None and 0 < validation < 1:
+            mask = np.random.random(size=len(labels)) < validation
+            self.val_mask = self.train_mask & mask
+            self.train_mask &= ~mask
+
+        early_stopping = check_early_stopping(self.early_stopping, self.val_mask, self.patience)
+
+        # List of sampled adjacencies (one per layer)
+        adjacencies = self._sample_nodes(adjacency)
+
+        best_val_accuracy = 0
+        count = 0
+
+        for epoch in range(n_epochs):
+
+            # Forward
+            output = self.forward(adjacencies, features)
+
+            # Compute predictions
+            labels_pred = self._compute_predictions(output)
+
+            # Loss
+            loss_value = self.loss.loss(output[self.train_mask], labels[self.train_mask])
+
+            # Accuracy
+            train_accuracy = get_accuracy_score(labels[self.train_mask], labels_pred[self.train_mask])
+            if self.val_mask is not None and any(self.val_mask):
+                val_accuracy = get_accuracy_score(labels[self.val_mask], labels_pred[self.val_mask])
+            else:
+                val_accuracy = None
+
+            # Backpropagation
+            self.backward(features, labels, self.train_mask)
+
+            # Update weights using optimizer
+            self.optimizer.step(self)
+
+            # Save results
+            self.history_['loss'].append(loss_value)
+            self.history_['train_accuracy'].append(train_accuracy)
+            if val_accuracy is not None:
+                self.history_['val_accuracy'].append(val_accuracy)
+
+            if n_epochs > 10 and epoch % int(n_epochs / 10) == 0:
+                if val_accuracy is not None:
+                    self.print_log(
+                        f'In epoch {epoch:>3}, loss: {loss_value:.3f}, train accuracy: {train_accuracy:.3f}, '
+                        f'val accuracy: {val_accuracy:.3f}')
+                else:
+                    self.print_log(
+                        f'In epoch {epoch:>3}, loss: {loss_value:.3f}, train accuracy: {train_accuracy:.3f}')
+            elif n_epochs <= 10:
+                if val_accuracy is not None:
+                    self.print_log(
+                        f'In epoch {epoch:>3}, loss: {loss_value:.3f}, train accuracy: {train_accuracy:.3f}, '
+                        f'val accuracy: {val_accuracy:.3f}')
+                else:
+                    self.print_log(
+                        f'In epoch {epoch:>3}, loss: {loss_value:.3f}, train accuracy: {train_accuracy:.3f}')
+
+            # Early stopping
+            if early_stopping:
+                if val_accuracy > best_val_accuracy:
+                    count = 0
+                    best_val_accuracy = val_accuracy
+                else:
+                    count += 1
+                    if count >= self.patience:
+                        self.print_log('Early stopping.')
+                        break
+
+        output = self.forward(adjacencies, features)
+        labels_pred = self._compute_predictions(output)
+
+        self.embedding_ = self.layers[-1].embedding
+        self.output_ = self.layers[-1].output
+        self.labels_ = labels_pred
+
+        return self
+
+    def _sample_nodes(self, adjacency: Union[sparse.csr_matrix, np.ndarray]) -> list:
+        """Perform node sampling on adjacency matrix for GraphSAGE layers. For other layers, the
+        adjacency matrix remains unchanged.
+
+        Parameters
+        ----------
+        adjacency : sparse.csr_matrix
+            Adjacency matrix of the graph.
+
+        Returns
+        -------
+        List of (sampled) adjacency matrices.
+        """
+        adjacencies = []
+
+        for layer in self.layers:
+            if layer.layer_type == 'sage':
+                sampler = UniformNeighborSampler(sample_size=layer.sample_size)
+                adjacencies.append(sampler(adjacency))
+            else:
+                adjacencies.append(adjacency)
+
+        return adjacencies

sknetwork/gnn/layer.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# coding: utf-8
+"""
+Created in April 2022
+@author: Simon Delarue <sdelarue@enst.fr>
+"""
+from typing import Optional, Union
+
+import numpy as np
+from scipy import sparse
+
+from sknetwork.gnn.activation import BaseActivation
+from sknetwork.gnn.loss import BaseLoss
+from sknetwork.gnn.base_layer import BaseLayer
+from sknetwork.utils.check import add_self_loops
+from sknetwork.linalg import diagonal_pseudo_inverse
+
+
+class Convolution(BaseLayer):
+    """Graph convolutional layer.
+
+    Apply the following function to the embedding :math:`X`:
+
+    :math:`\\sigma(\\bar AXW + b)`,
+
+    where :math:`\\bar A` is the normalized adjacency matrix (possibly with inserted self-embeddings),
+    :math:`W`, :math:`b` are trainable parameters and :math:`\\sigma` is the activation function.
+
+    Parameters
+    ----------
+    layer_type : str
+        Layer type. Can be either ``'Conv'``, convolutional operator as in [1] or ``'Sage'``, as in [2].
+    out_channels: int
+        Dimension of the output.
+    activation: str (default = ``'Relu'``) or custom activation.
+        Activation function.
+        If a string, can be either ``'Identity'``, ``'Relu'``, ``'Sigmoid'`` or ``'Softmax'``.
+    use_bias: bool (default = `True`)
+        If ``True``, add a bias vector.
+    normalization: str (default = ``'both'``)
+        Normalization of the adjacency matrix for message passing.
+        Can be either `'left'`` (left normalization by the degrees), ``'right'`` (right normalization by the degrees),
+        ``'both'`` (symmetric normalization by the square root of degrees, default) or ``None`` (no normalization).
+    self_embeddings: bool (default = `True`)
+        If ``True``, consider self-embedding in addition to neighbors embedding for each node of the graph.
+    sample_size: int (default = 25)
+        Size of neighborhood sampled for each node. Used only for ``'Sage'`` layer.
+
+    Attributes
+    ----------
+    weight: np.ndarray,
+        Trainable weight matrix.
+    bias: np.ndarray
+        Bias vector.
+    embedding: np.ndarray
+        Embedding of the nodes (before activation).
+    output: np.ndarray
+        Output of the layer (after activation).
+
+    References
+    ----------
+    [1] Kipf, T., & Welling, M. (2017).
+    `Semi-supervised Classification with Graph Convolutional Networks.
+    <https://arxiv.org/pdf/1609.02907.pdf>`_
+    5th International Conference on Learning Representations.
+
+    [2] Hamilton, W. Ying, R., & Leskovec, J. (2017)
+    `Inductive Representation Learning on Large Graphs.
+    <https://arxiv.org/pdf/1706.02216.pdf>`_
+    NIPS
+    """
+    def __init__(self, layer_type: str, out_channels: int, activation: Optional[Union[BaseActivation, str]] = 'Relu',
+                 use_bias: bool = True, normalization: str = 'both', self_embeddings: bool = True,
+                 sample_size: int = None, loss: Optional[Union[BaseLoss, str]] = None):
+        super(Convolution, self).__init__(layer_type, out_channels, activation, use_bias, normalization,
+                                          self_embeddings, sample_size, loss)
+
+    def forward(self, adjacency: Union[sparse.csr_matrix, np.ndarray],
+                features: Union[sparse.csr_matrix, np.ndarray]) -> np.ndarray:
+        """Compute graph convolution.
+
+        Parameters
+        ----------
+        adjacency
+            Adjacency matrix of the graph.
+        features : sparse.csr_matrix, np.ndarray
+            Input feature of shape :math:`(n, d)` with :math:`n` the number of nodes in the graph and :math:`d`
+            the size of feature space.
+
+        Returns
+        -------
+        output: np.ndarray
+            Output of the layer.
+        """
+        if not self.weights_initialized:
+            self._initialize_weights(features.shape[1])
+
+        n_row, n_col = adjacency.shape
+
+        weights = adjacency.dot(np.ones(n_col))
+        if self.normalization == 'left':
+            d_inv = diagonal_pseudo_inverse(weights)
+            adjacency = d_inv.dot(adjacency)
+        elif self.normalization == 'right':
+            d_inv = diagonal_pseudo_inverse(weights)
+            adjacency = adjacency.dot(d_inv)
+        elif self.normalization == 'both':
+            d_inv = diagonal_pseudo_inverse(np.sqrt(weights))
+            adjacency = d_inv.dot(adjacency).dot(d_inv)
+
+        if self.self_embeddings:
+            adjacency = add_self_loops(adjacency)
+
+        message = adjacency.dot(features)
+        embedding = message.dot(self.weight)
+
+        if self.use_bias:
+            embedding += self.bias
+
+        output = self.activation.output(embedding)
+
+        self.embedding = embedding
+        self.output = output
+
+        return output
+
+
+def get_layer(layer: Union[BaseLayer, str] = 'conv', **kwargs) -> BaseLayer:
+    """Get layer.
+
+    Parameters
+    ----------
+    layer : str or custom layer
+        If a string, must be either ``'Conv'`` (Convolution) or ``'Sage'`` (GraphSAGE).
+
+    Returns
+    -------
+    Layer object.
+    """
+    if issubclass(type(layer), BaseLayer):
+        return layer
+    elif type(layer) == str:
+        layer = layer.lower()
+        if 'sage' in layer:
+            kwargs['normalization'] = 'left'
+            kwargs['self_embeddings'] = True
+            return Convolution('sage', **kwargs)
+        elif 'conv' in layer:
+            return Convolution('conv', **kwargs)
+        else:
+            raise ValueError("Layer name must be \"Conv\" or \"Sage\".")
+    else:
+        raise TypeError("Layer must be a string or a \"BaseLayer\" object.")