torch-molecule 0.1.0.dev2__py3-none-any.whl

torch_molecule/predictor/grin/model.py

@@ -0,0 +1,116 @@
+import torch
+import torch.nn as nn
+from torch_geometric.nn import global_add_pool, global_mean_pool, global_max_pool
+
+from ...nn import GNN_node, GNN_node_Virtualnode, MLP
+from ...utils import init_weights
+
+class GRIN(nn.Module):
+    def __init__(
+        self,
+        num_task,
+        num_layer,
+        hidden_size=300,
+        gnn_type="gin-virtual",
+        drop_ratio=0.5,
+        norm_layer="batch_norm",
+        graph_pooling="max",
+        augmented_feature=['maccs', 'morgan'],
+        algorithm_aligned = 'mst'
+    ):
+        super(GRIN, self).__init__()
+        gnn_name = gnn_type.split("-")[0]
+        self.algorithm_aligned = algorithm_aligned
+        self.num_task = num_task
+        self.hidden_size = hidden_size
+
+        if "virtual" in gnn_type:
+            self.graph_encoder = GNN_node_Virtualnode(
+                num_layer,
+                hidden_size,
+                JK="last",
+                drop_ratio=drop_ratio,
+                residual=True,
+                gnn_name=gnn_name,
+                norm_layer=norm_layer,
+                algorithm_aligned=algorithm_aligned
+            )
+        else:
+            self.graph_encoder = GNN_node(
+                num_layer,
+                hidden_size,
+                JK="last",
+                drop_ratio=drop_ratio,
+                residual=True,
+                gnn_name=gnn_name,
+                norm_layer=norm_layer,
+                algorithm_aligned=algorithm_aligned
+            )
+        if graph_pooling == "sum":
+            self.pool = global_add_pool
+        elif graph_pooling == "mean":
+            self.pool = global_mean_pool
+        elif graph_pooling == "max":
+            self.pool = global_max_pool
+        else:
+            raise ValueError(f"Invalid graph pooling type {graph_pooling}.")
+
+        graph_dim = hidden_size
+        self.augmented_feature = augmented_feature
+        if augmented_feature:
+            if "morgan" in augmented_feature:
+                graph_dim += 1024
+            if "maccs" in augmented_feature:
+                graph_dim += 167
+        self.predictor = MLP(graph_dim, hidden_features=2 * hidden_size, out_features=num_task)
+    
+    def initialize_parameters(self, seed=None):
+        """
+        Randomly initialize all model parameters using the init_weights function.
+        
+        Args:
+            seed (int, optional): Random seed for reproducibility. Defaults to None.
+        """
+        if seed is not None:
+            torch.manual_seed(seed)
+        
+        # Initialize the main components
+        init_weights(self.graph_encoder)
+        init_weights(self.predictor)
+        
+        # Reset all parameters using PyTorch Geometric's reset function
+        def reset_parameters(module):
+            if hasattr(module, 'reset_parameters'):
+                module.reset_parameters()
+            elif hasattr(module, 'weight') and hasattr(module.weight, 'data'):
+                init_weights(module)
+        
+        self.apply(reset_parameters)
+
+    def _augmented_graph_features(self, batched_data, h_rep):
+        if self.augmented_feature:
+            if 'morgan' in self.augmented_feature:
+                morgan = batched_data.morgan.type_as(h_rep)
+                h_rep = torch.cat((h_rep, morgan), dim=1)
+            if 'maccs' in self.augmented_feature:
+                maccs = batched_data.maccs.type_as(h_rep)
+                h_rep = torch.cat((h_rep, maccs), dim=1)
+        return h_rep
+
+    def compute_loss(self, batched_data, criterion, l1_penalty=1e-3):
+        h_node, _ = self.graph_encoder(batched_data)
+        h_rep = self.pool(h_node, batched_data.batch)
+        h_rep = self._augmented_graph_features(batched_data, h_rep)
+        prediction = self.predictor(h_rep)
+        target = batched_data.y.to(torch.float32)
+        is_labeled = batched_data.y == batched_data.y
+        loss = criterion(prediction.to(torch.float32)[is_labeled], target[is_labeled]).mean()
+        l1_loss = sum(p.abs().sum() for p in self.graph_encoder.parameters()) + sum(p.abs().sum() for p in self.predictor.parameters())
+        return loss + l1_penalty * l1_loss
+
+    def forward(self, batched_data):
+        h_node, _ = self.graph_encoder(batched_data)
+        h_rep = self.pool(h_node, batched_data.batch)
+        h_rep = self._augmented_graph_features(batched_data, h_rep)
+        prediction = self.predictor(h_rep)
+        return {"prediction": prediction}

torch_molecule/predictor/grin/modeling_grin.py

@@ -0,0 +1,91 @@
+import os
+import numpy as np
+import warnings
+import datetime
+from tqdm import tqdm
+from typing import Optional, Union, Dict, Any, List, Type
+from dataclasses import dataclass, field
+
+import torch
+from torch_geometric.data import Data
+
+from .model import GRIN
+from ...utils import graph_from_smiles
+from ..gnn.modeling_gnn import GNNMolecularPredictor
+from ...utils.search import (
+    ParameterSpec,
+    ParameterType,
+)
+
+@dataclass
+class GRINMolecularPredictor(GNNMolecularPredictor):
+    """
+    This predictor implements GRIN for Max Spanning Tree algorithm aligned GNN.
+
+    The full name of GRIN is Graph Invariant Representation Learning.
+
+    References
+    ----------
+    - Learning Repetition-Invariant Representations for Polymer Informatics.
+      https://arxiv.org/pdf/2505.10726
+
+    :param l1_penalty: Weight for the L1 penalty
+    :type l1_penalty: float, default=1e-3
+    :param epochs_to_penalize: Number of epochs to train before starting L1 penalty
+    :type epochs_to_penalize: int, default=100
+    """
+
+    l1_penalty: float = 1e-3
+    epochs_to_penalize: int = 100
+
+    # Other Non-init fields
+    model_name: str = "GRINMolecularPredictor"
+    model_class: Type[GRIN] = field(default=GRIN, init=False)
+    
+    def __post_init__(self):
+        super().__post_init__()
+    
+    @staticmethod
+    def _get_param_names() -> List[str]:
+        return GNNMolecularPredictor._get_param_names() + [
+            "l1_penalty",
+            "epochs_to_penalize"
+        ]
+    def _get_default_search_space(self):
+        search_space = super()._get_default_search_space().copy()
+        search_space["l1_penalty"] = ParameterSpec(ParameterType.LOG_FLOAT, (1e-6, 1))
+        search_space["epochs_to_penalize"] = ParameterSpec(ParameterType.INTEGER, (0, 100))
+        return search_space
+
+    def _get_model_params(self, checkpoint: Optional[Dict] = None) -> Dict[str, Any]:
+        base_params = super()._get_model_params(checkpoint)
+        return base_params
+
+    def _train_epoch(self, train_loader, optimizer, epoch):
+        self.model.train()
+        losses = []
+
+        iterator = (
+            tqdm(train_loader, desc="Training", leave=False)
+            if self.verbose
+            else train_loader
+        )
+
+        for step, batch in enumerate(iterator):
+            batch = batch.to(self.device)
+            optimizer.zero_grad()
+            if epoch >= self.epochs_to_penalize:
+                l1_penalty = min(epoch - self.epochs_to_penalize, 1) * self.l1_penalty
+            else:
+                l1_penalty = 0
+            loss = self.model.compute_loss(batch, self.loss_criterion, l1_penalty)
+            loss.backward()
+            if self.grad_clip_value is not None:
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip_value)
+            optimizer.step()
+
+            if self.verbose:
+                iterator.set_postfix({"Epoch": epoch, "Total Loss": f"{loss.item():.4f}"})
+            losses.append(loss.item())
+
+        return losses

torch_molecule/predictor/irm/__init__.py

@@ -0,0 +1,3 @@
+from .modeling_irm import IRMMolecularPredictor
+
+__all__ = ['IRMMolecularPredictor']

torch_molecule/predictor/irm/model.py

@@ -0,0 +1,127 @@
+import torch
+import torch.nn as nn
+from torch_geometric.nn import global_add_pool, global_mean_pool, global_max_pool
+
+from ...nn import GNN_node, GNN_node_Virtualnode, MLP
+from ...utils import init_weights
+
+class GNN(nn.Module):
+    def __init__(
+        self,
+        num_task,
+        num_layer,
+        hidden_size=300,
+        gnn_type="gin-virtual",
+        drop_ratio=0.5,
+        norm_layer="batch_norm",
+        graph_pooling="max",
+        augmented_feature=['maccs', 'morgan']
+    ):
+        super(GNN, self).__init__()
+        gnn_name = gnn_type.split("-")[0]
+        self.num_task = num_task
+        self.hidden_size = hidden_size
+
+        if "virtual" in gnn_type:
+            self.graph_encoder = GNN_node_Virtualnode(
+                num_layer,
+                hidden_size,
+                JK="last",
+                drop_ratio=drop_ratio,
+                residual=True,
+                gnn_name=gnn_name,
+                norm_layer=norm_layer,
+            )
+        else:
+            self.graph_encoder = GNN_node(
+                num_layer,
+                hidden_size,
+                JK="last",
+                drop_ratio=drop_ratio,
+                residual=True,
+                gnn_name=gnn_name,
+                norm_layer=norm_layer,
+            )
+        if graph_pooling == "sum":
+            self.pool = global_add_pool
+        elif graph_pooling == "mean":
+            self.pool = global_mean_pool
+        elif graph_pooling == "max":
+            self.pool = global_max_pool
+        else:
+            raise ValueError(f"Invalid graph pooling type {graph_pooling}.")
+
+        graph_dim = hidden_size
+        self.augmented_feature = augmented_feature
+        if augmented_feature:
+            if "morgan" in augmented_feature:
+                graph_dim += 1024
+            if "maccs" in augmented_feature:
+                graph_dim += 167
+        self.predictor = MLP(graph_dim, hidden_features=2 * hidden_size, out_features=num_task)
+    
+    def initialize_parameters(self, seed=None):
+        """
+        Randomly initialize all model parameters using the init_weights function.
+        
+        Args:
+            seed (int, optional): Random seed for reproducibility. Defaults to None.
+        """
+        if seed is not None:
+            torch.manual_seed(seed)
+        
+        # Initialize the main components
+        init_weights(self.graph_encoder)
+        init_weights(self.predictor)
+        
+        # Reset all parameters using PyTorch Geometric's reset function
+        def reset_parameters(module):
+            if hasattr(module, 'reset_parameters'):
+                module.reset_parameters()
+            elif hasattr(module, 'weight') and hasattr(module.weight, 'data'):
+                init_weights(module)
+        
+        self.apply(reset_parameters)
+
+    def _augmented_graph_features(self, batched_data, h_rep):
+        if self.augmented_feature:
+            if 'morgan' in self.augmented_feature:
+                morgan = batched_data.morgan.type_as(h_rep)
+                h_rep = torch.cat((h_rep, morgan), dim=1)
+            if 'maccs' in self.augmented_feature:
+                maccs = batched_data.maccs.type_as(h_rep)
+                h_rep = torch.cat((h_rep, maccs), dim=1)
+        return h_rep
+
+    def compute_loss(self, batched_data, criterion, scale=1.0, penalty_weight=1.0):
+        h_node, _ = self.graph_encoder(batched_data)
+        h_rep = self.pool(h_node, batched_data.batch)
+        h_rep = self._augmented_graph_features(batched_data, h_rep)
+        prediction = self.predictor(h_rep)
+        target = batched_data.y.to(torch.float32)
+        is_labeled = batched_data.y == batched_data.y
+
+        dummy = torch.nn.Parameter(torch.Tensor([scale])).to(prediction.device)
+        losses_erm = criterion(prediction.to(torch.float32)[is_labeled] * dummy, target[is_labeled])
+        
+        environments = batched_data.environment[is_labeled]
+        unique_envs = environments.unique()  
+        env_losses = []
+        for env in unique_envs:
+            env_mask = environments == env
+            env_loss = losses_erm[env_mask].mean()
+            env_grad = torch.autograd.grad(env_loss, dummy, create_graph=True)[0]
+            env_losses.append(torch.sum(env_grad**2))
+        
+        penalty = torch.sum(torch.stack(env_losses))
+        total_loss = losses_erm.mean() + penalty_weight * penalty
+        if penalty_weight > 1.0:
+            total_loss /= penalty_weight
+        return total_loss, losses_erm.mean(), penalty
+
+    def forward(self, batched_data):
+        h_node, _ = self.graph_encoder(batched_data)
+        h_rep = self.pool(h_node, batched_data.batch)
+        h_rep = self._augmented_graph_features(batched_data, h_rep)
+        prediction = self.predictor(h_rep)
+        return {"prediction": prediction}

torch_molecule/predictor/irm/modeling_irm.py

@@ -0,0 +1,179 @@
+import os
+import numpy as np
+import warnings
+import datetime
+from tqdm import tqdm
+from typing import Optional, Union, Dict, Any, List, Type
+from dataclasses import dataclass, field
+
+import torch
+from torch_geometric.data import Data
+
+from .model import GNN
+from ...utils import graph_from_smiles
+from ..gnn.modeling_gnn import GNNMolecularPredictor
+from ...utils.search import (
+    ParameterSpec,
+    ParameterType,
+)
+
+@dataclass
+class IRMMolecularPredictor(GNNMolecularPredictor):
+    """This predictor implements a Invariant Risk Minimization model with the GNN.
+    
+    The full name of IRM is Invariant Risk Minimization.
+
+    References
+    ----------
+    - Invariant Risk Minimization.
+      https://arxiv.org/abs/1907.02893
+
+    - Reference Code: https://github.com/facebookresearch/InvariantRiskMinimization
+    
+    Parameters
+    ----------
+    IRM_environment : Union[torch.Tensor, np.ndarray, List, str], default="random"
+        Environment assignments for IRM. Can be a list of integers (one per sample),
+        or "random" to assign environments randomly.
+    scale : float, default=1.0
+        Scaling factor for the IRM penalty term.
+    penalty_weight : float, default=1.0
+        Weight of the IRM penalty in the loss function.
+    penalty_anneal_iters : int, default=100
+        Number of iterations for annealing the penalty weight.
+    """
+    
+    IRM_environment: Union[torch.Tensor, np.ndarray, List, str] = "random"
+    scale: float = 1.0
+    penalty_weight: float = 1.0
+    penalty_anneal_iters: int = 100
+
+    # Other Non-init fields
+    model_name: str = "IRMMolecularPredictor"
+    model_class: Type[GNN] = field(default=GNN, init=False)
+    
+    def __post_init__(self):
+        super().__post_init__()
+
+    @staticmethod
+    def _get_param_names() -> List[str]:
+        return GNNMolecularPredictor._get_param_names() + [
+            "IRM_environment",
+            "scale",
+            "penalty_weight",
+            "penalty_anneal_iters",
+        ]
+
+    def _get_default_search_space(self):
+        search_space = super()._get_default_search_space().copy()
+        search_space["penalty_weight"] = ParameterSpec(ParameterType.LOG_FLOAT, (1e-10, 1))
+        search_space["penalty_anneal_iters"] = ParameterSpec(ParameterType.INTEGER, (10, 100))
+        return search_space
+
+    def _get_model_params(self, checkpoint: Optional[Dict] = None) -> Dict[str, Any]:
+        base_params = super()._get_model_params(checkpoint)
+        return base_params
+    
+    def set_IRM_environment(self, environment: Union[torch.Tensor, np.ndarray, List, str]):
+        if isinstance(environment, str):
+            if environment != "random":
+                raise ValueError("IRM_environment must be 'random' if specified with a string")
+            self.IRM_environment = environment
+        else:
+            if isinstance(environment, np.ndarray) or isinstance(environment, torch.Tensor):
+                self.IRM_environment = environment.reshape(-1).tolist()
+            else:
+                self.IRM_environment = environment
+            
+            if not all(isinstance(item, int) for item in self.IRM_environment):
+                raise ValueError("IRM_environment must be a list of integers")
+
+    def _convert_to_pytorch_data(self, X, y=None):
+        """Convert numpy arrays to PyTorch Geometric data format.
+        """
+        if self.verbose:
+            iterator = tqdm(enumerate(X), desc="Converting molecules to graphs", total=len(X))
+        else:
+            iterator = enumerate(X)
+
+        pyg_graph_list = []
+        for idx, smiles_or_mol in iterator:
+            if y is not None:
+                properties = y[idx]
+            else:
+                properties = None
+            graph = graph_from_smiles(smiles_or_mol, properties, self.augmented_feature)
+            g = Data()
+            g.num_nodes = graph["num_nodes"]
+            g.edge_index = torch.from_numpy(graph["edge_index"])
+
+            del graph["num_nodes"]
+            del graph["edge_index"]
+
+            if graph["edge_feat"] is not None:
+                g.edge_attr = torch.from_numpy(graph["edge_feat"])
+                del graph["edge_feat"]
+
+            if graph["node_feat"] is not None:
+                g.x = torch.from_numpy(graph["node_feat"])
+                del graph["node_feat"]
+
+            if graph["y"] is not None:
+                g.y = torch.from_numpy(graph["y"])
+                del graph["y"]
+   
+            if graph["morgan"] is not None:
+                g.morgan = torch.tensor(graph["morgan"], dtype=torch.int8).view(1, -1)
+                del graph["morgan"]
+            
+            if graph["maccs"] is not None:
+                g.maccs = torch.tensor(graph["maccs"], dtype=torch.int8).view(1, -1)
+                del graph["maccs"]
+    
+            if self.IRM_environment == "random":
+                g.environment = torch.randint(0, 2, (1,)).view(1, 1)
+            elif len(X) != len(self.IRM_environment):
+                raise ValueError("IRM_environment must has the same length as the input, which is {}".format(len(X)))
+            else:
+                if isinstance(self.IRM_environment[idx], int):
+                    g.environment = torch.tensor(self.IRM_environment[idx], dtype=torch.int64).view(1, 1)
+                else:
+                    raise ValueError("IRM_environment must be a list of integers")
+            pyg_graph_list.append(g)
+
+        return pyg_graph_list
+
+    def _train_epoch(self, train_loader, optimizer, epoch):
+        self.model.train()
+        losses = []
+        losses_erm = []
+        penalties = []
+
+        iterator = (
+            tqdm(train_loader, desc="Training", leave=False)
+            if self.verbose
+            else train_loader
+        )
+
+        for step, batch in enumerate(iterator):
+            batch = batch.to(self.device)
+            optimizer.zero_grad()
+
+            if epoch >= self.penalty_anneal_iters:
+                penalty_weight = self.penalty_weight
+            else:
+                penalty_weight = 1.0
+            loss, loss_erm, penalty = self.model.compute_loss(batch, self.loss_criterion, self.scale, penalty_weight)
+            loss.backward()
+            if self.grad_clip_value is not None:
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip_value)
+
+            optimizer.step()
+            losses.append(loss.item())
+            losses_erm.append(loss_erm.item())
+            penalties.append(penalty.item())
+
+            if self.verbose:
+                iterator.set_postfix({"Epoch": epoch, "Total Loss": f"{loss.item():.4f}", "ERM Loss": f"{loss_erm.item():.4f}", "IRM Penalty": f"{penalty.item():.4f}"})
+
+        return losses

torch_molecule/predictor/lstm/__init__.py

@@ -0,0 +1,3 @@
+from .modeling_lstm import LSTMMolecularPredictor
+
+__all__ = ['LSTMMolecularPredictor']

torch_molecule/predictor/lstm/model.py

@@ -0,0 +1,145 @@
+import torch
+import torch.nn as nn
+from sklearn.metrics import mean_absolute_error, r2_score, mean_squared_error
+
+from ...utils import init_weights
+
+# Define the PyTorch-based LSTM model
+class LSTM(nn.Module):
+    """LSTM-based model for molecular property prediction.
+    
+    Parameters
+    ----------
+    num_task : int
+        Number of prediction tasks.
+    input_dim : int
+        Size of vocabulary for SMILES tokenization.
+    output_dim : int
+        Dimension of embedding vectors.
+    LSTMunits : int
+        Number of hidden units in LSTM layers.
+    max_input_len : int
+        Maximum length of input sequences.
+
+    Attributes
+    ----------
+    embedding : nn.Embedding
+        Embedding layer that converts token indices to dense vectors.
+    lstm1 : nn.LSTM
+        First bidirectional LSTM layer.
+    lstm2 : nn.LSTM
+        Second bidirectional LSTM layer.
+    timedist_dense : nn.Linear
+        Time-distributed dense layer for feature transformation.
+    relu : nn.ReLU
+        ReLU activation function.
+    fc : nn.Linear
+        Final fully connected layer for prediction.
+
+    Notes
+    -----
+    The model architecture consists of:
+    1. An embedding layer to convert tokens to vectors
+    2. Two stacked bidirectional LSTM layers
+    3. A time-distributed dense layer with ReLU activation
+    4. A final fully connected layer for prediction
+    """
+
+    def __init__(self, num_task, input_dim, output_dim, LSTMunits, max_input_len):
+        """
+        input_dim: Vocabulary size
+        output_dim: Embedding dimension
+        LSTMunits: Number of hidden units in LSTM (unidirectional)
+        max_input_len: Input sequence length (used for later flattening)
+        """
+        super(LSTM, self).__init__()
+        self.num_task = num_task
+        hidden_dim = int(LSTMunits / 2)  
+        self.embedding = nn.Embedding(input_dim, output_dim)
+        self.lstm1 = nn.LSTM(input_size=output_dim, hidden_size=LSTMunits, bidirectional=True, batch_first=True)
+        self.lstm2 = nn.LSTM(input_size=LSTMunits * 2, hidden_size=LSTMunits, bidirectional=True, batch_first=True)
+        self.timedist_dense = nn.Linear(LSTMunits * 2, hidden_dim)
+        self.relu = nn.ReLU()
+        self.fc = nn.Linear(hidden_dim * max_input_len, 1)
+
+
+    def initialize_parameters(self, seed=None):
+        """Initialize model parameters randomly.
+        
+        Parameters
+        ----------
+        seed : int, optional
+            Random seed for reproducibility.
+        """
+        if seed is not None:
+            torch.manual_seed(seed)
+        
+        # Initialize the main components
+        init_weights(self.embedding)
+        init_weights(self.lstm1)
+        init_weights(self.lstm2)
+        init_weights(self.timedist_dense)
+        init_weights(self.relu)
+        init_weights(self.fc)
+        
+        # Reset all parameters using PyTorch Geometric's reset function
+        def reset_parameters(module):
+            if hasattr(module, 'reset_parameters'):
+                module.reset_parameters()
+            elif hasattr(module, 'weight') and hasattr(module.weight, 'data'):
+                init_weights(module)
+        
+        self.apply(reset_parameters) 
+
+    def compute_loss(self, batched_input, batched_label, criterion):
+        """Compute the loss for a batch of data.
+        
+        Parameters
+        ----------
+        batched_input : torch.Tensor
+            Batch of input sequences, shape (batch_size, seq_len).
+        batched_label : torch.Tensor
+            Batch of target values, shape (batch_size, 1).
+        criterion : callable
+            Loss function to use.
+
+        Returns
+        -------
+        torch.Tensor
+            Scalar loss value.
+        """
+        emb = self.embedding(batched_input)                
+        emb, _ = self.lstm1(emb)              
+        emb, _ = self.lstm2(emb)              
+        emb = self.relu(self.timedist_dense(emb))  
+        emb = emb.contiguous().view(emb.size(0), -1)  
+        prediction = self.fc(emb)     
+        target = batched_label.to(torch.float32)   
+        is_labeled = batched_label == batched_label
+        loss = criterion(prediction.to(torch.float32)[is_labeled], target[is_labeled])
+        return loss                 
+    
+    def forward(self, batched_input):
+        """Forward pass of the model.
+        
+        Parameters
+        ----------
+        batched_input : torch.Tensor
+            Batch of input sequences, shape (batch_size, seq_len).
+
+        Returns
+        -------
+        dict
+            Dictionary containing:
+                - prediction: Model predictions (shape: [batch_size, 1])
+        """
+        # batched_data: (batch_size, seq_len)
+        emb = self.embedding(batched_input)                   # -> (batch, seq_len, output_dim)
+        emb, _ = self.lstm1(emb)                    # -> (batch, seq_len, 2*LSTMunits)
+        emb, _ = self.lstm2(emb)                    # -> (batch, seq_len, 2*LSTMunits)
+        emb = self.relu(self.timedist_dense(emb))   # -> (batch, seq_len, hidden_dim)
+        emb = emb.contiguous().view(emb.size(0), -1)    # flatten: (batch, seq_len * hidden_dim)
+        prediction = self.fc(emb)                          # -> (batch, 1)
+        return {"prediction": prediction}
+
+