pyg-nightly 2.6.0.dev20240704__py3-none-any.whl → 2.8.0.dev20251207__py3-none-any.whl

torch_geometric/llm/models/molecule_gpt.py

@@ -0,0 +1,222 @@
+from typing import List, Optional
+
+import torch
+from torch import Tensor
+
+from torch_geometric.llm.models.llm import BOS, LLM, MAX_NEW_TOKENS
+from torch_geometric.nn.attention import QFormer
+from torch_geometric.utils import to_dense_batch
+
+
+def pad_or_truncate(embeddings: Tensor, max_seq_len: int,
+                    padding_value: int = 0) -> Tensor:
+    batch_size, current_seq_len, d = embeddings.size()
+
+    if current_seq_len > max_seq_len:
+        return embeddings[:, :max_seq_len, :]
+    elif current_seq_len < max_seq_len:
+        pad_tensor = torch.full((batch_size, max_seq_len - current_seq_len, d),
+                                padding_value, dtype=embeddings.dtype,
+                                device=embeddings.device)
+        return torch.cat([embeddings, pad_tensor], dim=1)
+    else:
+        return embeddings
+
+
+class MoleculeGPT(torch.nn.Module):
+    r"""The MoleculeGPT model from the `"MoleculeGPT: Instruction
+    Following Large Language Models for Molecular Property Prediction"
+    <https://ai4d3.github.io/papers/34.pdf>`_ paper.
+
+    Args:
+        llm (LLM): The LLM to use.
+        graph_encoder (torch.nn.Module): Encode 2D molecule graph.
+        smiles_encoder (torch.nn.Module): Encode 1D SMILES.
+        mlp_out_channels (int, optional): The size of each embedding
+            after qformer encoding. (default: :obj:`32`)
+        max_tokens (int, optional): Max output tokens of 1D/2D encoder.
+            (default: :obj:`20`)
+
+    .. warning::
+        This module has been tested with the following HuggingFace models
+
+        * :obj:`llm_to_use="lmsys/vicuna-7b-v1.5"`
+
+        and may not work with other models. See other models at `HuggingFace
+        Models <https://huggingface.co/models>`_ and let us know if you
+        encounter any issues.
+
+    .. note::
+        For an example of using :class:`MoleculeGPT`, see
+        `examples/llm/molecule_gpt.py <https://github.com/pyg-team/
+        pytorch_geometric/blob/master/examples/llm/molecule_gpt.py>`_.
+    """
+    def __init__(
+        self,
+        llm: LLM,
+        graph_encoder: torch.nn.Module,
+        smiles_encoder: torch.nn.Module,
+        mlp_out_channels: int = 32,
+        max_tokens: Optional[int] = 20,
+    ) -> None:
+        super().__init__()
+        self.llm = llm
+        self.graph_encoder = graph_encoder.to(self.llm.device)
+        self.smiles_encoder = smiles_encoder.to(self.llm.device)
+
+        self.graph_qformer = QFormer(
+            input_dim=self.graph_encoder.nn[-1].out_features,
+            hidden_dim=mlp_out_channels,
+            output_dim=mlp_out_channels,
+            num_heads=4,
+            num_layers=2,
+        ).to(self.llm.device)
+
+        self.smiles_qformer = QFormer(
+            input_dim=self.smiles_encoder.model.pooler.dense.out_features,
+            hidden_dim=mlp_out_channels,
+            output_dim=mlp_out_channels,
+            num_heads=4,
+            num_layers=2,
+        ).to(self.llm.device)
+
+        self.max_tokens = max_tokens
+
+        self.word_embedding = self.llm.word_embedding
+        self.llm_generator = self.llm.llm
+
+        # LLMs
+        in_dim = 2 * mlp_out_channels * max_tokens
+        out_dim = self.llm.llm.model.embed_tokens.embedding_dim
+        self.projector = torch.nn.Sequential(
+            torch.nn.Linear(in_dim, in_dim),
+            torch.nn.Sigmoid(),
+            torch.nn.Linear(in_dim, out_dim),
+        ).to(self.llm.device)
+
+    def encode(
+        self,
+        x: Tensor,
+        edge_index: Tensor,
+        batch: Tensor,
+        edge_attr: Optional[Tensor],
+        smiles: List[str],
+    ) -> Tensor:
+        batch_size = len(smiles)
+        # 2D Graph Branch: [bs, node_len, d]
+        x = x.to(self.llm.device)
+        edge_index = edge_index.to(self.llm.device)
+        if edge_attr is not None:
+            edge_attr = edge_attr.to(self.llm.device)
+        batch = batch.to(self.llm.device)
+
+        x_graph = self.graph_encoder(x, edge_index, edge_attr=edge_attr)
+        x_graph = to_dense_batch(x_graph, batch)[0]
+        out_graph = self.graph_qformer(x_graph)
+        out_graph = pad_or_truncate(out_graph, max_seq_len=self.max_tokens,
+                                    padding_value=0)
+        out_graph = out_graph.view(batch_size, -1)
+
+        # 1D SMILES Branch: [bs, seq_len, d]
+        x_smiles = self.smiles_encoder.encode(smiles,
+                                              output_device=self.llm.device)
+        out_smiles = self.smiles_qformer(x_smiles)
+        out_smiles = pad_or_truncate(out_smiles, max_seq_len=self.max_tokens,
+                                     padding_value=0)
+        out_smiles = out_smiles.view(batch_size, -1)
+
+        # Merge into LLMs
+        x_cat = torch.cat([out_graph, out_smiles], dim=1)
+        return x_cat
+
+    def forward(
+        self,
+        x: Tensor,
+        edge_index: Tensor,
+        batch: Tensor,
+        edge_attr: Optional[Tensor],
+        smiles: List[str],
+        instructions: List[str],
+        label: List[str],
+        additional_text_context: Optional[List[str]] = None,
+    ):
+        x = self.encode(x, edge_index, batch, edge_attr, smiles)
+        x = self.projector(x)
+        xs = x.split(1, dim=0)
+
+        batch_unique = batch.unique()
+        batch_size = len(instructions)
+        if len(batch_unique) < batch_size:
+            xs = [
+                xs[i] if i in batch_unique else None for i in range(batch_size)
+            ]
+
+        (
+            inputs_embeds,
+            attention_mask,
+            label_input_ids,
+        ) = self.llm._get_embeds(instructions, additional_text_context, xs,
+                                 label)
+
+        with self.llm.autocast_context:
+            outputs = self.llm_generator(
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                return_dict=True,
+                labels=label_input_ids,
+            )
+
+        return outputs.loss
+
+    @torch.no_grad()
+    def inference(
+        self,
+        x: Tensor,
+        edge_index: Tensor,
+        batch: Tensor,
+        edge_attr: Optional[Tensor],
+        smiles: List[str],
+        instructions: List[str],
+        additional_text_context: Optional[List[str]] = None,
+        max_out_tokens: Optional[int] = MAX_NEW_TOKENS,
+    ):
+        x = self.encode(x, edge_index, batch, edge_attr, smiles)
+        x = self.projector(x)
+        xs = x.split(1, dim=0)
+
+        # Handle questions without node features:
+        batch_unique = batch.unique()
+        batch_size = len(instructions)
+        if len(batch_unique) < batch_size:
+            xs = [
+                xs[i] if i in batch_unique else None for i in range(batch_size)
+            ]
+
+        inputs_embeds, attention_mask, _ = self.llm._get_embeds(
+            instructions, additional_text_context, xs)
+
+        bos_token = self.llm.tokenizer(
+            BOS,
+            add_special_tokens=False,
+        ).input_ids[0]
+
+        with self.llm.autocast_context:
+            outputs = self.llm_generator.generate(
+                inputs_embeds=inputs_embeds,
+                max_new_tokens=max_out_tokens,
+                attention_mask=attention_mask,
+                bos_token_id=bos_token,
+                use_cache=True  # Important to set!
+            )
+
+        return self.llm.tokenizer.batch_decode(
+            outputs,
+            skip_special_tokens=True,
+        )
+
+    def __repr__(self) -> str:
+        return (f'{self.__class__.__name__}(\n'
+                f'  llm={self.llm},\n'
+                f'  graph={self.graph_encoder.__class__.__name__},\n'
+                f'  smiles={self.smiles_encoder},\n'
+                f')')

torch_geometric/llm/models/protein_mpnn.py

@@ -0,0 +1,333 @@
+from itertools import product
+from typing import Tuple
+
+import torch
+import torch.nn.functional as F
+
+from torch_geometric.nn import knn_graph
+from torch_geometric.nn.conv import MessagePassing
+from torch_geometric.utils import to_dense_adj, to_dense_batch
+
+
+class PositionWiseFeedForward(torch.nn.Module):
+    def __init__(self, in_channels: int, hidden_channels: int) -> None:
+        super().__init__()
+        self.out = torch.nn.Sequential(
+            torch.nn.Linear(in_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, in_channels),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.out(x)
+
+
+class PositionalEncoding(torch.nn.Module):
+    def __init__(self, hidden_channels: int,
+                 max_relative_feature: int = 32) -> None:
+        super().__init__()
+        self.max_relative_feature = max_relative_feature
+        self.emb = torch.nn.Embedding(2 * max_relative_feature + 2,
+                                      hidden_channels)
+
+    def forward(self, offset, mask) -> torch.Tensor:
+        d = torch.clip(offset + self.max_relative_feature, 0,
+                       2 * self.max_relative_feature) * mask + (1 - mask) * (
+                           2 * self.max_relative_feature + 1)  # noqa: E501
+        return self.emb(d.long())
+
+
+class Encoder(MessagePassing):
+    def __init__(
+        self,
+        in_channels: int,
+        hidden_channels: int,
+        dropout: float = 0.1,
+        scale: float = 30,
+    ) -> None:
+        super().__init__()
+        self.out_v = torch.nn.Sequential(
+            torch.nn.Linear(in_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+        )
+        self.out_e = torch.nn.Sequential(
+            torch.nn.Linear(in_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+        )
+        self.dropout1 = torch.nn.Dropout(dropout)
+        self.dropout2 = torch.nn.Dropout(dropout)
+        self.dropout3 = torch.nn.Dropout(dropout)
+        self.norm1 = torch.nn.LayerNorm(hidden_channels)
+        self.norm2 = torch.nn.LayerNorm(hidden_channels)
+        self.norm3 = torch.nn.LayerNorm(hidden_channels)
+        self.scale = scale
+        self.dense = PositionWiseFeedForward(hidden_channels,
+                                             hidden_channels * 4)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        edge_index: torch.Tensor,
+        edge_attr: torch.Tensor,
+    ) -> torch.Tensor:
+        # x: [N, d_v]
+        # edge_index: [2, E]
+        # edge_attr: [E, d_e]
+        # update node features
+        h_message = self.propagate(x=x, edge_index=edge_index,
+                                   edge_attr=edge_attr)
+        dh = h_message / self.scale
+        x = self.norm1(x + self.dropout1(dh))
+        dh = self.dense(x)
+        x = self.norm2(x + self.dropout2(dh))
+        # update edge features
+        row, col = edge_index
+        x_i, x_j = x[row], x[col]
+        h_e = torch.cat([x_i, x_j, edge_attr], dim=-1)
+        h_e = self.out_e(h_e)
+        edge_attr = self.norm3(edge_attr + self.dropout3(h_e))
+        return x, edge_attr
+
+    def message(self, x_i: torch.Tensor, x_j: torch.Tensor,
+                edge_attr: torch.Tensor) -> torch.Tensor:
+        h = torch.cat([x_i, x_j, edge_attr], dim=-1)  # [E, 2*d_v + d_e]
+        h = self.out_e(h)  # [E, d_e]
+        return h
+
+
+class Decoder(MessagePassing):
+    def __init__(
+        self,
+        in_channels: int,
+        hidden_channels: int,
+        dropout: float = 0.1,
+        scale: float = 30,
+    ) -> None:
+        super().__init__()
+        self.out_v = torch.nn.Sequential(
+            torch.nn.Linear(in_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+            torch.nn.GELU(),
+            torch.nn.Linear(hidden_channels, hidden_channels),
+        )
+        self.dropout1 = torch.nn.Dropout(dropout)
+        self.dropout2 = torch.nn.Dropout(dropout)
+        self.norm1 = torch.nn.LayerNorm(hidden_channels)
+        self.norm2 = torch.nn.LayerNorm(hidden_channels)
+        self.scale = scale
+        self.dense = PositionWiseFeedForward(hidden_channels,
+                                             hidden_channels * 4)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        edge_index: torch.Tensor,
+        edge_attr: torch.Tensor,
+        x_label: torch.Tensor,
+        mask: torch.Tensor,
+    ) -> torch.Tensor:
+        # x: [N, d_v]
+        # edge_index: [2, E]
+        # edge_attr: [E, d_e]
+        h_message = self.propagate(x=x, x_label=x_label, edge_index=edge_index,
+                                   edge_attr=edge_attr, mask=mask)
+        dh = h_message / self.scale
+        x = self.norm1(x + self.dropout1(dh))
+        dh = self.dense(x)
+        x = self.norm2(x + self.dropout2(dh))
+        return x
+
+    def message(self, x_i: torch.Tensor, x_j: torch.Tensor,
+                x_label_j: torch.Tensor, edge_attr: torch.Tensor,
+                mask: torch.Tensor) -> torch.Tensor:
+        h_1 = torch.cat([x_j, edge_attr, x_label_j], dim=-1)
+        h_0 = torch.cat([x_j, edge_attr, torch.zeros_like(x_label_j)], dim=-1)
+        h = h_1 * mask + h_0 * (1 - mask)
+        h = torch.concat([x_i, h], dim=-1)
+        h = self.out_v(h)
+        return h
+
+
+class ProteinMPNN(torch.nn.Module):
+    r"""The ProteinMPNN model from the `"Robust deep learning--based
+    protein sequence design using ProteinMPNN"
+    <https://www.biorxiv.org/content/10.1101/2022.06.03.494563v1>`_ paper.
+
+    Args:
+        hidden_dim (int): Hidden channels.
+            (default: :obj:`128`)
+        num_encoder_layers (int): Number of encode layers.
+            (default: :obj:`3`)
+        num_decoder_layers (int): Number of decode layers.
+            (default: :obj:`3`)
+        num_neighbors (int): Number of neighbors for each atom.
+            (default: :obj:`30`)
+        num_rbf (int): Number of radial basis functions.
+            (default: :obj:`16`)
+        dropout (float): Dropout rate.
+            (default: :obj:`0.1`)
+        augment_eps (float): Augmentation epsilon for input coordinates.
+            (default: :obj:`0.2`)
+        num_positional_embedding (int): Number of positional embeddings.
+            (default: :obj:`16`)
+        vocab_size (int): Number of vocabulary.
+            (default: :obj:`21`)
+
+    .. note::
+        For an example of using :class:`ProteinMPNN`, see
+        `examples/llm/protein_mpnn.py <https://github.com/pyg-team/
+        pytorch_geometric/blob/master/examples/llm/protein_mpnn.py>`_.
+    """
+    def __init__(
+        self,
+        hidden_dim: int = 128,
+        num_encoder_layers: int = 3,
+        num_decoder_layers: int = 3,
+        num_neighbors: int = 30,
+        num_rbf: int = 16,
+        dropout: float = 0.1,
+        augment_eps: float = 0.2,
+        num_positional_embedding: int = 16,
+        vocab_size: int = 21,
+    ) -> None:
+        super().__init__()
+        self.augment_eps = augment_eps
+        self.hidden_dim = hidden_dim
+        self.num_neighbors = num_neighbors
+        self.num_rbf = num_rbf
+        self.embedding = PositionalEncoding(num_positional_embedding)
+        self.edge_mlp = torch.nn.Sequential(
+            torch.nn.Linear(num_positional_embedding + 400, hidden_dim),
+            torch.nn.LayerNorm(hidden_dim),
+            torch.nn.Linear(hidden_dim, hidden_dim),
+        )
+        self.label_embedding = torch.nn.Embedding(vocab_size, hidden_dim)
+        self.encoder_layers = torch.nn.ModuleList([
+            Encoder(hidden_dim * 3, hidden_dim, dropout)
+            for _ in range(num_encoder_layers)
+        ])
+
+        self.decoder_layers = torch.nn.ModuleList([
+            Decoder(hidden_dim * 4, hidden_dim, dropout)
+            for _ in range(num_decoder_layers)
+        ])
+        self.output = torch.nn.Linear(hidden_dim, vocab_size)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                torch.nn.init.xavier_uniform_(p)
+
+    def _featurize(
+        self,
+        x: torch.Tensor,
+        mask: torch.Tensor,
+        batch: torch.Tensor,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        N, Ca, C, O = (x[:, i, :] for i in range(4))  # noqa: E741
+        b = Ca - N
+        c = C - Ca
+        a = torch.cross(b, c, dim=-1)
+        Cb = -0.58273431 * a + 0.56802827 * b - 0.54067466 * c + Ca
+
+        valid_mask = mask.bool()
+        valid_Ca = Ca[valid_mask]
+        valid_batch = batch[valid_mask]
+
+        edge_index = knn_graph(valid_Ca, k=self.num_neighbors,
+                               batch=valid_batch, loop=True)
+
+        row, col = edge_index
+        original_indices = torch.arange(Ca.size(0),
+                                        device=x.device)[valid_mask]
+        edge_index_original = torch.stack(
+            [original_indices[row], original_indices[col]], dim=0)
+        row, col = edge_index_original
+
+        rbf_all = []
+        for A, B in list(product([N, Ca, C, O, Cb], repeat=2)):
+            distances = torch.sqrt(torch.sum((A[row] - B[col])**2, 1) + 1e-6)
+            rbf = self._rbf(distances)
+            rbf_all.append(rbf)
+
+        return edge_index_original, torch.cat(rbf_all, dim=-1)
+
+    def _rbf(self, D: torch.Tensor) -> torch.Tensor:
+        D_min, D_max, D_count = 2., 22., self.num_rbf
+        D_mu = torch.linspace(D_min, D_max, D_count, device=D.device)
+        D_mu = D_mu.view([1, -1])
+        D_sigma = (D_max - D_min) / D_count
+        D_expand = torch.unsqueeze(D, -1)
+        RBF = torch.exp(-((D_expand - D_mu) / D_sigma)**2)
+        return RBF
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        chain_seq_label: torch.Tensor,
+        mask: torch.Tensor,
+        chain_mask_all: torch.Tensor,
+        residue_idx: torch.Tensor,
+        chain_encoding_all: torch.Tensor,
+        batch: torch.Tensor,
+    ) -> torch.Tensor:
+        device = x.device
+        if self.training and self.augment_eps > 0:
+            x = x + self.augment_eps * torch.randn_like(x)
+
+        edge_index, edge_attr = self._featurize(x, mask, batch)
+
+        row, col = edge_index
+        offset = residue_idx[row] - residue_idx[col]
+        # find self vs non-self interaction
+        e_chains = ((chain_encoding_all[row] -
+                     chain_encoding_all[col]) == 0).long()
+        e_pos = self.embedding(offset, e_chains)
+        h_e = self.edge_mlp(torch.cat([edge_attr, e_pos], dim=-1))
+        h_v = torch.zeros(x.size(0), self.hidden_dim, device=x.device)
+
+        # encoder
+        for encoder in self.encoder_layers:
+            h_v, h_e = encoder(h_v, edge_index, h_e)
+
+        # mask
+        h_label = self.label_embedding(chain_seq_label)
+        batch_chain_mask_all, _ = to_dense_batch(chain_mask_all * mask,
+                                                 batch)  # [B, N]
+        # 0 - visible - encoder, 1 - masked - decoder
+        decoding_order = torch.argsort(
+            (batch_chain_mask_all + 1e-4) * (torch.abs(
+                torch.randn(batch_chain_mask_all.shape, device=device))))
+        mask_size = batch_chain_mask_all.size(1)
+        permutation_matrix_reverse = F.one_hot(decoding_order,
+                                               num_classes=mask_size).float()
+        order_mask_backward = torch.einsum(
+            'ij, biq, bjp->bqp',
+            1 - torch.triu(torch.ones(mask_size, mask_size, device=device)),
+            permutation_matrix_reverse,
+            permutation_matrix_reverse,
+        )
+        adj = to_dense_adj(edge_index, batch)
+        mask_attend = order_mask_backward[adj.bool()].unsqueeze(-1)
+
+        # decoder
+        for decoder in self.decoder_layers:
+            h_v = decoder(
+                h_v,
+                edge_index,
+                h_e,
+                h_label,
+                mask_attend,
+            )
+
+        logits = self.output(h_v)
+        return F.log_softmax(logits, dim=-1)