pyg-nightly 2.7.0.dev20250702__py3-none-any.whl → 2.7.0.dev20250704__py3-none-any.whl

torch_geometric/datasets/teeth3ds.py

@@ -0,0 +1,269 @@
+import json
+import os
+import os.path as osp
+from glob import glob
+from typing import Callable, Dict, List, Optional
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from torch_geometric.data import (
+    Data,
+    InMemoryDataset,
+    download_url,
+    extract_zip,
+)
+
+
+class Teeth3DS(InMemoryDataset):
+    r"""The Teeth3DS+ dataset from the `"An Extended Benchmark for Intra-oral
+    3D Scans Analysis" <https://crns-smartvision.github.io/teeth3ds/>`_ paper.
+
+    This dataset is the first comprehensive public benchmark designed to
+    advance the field of intra-oral 3D scan analysis developed as part of the
+    3DTeethSeg 2022 and 3DTeethLand 2024 MICCAI challenges, aiming to drive
+    research in teeth identification, segmentation, labeling, 3D modeling,
+    and dental landmark identification.
+    The dataset includes at least 1,800 intra-oral scans (containing 23,999
+    annotated teeth) collected from 900 patients, covering both upper and lower
+    jaws separately.
+
+    Args:
+        root (str): Root directory where the dataset should be saved.
+        split (str): The split name (one of :obj:`"Teeth3DS"`,
+            :obj:`"3DTeethSeg22_challenge"` or :obj:`"3DTeethLand_challenge"`).
+        train (bool, optional): If :obj:`True`, loads the training dataset,
+            otherwise the test dataset. (default: :obj:`True`)
+        num_samples (int, optional): Number of points to sample from each mesh.
+            (default: :obj:`30000`)
+        transform (callable, optional): A function/transform that takes in an
+            :obj:`torch_geometric.data.Data` object and returns a transformed
+            version. The data object will be transformed before every access.
+            (default: :obj:`None`)
+        pre_transform (callable, optional): A function/transform that takes in
+            an :obj:`torch_geometric.data.Data` object and returns a
+            transformed version. The data object will be transformed before
+            being saved to disk. (default: :obj:`None`)
+        force_reload (bool, optional): Whether to re-process the dataset.
+            (default: :obj:`False`)
+    """
+    urls = {
+        'data_part_1.zip':
+        'https://osf.io/download/qhprs/',
+        'data_part_2.zip':
+        'https://osf.io/download/4pwnr/',
+        'data_part_3.zip':
+        'https://osf.io/download/frwdp/',
+        'data_part_4.zip':
+        'https://osf.io/download/2arn4/',
+        'data_part_5.zip':
+        'https://osf.io/download/xrz5f/',
+        'data_part_6.zip':
+        'https://osf.io/download/23hgq/',
+        'data_part_7.zip':
+        'https://osf.io/download/u83ad/',
+        'train_test_split':
+        'https://files.de-1.osf.io/v1/'
+        'resources/xctdy/providers/osfstorage/?zip='
+    }
+
+    sample_url = {
+        'teeth3ds_sample': 'https://osf.io/download/vr38s/',
+    }
+
+    landmarks_urls = {
+        '3DTeethLand_landmarks_train.zip': 'https://osf.io/download/k5hbj/',
+        '3DTeethLand_landmarks_test.zip': 'https://osf.io/download/sqw5e/',
+    }
+
+    def __init__(
+        self,
+        root: str,
+        split:
+        str = 'Teeth3DS',  # [3DTeethSeg22_challenge, 3DTeethLand_challenge]
+        train: bool = True,
+        num_samples: int = 30000,
+        transform: Optional[Callable] = None,
+        pre_transform: Optional[Callable] = None,
+        force_reload: bool = False,
+    ) -> None:
+
+        self.mode = 'training' if train else 'testing'
+        self.split = split
+        self.num_samples = num_samples
+
+        super().__init__(root, transform, pre_transform,
+                         force_reload=force_reload)
+
+    @property
+    def processed_dir(self) -> str:
+        return os.path.join(self.root, f'processed_{self.split}_{self.mode}')
+
+    @property
+    def raw_file_names(self) -> List[str]:
+        return ['license.txt']
+
+    @property
+    def processed_file_names(self) -> List[str]:
+        # Directory containing train/test split files:
+        split_subdir = 'teeth3ds_sample' if self.split == 'sample' else ''
+        split_dir = osp.join(
+            self.raw_dir,
+            split_subdir,
+            f'{self.split}_train_test_split',
+        )
+
+        split_files = glob(osp.join(split_dir, f'{self.mode}*.txt'))
+
+        # Collect all file names from the split files:
+        combined_list = []
+        for file_path in split_files:
+            with open(file_path) as file:
+                combined_list.extend(file.read().splitlines())
+
+        # Generate the list of processed file paths:
+        return [f'{file_name}.pt' for file_name in combined_list]
+
+    def download(self) -> None:
+        if self.split == 'sample':
+            for key, url in self.sample_url.items():
+                path = download_url(url, self.root, filename=key)
+                extract_zip(path, self.raw_dir)
+                os.unlink(path)
+        else:
+            for key, url in self.urls.items():
+                path = download_url(url, self.root, filename=key)
+                extract_zip(path, self.raw_dir)
+                os.unlink(path)
+            for key, url in self.landmarks_urls.items():
+                path = download_url(url, self.root, filename=key)
+                extract_zip(path, self.raw_dir)  # Extract each downloaded part
+                os.unlink(path)
+
+    def process_file(self, file_path: str) -> Optional[Data]:
+        """Processes the input file path to load mesh data, annotations,
+        and prepare the input features for a graph-based deep learning model.
+        """
+        import trimesh
+        from fpsample import bucket_fps_kdline_sampling
+
+        mesh = trimesh.load_mesh(file_path)
+
+        if isinstance(mesh, list):
+            # Handle the case where a list of Geometry objects is returned
+            mesh = mesh[0]
+
+        vertices = mesh.vertices
+        vertex_normals = mesh.vertex_normals
+
+        # Perform sampling on mesh vertices:
+        if len(vertices) < self.num_samples:
+            sampled_indices = np.random.choice(
+                len(vertices),
+                self.num_samples,
+                replace=True,
+            )
+        else:
+            sampled_indices = bucket_fps_kdline_sampling(
+                vertices,
+                self.num_samples,
+                h=5,
+                start_idx=0,
+            )
+
+        if len(sampled_indices) != self.num_samples:
+            raise RuntimeError(f"Sampled points mismatch, expected "
+                               f"{self.num_samples} points, but got "
+                               f"{len(sampled_indices)} for '{file_path}'")
+
+        # Extract features and annotations for the sampled points:
+        pos = torch.tensor(vertices[sampled_indices], dtype=torch.float)
+        x = torch.tensor(vertex_normals[sampled_indices], dtype=torch.float)
+
+        # Load segmentation annotations:
+        seg_annotation_path = file_path.replace('.obj', '.json')
+        if osp.exists(seg_annotation_path):
+            with open(seg_annotation_path) as f:
+                seg_annotations = json.load(f)
+            y = torch.tensor(
+                np.asarray(seg_annotations['labels'])[sampled_indices],
+                dtype=torch.float)
+            instances = torch.tensor(
+                np.asarray(seg_annotations['instances'])[sampled_indices],
+                dtype=torch.float)
+        else:
+            y = torch.empty(0, 3)
+            instances = torch.empty(0, 3)
+
+        # Load landmarks annotations:
+        landmarks_annotation_path = file_path.replace('.obj', '__kpt.json')
+
+        # Parse keypoint annotations into structured tensors:
+        keypoints_dict: Dict[str, List] = {
+            key: []
+            for key in [
+                'Mesial', 'Distal', 'Cusp', 'InnerPoint', 'OuterPoint',
+                'FacialPoint'
+            ]
+        }
+        keypoint_tensors: Dict[str, torch.Tensor] = {
+            key: torch.empty(0, 3)
+            for key in [
+                'Mesial', 'Distal', 'Cusp', 'InnerPoint', 'OuterPoint',
+                'FacialPoint'
+            ]
+        }
+        if osp.exists(landmarks_annotation_path):
+            with open(landmarks_annotation_path) as f:
+                landmarks_annotations = json.load(f)
+
+            for keypoint in landmarks_annotations['objects']:
+                keypoints_dict[keypoint['class']].extend(keypoint['coord'])
+
+            keypoint_tensors = {
+                k: torch.tensor(np.asarray(v),
+                                dtype=torch.float).reshape(-1, 3)
+                for k, v in keypoints_dict.items()
+            }
+
+        data = Data(
+            pos=pos,
+            x=x,
+            y=y,
+            instances=instances,
+            jaw=file_path.split('.obj')[0].split('_')[1],
+            mesial=keypoint_tensors['Mesial'],
+            distal=keypoint_tensors['Distal'],
+            cusp=keypoint_tensors['Cusp'],
+            inner_point=keypoint_tensors['InnerPoint'],
+            outer_point=keypoint_tensors['OuterPoint'],
+            facial_point=keypoint_tensors['FacialPoint'],
+        )
+
+        if self.pre_transform is not None:
+            data = self.pre_transform(data)
+
+        return data
+
+    def process(self) -> None:
+        for file in tqdm(self.processed_file_names):
+            name = file.split('.')[0]
+            path = osp.join(self.raw_dir, '**', '*', name + '.obj')
+            paths = glob(path)
+            if len(paths) == 1:
+                data = self.process_file(paths[0])
+                torch.save(data, osp.join(self.processed_dir, file))
+
+    def len(self) -> int:
+        return len(self.processed_file_names)
+
+    def get(self, idx: int) -> Data:
+        return torch.load(
+            osp.join(self.processed_dir, self.processed_file_names[idx]),
+            weights_only=False,
+        )
+
+    def __repr__(self) -> str:
+        return (f'{self.__class__.__name__}({len(self)}, '
+                f'mode={self.mode}, split={self.split})')

torch_geometric/nn/attention/__init__.py

@@ -1,9 +1,11 @@
 from .performer import PerformerAttention
 from .qformer import QFormer
 from .sgformer import SGFormerAttention
+from .polynormer import PolynormerAttention
 
 __all__ = [
     'PerformerAttention',
     'QFormer',
     'SGFormerAttention',
+    'PolynormerAttention',
 ]

torch_geometric/nn/attention/polynormer.py

@@ -0,0 +1,107 @@
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+
+
+class PolynormerAttention(torch.nn.Module):
+    r"""The polynomial-expressive attention mechanism from the
+    `"Polynormer: Polynomial-Expressive Graph Transformer in Linear Time"
+    <https://arxiv.org/abs/2403.01232>`_ paper.
+
+    Args:
+        channels (int): Size of each input sample.
+        heads (int, optional): Number of parallel attention heads.
+        head_channels (int, optional): Size of each attention head.
+            (default: :obj:`64.`)
+        beta (float, optional): Polynormer beta initialization.
+            (default: :obj:`0.9`)
+        qkv_bias (bool, optional): If specified, add bias to query, key
+            and value in the self attention. (default: :obj:`False`)
+        qk_shared (bool optional): Whether weight of query and key are shared.
+            (default: :obj:`True`)
+        dropout (float, optional): Dropout probability of the final
+            attention output. (default: :obj:`0.0`)
+    """
+    def __init__(
+        self,
+        channels: int,
+        heads: int,
+        head_channels: int = 64,
+        beta: float = 0.9,
+        qkv_bias: bool = False,
+        qk_shared: bool = True,
+        dropout: float = 0.0,
+    ) -> None:
+        super().__init__()
+
+        self.head_channels = head_channels
+        self.heads = heads
+        self.beta = beta
+        self.qk_shared = qk_shared
+
+        inner_channels = heads * head_channels
+        self.h_lins = torch.nn.Linear(channels, inner_channels)
+        if not self.qk_shared:
+            self.q = torch.nn.Linear(channels, inner_channels, bias=qkv_bias)
+        self.k = torch.nn.Linear(channels, inner_channels, bias=qkv_bias)
+        self.v = torch.nn.Linear(channels, inner_channels, bias=qkv_bias)
+        self.lns = torch.nn.LayerNorm(inner_channels)
+        self.lin_out = torch.nn.Linear(inner_channels, inner_channels)
+        self.dropout = torch.nn.Dropout(dropout)
+
+    def forward(self, x: Tensor, mask: Optional[Tensor] = None) -> Tensor:
+        r"""Forward pass.
+
+        Args:
+            x (torch.Tensor): Node feature tensor
+                :math:`\mathbf{X} \in \mathbb{R}^{B \times N \times F}`, with
+                batch-size :math:`B`, (maximum) number of nodes :math:`N` for
+                each graph, and feature dimension :math:`F`.
+            mask (torch.Tensor, optional): Mask matrix
+                :math:`\mathbf{M} \in {\{ 0, 1 \}}^{B \times N}` indicating
+                the valid nodes for each graph. (default: :obj:`None`)
+        """
+        B, N, *_ = x.shape
+        h = self.h_lins(x)
+        k = self.k(x).sigmoid().view(B, N, self.head_channels, self.heads)
+        if self.qk_shared:
+            q = k
+        else:
+            q = F.sigmoid(self.q(x)).view(B, N, self.head_channels, self.heads)
+        v = self.v(x).view(B, N, self.head_channels, self.heads)
+
+        if mask is not None:
+            mask = mask[:, :, None, None]
+            v.masked_fill_(~mask, 0.)
+
+        # numerator
+        kv = torch.einsum('bndh, bnmh -> bdmh', k, v)
+        num = torch.einsum('bndh, bdmh -> bnmh', q, kv)
+
+        # denominator
+        k_sum = torch.einsum('bndh -> bdh', k)
+        den = torch.einsum('bndh, bdh -> bnh', q, k_sum).unsqueeze(2)
+
+        # linear global attention based on kernel trick
+        x = (num / (den + 1e-6)).reshape(B, N, -1)
+        x = self.lns(x) * (h + self.beta)
+        x = F.relu(self.lin_out(x))
+        x = self.dropout(x)
+
+        return x
+
+    def reset_parameters(self) -> None:
+        self.h_lins.reset_parameters()
+        if not self.qk_shared:
+            self.q.reset_parameters()
+        self.k.reset_parameters()
+        self.v.reset_parameters()
+        self.lns.reset_parameters()
+        self.lin_out.reset_parameters()
+
+    def __repr__(self) -> str:
+        return (f'{self.__class__.__name__}('
+                f'heads={self.heads}, '
+                f'head_channels={self.head_channels})')

torch_geometric/nn/conv/meshcnn_conv.py

@@ -1,7 +1,7 @@
 # The below is to suppress the warning on torch.nn.conv.MeshCNNConv::update
 # pyright: reportIncompatibleMethodOverride=false
+import warnings
 from typing import Optional
-from warnings import warn
 
 import torch
 from torch.nn import Linear, Module, ModuleList
@@ -456,13 +456,10 @@ class MeshCNNConv(MessagePassing):
                 {type(network)}"
             if not hasattr(network, "in_channels") and \
                     not hasattr(network, "in_features"):
-                warn(
-                    f"kernel[{i}] does not have attribute \
-                            'in_channels' nor 'out_features'. The \
-                            network must take as input a \
-                            {self.in_channels}-dimensional tensor. \
-                            Still, assuming user configured \
-                            correctly. Continuing..", stacklevel=2)
+                warnings.warn(
+                    f"kernel[{i}] does not have attribute 'in_channels' nor "
+                    f"'out_features'. The network must take as input a "
+                    f"{self.in_channels}-dimensional tensor.", stacklevel=2)
             else:
                 input_dimension = getattr(network, "in_channels",
                                           network.in_features)
@@ -475,13 +472,10 @@ class MeshCNNConv(MessagePassing):
 
             if not hasattr(network, "out_channels") and \
                     not hasattr(network, "out_features"):
-                warn(
-                    f"kernel[{i}] does not have attribute \
-                                'in_channels' nor 'out_features'. The \
-                                network must take as input a \
-                                {self.in_channels}-dimensional tensor. \
-                                Still, assuming user configured \
-                                correctly. Continuing..", stacklevel=2)
+                warnings.warn(
+                    f"kernel[{i}] does not have attribute 'in_channels' nor "
+                    f"'out_features'. The network must take as input a "
+                    f"{self.in_channels}-dimensional tensor.", stacklevel=2)
             else:
                 output_dimension = getattr(network, "out_channels",
                                            network.out_features)

torch_geometric/nn/models/__init__.py

@@ -32,8 +32,10 @@ from .visnet import ViSNet
 from .g_retriever import GRetriever
 from .git_mol import GITMol
 from .molecule_gpt import MoleculeGPT
+from .protein_mpnn import ProteinMPNN
 from .glem import GLEM
 from .sgformer import SGFormer
+from .polynormer import Polynormer
 # Deprecated:
 from torch_geometric.explain.algorithm.captum import (to_captum_input,
                                                       captum_output_to_dicts)
@@ -86,7 +88,9 @@ __all__ = classes = [
     'GRetriever',
     'GITMol',
     'MoleculeGPT',
+    'ProteinMPNN',
     'GLEM',
     'SGFormer',
+    'Polynormer',
     'ARLinkPredictor',
 ]

torch_geometric/nn/models/glem.py

@@ -8,6 +8,13 @@ from torch_geometric.loader import DataLoader, NeighborLoader
 from torch_geometric.nn.models import GraphSAGE, basic_gnn
 
 
+def deal_nan(x):
+    if isinstance(x, torch.Tensor):
+        x = x.clone()
+        x[torch.isnan(x)] = 0.0
+    return x
+
+
 class GLEM(torch.nn.Module):
     r"""This GNN+LM co-training model is based on GLEM from the `"Learning on
     Large-scale Text-attributed Graphs via Variational Inference"
@@ -379,9 +386,6 @@ class GLEM(torch.nn.Module):
             is_augmented: use EM or just train GNN and LM with gold data
 
         """
-        def deal_nan(x):
-            return 0 if torch.isnan(x) else x
-
         if is_augmented and (sum(~is_gold) > 0):
             mle_loss = deal_nan(loss_func(logits[is_gold], labels[is_gold]))
             # all other labels beside from ground truth(gold labels)

torch_geometric/nn/models/polynormer.py

@@ -0,0 +1,206 @@
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+
+from torch_geometric.nn import GATConv, GCNConv
+from torch_geometric.nn.attention import PolynormerAttention
+from torch_geometric.utils import to_dense_batch
+
+
+class Polynormer(torch.nn.Module):
+    r"""The polynormer module from the
+    `"Polynormer: polynomial-expressive graph
+    transformer in linear time"
+    <https://arxiv.org/abs/2403.01232>`_ paper.
+
+    Args:
+        in_channels (int): Input channels.
+        hidden_channels (int): Hidden channels.
+        out_channels (int): Output channels.
+        local_layers (int): The number of local attention layers.
+            (default: :obj:`7`)
+        global_layers (int): The number of global attention layers.
+            (default: :obj:`2`)
+        in_dropout (float): Input dropout rate.
+            (default: :obj:`0.15`)
+        dropout (float): Dropout rate.
+            (default: :obj:`0.5`)
+        global_dropout (float): Global dropout rate.
+            (default: :obj:`0.5`)
+        heads (int): The number of heads.
+            (default: :obj:`1`)
+        beta (float): Aggregate type.
+            (default: :obj:`0.9`)
+        qk_shared (bool optional): Whether weight of query and key are shared.
+            (default: :obj:`True`)
+        pre_ln (bool): Pre layer normalization.
+            (default: :obj:`False`)
+        post_bn (bool): Post batch normlization.
+            (default: :obj:`True`)
+        local_attn (bool): Whether use local attention.
+            (default: :obj:`False`)
+    """
+    def __init__(
+        self,
+        in_channels: int,
+        hidden_channels: int,
+        out_channels: int,
+        local_layers: int = 7,
+        global_layers: int = 2,
+        in_dropout: float = 0.15,
+        dropout: float = 0.5,
+        global_dropout: float = 0.5,
+        heads: int = 1,
+        beta: float = 0.9,
+        qk_shared: bool = False,
+        pre_ln: bool = False,
+        post_bn: bool = True,
+        local_attn: bool = False,
+    ) -> None:
+        super().__init__()
+        self._global = False
+        self.in_drop = in_dropout
+        self.dropout = dropout
+        self.pre_ln = pre_ln
+        self.post_bn = post_bn
+
+        self.beta = beta
+
+        self.h_lins = torch.nn.ModuleList()
+        self.local_convs = torch.nn.ModuleList()
+        self.lins = torch.nn.ModuleList()
+        self.lns = torch.nn.ModuleList()
+        if self.pre_ln:
+            self.pre_lns = torch.nn.ModuleList()
+        if self.post_bn:
+            self.post_bns = torch.nn.ModuleList()
+
+        # first layer
+        inner_channels = heads * hidden_channels
+        self.h_lins.append(torch.nn.Linear(in_channels, inner_channels))
+        if local_attn:
+            self.local_convs.append(
+                GATConv(in_channels, hidden_channels, heads=heads, concat=True,
+                        add_self_loops=False, bias=False))
+        else:
+            self.local_convs.append(
+                GCNConv(in_channels, inner_channels, cached=False,
+                        normalize=True))
+
+        self.lins.append(torch.nn.Linear(in_channels, inner_channels))
+        self.lns.append(torch.nn.LayerNorm(inner_channels))
+        if self.pre_ln:
+            self.pre_lns.append(torch.nn.LayerNorm(in_channels))
+        if self.post_bn:
+            self.post_bns.append(torch.nn.BatchNorm1d(inner_channels))
+
+        # following layers
+        for _ in range(local_layers - 1):
+            self.h_lins.append(torch.nn.Linear(inner_channels, inner_channels))
+            if local_attn:
+                self.local_convs.append(
+                    GATConv(inner_channels, hidden_channels, heads=heads,
+                            concat=True, add_self_loops=False, bias=False))
+            else:
+                self.local_convs.append(
+                    GCNConv(inner_channels, inner_channels, cached=False,
+                            normalize=True))
+
+            self.lins.append(torch.nn.Linear(inner_channels, inner_channels))
+            self.lns.append(torch.nn.LayerNorm(inner_channels))
+            if self.pre_ln:
+                self.pre_lns.append(torch.nn.LayerNorm(heads *
+                                                       hidden_channels))
+            if self.post_bn:
+                self.post_bns.append(torch.nn.BatchNorm1d(inner_channels))
+
+        self.lin_in = torch.nn.Linear(in_channels, inner_channels)
+        self.ln = torch.nn.LayerNorm(inner_channels)
+
+        self.global_attn = torch.nn.ModuleList()
+        for _ in range(global_layers):
+            self.global_attn.append(
+                PolynormerAttention(
+                    channels=hidden_channels,
+                    heads=heads,
+                    head_channels=hidden_channels,
+                    beta=beta,
+                    dropout=global_dropout,
+                    qk_shared=qk_shared,
+                ))
+        self.pred_local = torch.nn.Linear(inner_channels, out_channels)
+        self.pred_global = torch.nn.Linear(inner_channels, out_channels)
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        for local_conv in self.local_convs:
+            local_conv.reset_parameters()
+        for attn in self.global_attn:
+            attn.reset_parameters()
+        for lin in self.lins:
+            lin.reset_parameters()
+        for h_lin in self.h_lins:
+            h_lin.reset_parameters()
+        for ln in self.lns:
+            ln.reset_parameters()
+        if self.pre_ln:
+            for p_ln in self.pre_lns:
+                p_ln.reset_parameters()
+        if self.post_bn:
+            for p_bn in self.post_bns:
+                p_bn.reset_parameters()
+        self.lin_in.reset_parameters()
+        self.ln.reset_parameters()
+        self.pred_local.reset_parameters()
+        self.pred_global.reset_parameters()
+
+    def forward(
+        self,
+        x: Tensor,
+        edge_index: Tensor,
+        batch: Optional[Tensor],
+    ) -> Tensor:
+        r"""Forward pass.
+
+        Args:
+            x (torch.Tensor): The input node features.
+            edge_index (torch.Tensor or SparseTensor): The edge indices.
+            batch (torch.Tensor, optional): The batch vector
+                :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns
+                each element to a specific example.
+        """
+        x = F.dropout(x, p=self.in_drop, training=self.training)
+
+        # equivariant local attention
+        x_local = 0
+        for i, local_conv in enumerate(self.local_convs):
+            if self.pre_ln:
+                x = self.pre_lns[i](x)
+            h = self.h_lins[i](x)
+            h = F.relu(h)
+            x = local_conv(x, edge_index) + self.lins[i](x)
+            if self.post_bn:
+                x = self.post_bns[i](x)
+            x = F.relu(x)
+            x = F.dropout(x, p=self.dropout, training=self.training)
+            x = (1 - self.beta) * self.lns[i](h * x) + self.beta * x
+            x_local = x_local + x
+
+        # equivariant global attention
+        if self._global:
+            batch, indices = batch.sort()
+            rev_perm = torch.empty_like(indices)
+            rev_perm[indices] = torch.arange(len(indices),
+                                             device=indices.device)
+            x_local = self.ln(x_local[indices])
+            x_global, mask = to_dense_batch(x_local, batch)
+            for attn in self.global_attn:
+                x_global = attn(x_global, mask)
+            x = x_global[mask][rev_perm]
+            x = self.pred_global(x)
+        else:
+            x = self.pred_local(x_local)
+
+        return F.log_softmax(x, dim=-1)