gsMap 1.66__py3-none-any.whl → 1.67__py3-none-any.whl

gsMap/GNN_VAE/adjacency_matrix.py

@@ -1,95 +1,75 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Jul  4 21:31:27 2023
-
-@author: songliyang
-"""
 import numpy as np
 import pandas as pd
 import scipy.sparse as sp
-import sklearn.neighbors
+from sklearn.neighbors import NearestNeighbors
 import torch
 
-
-def Cal_Spatial_Net(adata, n_neighbors=5, verbose=True):
-    """\
-    Construct the spatial neighbor networks.
-    """
-    #- 
+def cal_spatial_net(adata, n_neighbors=5, verbose=True):
+    """Construct the spatial neighbor network."""
     if verbose:
         print('------Calculating spatial graph...')
-    coor = pd.DataFrame(adata.obsm['spatial'])
-    coor.index = adata.obs.index
-    #- 
-    nbrs = sklearn.neighbors.NearestNeighbors(n_neighbors=n_neighbors).fit(coor)
-    #- 
-    distances, indices = nbrs.kneighbors(coor, return_distance=True)
-    KNN_list = []
-    for it in range(indices.shape[0]):
-        KNN_list.append(pd.DataFrame(zip([it]*indices[it].shape[0], indices[it], distances[it])))
-    #- 
-    KNN_df = pd.concat(KNN_list)
-    KNN_df.columns = ['Cell1', 'Cell2', 'Distance']
-    #- 
-    Spatial_Net = KNN_df.copy()
-    Spatial_Net = Spatial_Net.loc[Spatial_Net['Distance']>0,]
-    id_cell_trans = dict(zip(range(coor.shape[0]), np.array(coor.index), ))
-    Spatial_Net['Cell1'] = Spatial_Net['Cell1'].map(id_cell_trans)
-    Spatial_Net['Cell2'] = Spatial_Net['Cell2'].map(id_cell_trans)
-    #- 
-    return Spatial_Net
-
+    coor = pd.DataFrame(adata.obsm['spatial'], index=adata.obs.index)
+    nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(coor)
+    distances, indices = nbrs.kneighbors(coor)
+    n_cells, n_neighbors = indices.shape
+    cell_indices = np.arange(n_cells)
+    cell1 = np.repeat(cell_indices, n_neighbors)
+    cell2 = indices.flatten()
+    distance = distances.flatten()
+    knn_df = pd.DataFrame({'Cell1': cell1, 'Cell2': cell2, 'Distance': distance})
+    knn_df = knn_df[knn_df['Distance'] > 0].copy()
+    cell_id_map = dict(zip(cell_indices, coor.index))
+    knn_df['Cell1'] = knn_df['Cell1'].map(cell_id_map)
+    knn_df['Cell2'] = knn_df['Cell2'].map(cell_id_map)
+    return knn_df
 
 def sparse_mx_to_torch_sparse_tensor(sparse_mx):
     """Convert a scipy sparse matrix to a torch sparse tensor."""
     sparse_mx = sparse_mx.tocoo().astype(np.float32)
-    indices = torch.from_numpy(np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
+    indices = torch.from_numpy(
+        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64)
+    )
     values = torch.from_numpy(sparse_mx.data)
     shape = torch.Size(sparse_mx.shape)
     return torch.sparse.FloatTensor(indices, values, shape)
 
-
 def preprocess_graph(adj):
+    """Symmetrically normalize the adjacency matrix."""
     adj = sp.coo_matrix(adj)
     adj_ = adj + sp.eye(adj.shape[0])
-    rowsum = np.array(adj_.sum(1))
-    degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5).flatten())
+    rowsum = np.array(adj_.sum(1)).flatten()
+    degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5))
     adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo()
     return sparse_mx_to_torch_sparse_tensor(adj_normalized)
 
-
-
-def Construct_Adjacency_Matrix(adata,Params, verbose=True):
-    # Construct the neighbor graph 
-    Spatial_Net = Cal_Spatial_Net(adata, n_neighbors=Params.n_neighbors)
-    #- 
+def construct_adjacency_matrix(adata, params, verbose=True):
+    """Construct the adjacency matrix from spatial data."""
+    spatial_net = cal_spatial_net(adata, n_neighbors=params.n_neighbors, verbose=verbose)
     if verbose:
-        print('The graph contains %d edges, %d cells.' %(Spatial_Net.shape[0], adata.n_obs))
-        print('%.2f neighbors per cell on average.' %(Spatial_Net.shape[0]/adata.n_obs))
-    #-  
-    cells = np.array(adata.obs.index)
-    cells_id_tran = dict(zip(cells, range(cells.shape[0])))
-    #- 
-    G_df = Spatial_Net.copy()
-    G_df['Cell1'] = G_df['Cell1'].map(cells_id_tran)
-    G_df['Cell2'] = G_df['Cell2'].map(cells_id_tran)
-    #- 
-    if Params.weighted_adj:
-        distance_normalized = G_df.Distance/(max(G_df.Distance)+1)
-        adj_org = sp.coo_matrix((np.exp(-distance_normalized**2/(2)), (G_df['Cell1'], G_df['Cell2'])), shape=(adata.n_obs, adata.n_obs))
+        num_edges = spatial_net.shape[0]
+        num_cells = adata.n_obs
+        print(f'The graph contains {num_edges} edges, {num_cells} cells.')
+        print(f'{num_edges / num_cells:.2f} neighbors per cell on average.')
+    cell_ids = {cell: idx for idx, cell in enumerate(adata.obs.index)}
+    spatial_net['Cell1'] = spatial_net['Cell1'].map(cell_ids)
+    spatial_net['Cell2'] = spatial_net['Cell2'].map(cell_ids)
+    if params.weighted_adj:
+        distance_normalized = spatial_net['Distance'] / (spatial_net['Distance'].max() + 1)
+        weights = np.exp(-0.5 * distance_normalized ** 2)
+        adj_org = sp.coo_matrix(
+            (weights, (spatial_net['Cell1'], spatial_net['Cell2'])),
+            shape=(adata.n_obs, adata.n_obs)
+        )
     else:
-        adj_org = sp.coo_matrix((np.ones(G_df.shape[0]), (G_df['Cell1'], G_df['Cell2'])), shape=(adata.n_obs, adata.n_obs))
-    #- 
-    adj_m1 = adj_org
-    adj_norm_m1 = preprocess_graph(adj_m1)
-    adj_label_m1 = adj_m1 + sp.eye(adj_m1.shape[0])
-    norm_m1 = adj_m1.shape[0] * adj_m1.shape[0] / float((adj_m1.shape[0] * adj_m1.shape[0] - adj_m1.sum()) * 2)
-    #- 
+        adj_org = sp.coo_matrix(
+            (np.ones(spatial_net.shape[0]), (spatial_net['Cell1'], spatial_net['Cell2'])),
+            shape=(adata.n_obs, adata.n_obs)
+        )
+    adj_norm = preprocess_graph(adj_org)
+    norm_value = adj_org.shape[0] ** 2 / ((adj_org.shape[0] ** 2 - adj_org.sum()) * 2)
     graph_dict = {
         "adj_org": adj_org,
-        "adj_norm": adj_norm_m1,
-        "norm_value": norm_m1
+        "adj_norm": adj_norm,
+        "norm_value": norm_value
     }
-    #- 
     return graph_dict

gsMap/GNN_VAE/model.py

@@ -1,87 +1,89 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Jul  3 11:42:44 2023
-
-@author: songliyang
-"""
-
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch_geometric.nn import GATConv
 
-
 def full_block(in_features, out_features, p_drop):
-    return nn.Sequential(nn.Linear(in_features, out_features),
-                         nn.BatchNorm1d(out_features),
-                         nn.ELU(),
-                         nn.Dropout(p=p_drop))
+    return nn.Sequential(
+        nn.Linear(in_features, out_features),
+        nn.BatchNorm1d(out_features),
+        nn.ELU(),
+        nn.Dropout(p=p_drop)
+    )
 
-
-class GNN(nn.Module):
-    def __init__(self, in_features, out_features, dr=0, act=F.relu,heads=1):
+class GATModel(nn.Module):
+    def __init__(self, input_dim, params, num_classes=1):
         super().__init__()
-        self.conv1 = GATConv(in_features, out_features,heads)
-        self.act = act
-        self.dr = dr
-    #-
-    def forward(self, x, edge_index):
-        out = self.conv1(x, edge_index)
-        out = self.act(out)
-        out = F.dropout(out, self.dr, self.training)
-        return out
-
-
-    
-class GNN_VAE_Model(nn.Module):
-    def __init__(self, input_dim,params,num_classes=1):
-        super(GNN_VAE_Model, self).__init__()
         self.var = params.var
         self.num_classes = num_classes
-            
+        self.params = params
+
         # Encoder
-        self.encoder = nn.Sequential()
-        self.encoder.add_module('encoder_L1', full_block(input_dim, params.feat_hidden1, params.p_drop))
-        self.encoder.add_module('encoder_L2', full_block(params.feat_hidden1, params.feat_hidden2, params.p_drop))
-        
-        # GNN (GAT)
-        self.gn1 = GNN(params.feat_hidden2, params.gat_hidden1, params.p_drop, act=F.relu,heads = params.nheads)
-        self.gn2 = GNN(params.gat_hidden1*params.nheads, params.gat_hidden2, params.p_drop, act=lambda x: x)
-        self.gn3 = GNN(params.gat_hidden1*params.nheads, params.gat_hidden2, params.p_drop, act=lambda x: x) 
-        
+        self.encoder = nn.Sequential(
+            full_block(input_dim, params.feat_hidden1, params.p_drop),
+            full_block(params.feat_hidden1, params.feat_hidden2, params.p_drop)
+        )
+
+        # GAT Layers
+        self.gat1 = GATConv(
+            in_channels=params.feat_hidden2,
+            out_channels=params.gat_hidden1,
+            heads=params.nheads,
+            dropout=params.p_drop
+        )
+        self.gat2 = GATConv(
+            in_channels=params.gat_hidden1 * params.nheads,
+            out_channels=params.gat_hidden2,
+            heads=1,
+            concat=False,
+            dropout=params.p_drop
+        )
+        if self.var:
+            self.gat3 = GATConv(
+                in_channels=params.gat_hidden1 * params.nheads,
+                out_channels=params.gat_hidden2,
+                heads=1,
+                concat=False,
+                dropout=params.p_drop
+            )
+
         # Decoder
-        self.decoder = nn.Sequential()
-        self.decoder.add_module('decoder_L1', full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop))
-        self.decoder.add_module('decoder_L2', full_block(params.feat_hidden2, params.feat_hidden1, params.p_drop))
-        self.decoder.add_module('decoder_output', nn.Sequential(nn.Linear(params.feat_hidden1, input_dim)))
-        
-        # Cluster
-        self.cluster = nn.Sequential()
-        self.cluster.add_module('cluster_L1', full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop))
-        self.cluster.add_module('cluster_output', nn.Linear(params.feat_hidden2, self.num_classes))
-           
-    def encode(self, x, adj):
-        feat_x = self.encoder(x)
-        hidden1 = self.gn1(feat_x, adj)
-        mu = self.gn2(hidden1, adj)
+        self.decoder = nn.Sequential(
+            full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop),
+            full_block(params.feat_hidden2, params.feat_hidden1, params.p_drop),
+            nn.Linear(params.feat_hidden1, input_dim)
+        )
+
+        # Clustering Layer
+        self.cluster = nn.Sequential(
+            full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop),
+            nn.Linear(params.feat_hidden2, self.num_classes)
+        )
+
+    def encode(self, x, edge_index):
+        x = self.encoder(x)
+        x = self.gat1(x, edge_index)
+        x = F.relu(x)
+        x = F.dropout(x, p=self.params.p_drop, training=self.training)
+
+        mu = self.gat2(x, edge_index)
         if self.var:
-            logvar = self.gn3(hidden1, adj)
+            logvar = self.gat3(x, edge_index)
             return mu, logvar
         else:
             return mu, None
-     
+
     def reparameterize(self, mu, logvar):
         if self.training and logvar is not None:
-            std = torch.exp(logvar)
+            std = torch.exp(0.5 * logvar)
             eps = torch.randn_like(std)
-            return eps.mul(std).add_(mu)
+            return eps * std + mu
         else:
             return mu
-    
-    def forward(self, x, adj):
-        mu, logvar = self.encode(x, adj)
-        gnn_z = self.reparameterize(mu, logvar)
-        x_reconstructed = self.decoder(gnn_z)
-        pred_label = F.softmax(self.cluster(gnn_z),dim=1)
-        return pred_label, x_reconstructed, gnn_z, mu, logvar
+
+    def forward(self, x, edge_index):
+        mu, logvar = self.encode(x, edge_index)
+        z = self.reparameterize(mu, logvar)
+        x_reconstructed = self.decoder(z)
+        pred_label = F.softmax(self.cluster(z), dim=1)
+        return pred_label, x_reconstructed, z, mu, logvar

gsMap/GNN_VAE/train.py

@@ -1,97 +1,86 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Jul  4 19:58:58 2023
-
-@author: songliyang
-"""
+import logging
 import time
 
 import torch
+import torch.nn.functional as F
 from progress.bar import Bar
 
-from gsMap.GNN_VAE.model import GNN_VAE_Model
+from gsMap.GNN_VAE.model import GATModel
+
+logger = logging.getLogger(__name__)
 
 
 def reconstruction_loss(decoded, x):
-    loss_fn = torch.nn.MSELoss()
-    loss = loss_fn(decoded, x)
-    return loss
+    """Compute the mean squared error loss."""
+    return F.mse_loss(decoded, x)
 
 
 def label_loss(pred_label, true_label):
-    loss_fn = torch.nn.CrossEntropyLoss()
-    loss = loss_fn(pred_label, true_label)
-    return loss
-
-
-class Model_Train:
-    def __init__(self, node_X, graph_dict, params, label=None):
-        device = 'cuda' if torch.cuda.is_available() else 'cpu'
-        torch.cuda.empty_cache()
+    """Compute the cross-entropy loss."""
+    return F.cross_entropy(pred_label, true_label)
 
+class ModelTrainer:
+    def __init__(self, node_x, graph_dict, params, label=None):
+        """Initialize the ModelTrainer with data and hyperparameters."""
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
         self.params = params
-        self.device = device
         self.epochs = params.epochs
-        self.node_X = torch.FloatTensor(node_X.copy()).to(device)
-        self.adj_norm = graph_dict["adj_norm"].to(device).coalesce()
+        self.node_x = torch.FloatTensor(node_x).to(self.device)
+        self.adj_norm = graph_dict["adj_norm"].to(self.device).coalesce()
         self.label = label
         self.num_classes = 1
-        
-        if not self.label is None:
+
+        if self.label is not None:
             self.label = torch.tensor(self.label).to(self.device)
-            self.num_classes = len(self.label.unique())
-        
-        # Set Model 
-        self.model = GNN_VAE_Model(self.params.feat_cell,self.params,self.num_classes).to(device)
-        self.optimizer = torch.optim.Adam(params = list(self.model.parameters()),
-                                          lr = self.params.gat_lr, weight_decay = self.params.gcn_decay)               
-    
-    # Train    
+            self.num_classes = len(torch.unique(self.label))
+
+        # Set up the model
+        self.model = GATModel(self.params.feat_cell, self.params, self.num_classes).to(self.device)
+        self.optimizer = torch.optim.Adam(
+            self.model.parameters(),
+            lr=self.params.gat_lr,
+            weight_decay=self.params.gcn_decay
+        )
+
     def run_train(self):
+        """Train the model."""
         self.model.train()
         prev_loss = float('inf')
-        
-        bar = Bar('GAT-AE model train:', max = self.epochs)
-        bar.check_tty = False 
+        bar = Bar('GAT-AE model train:', max=self.epochs)
+        bar.check_tty = False
+
+        logger.info('Start training...')
         for epoch in range(self.epochs):
             start_time = time.time()
-            self.model.train()
             self.optimizer.zero_grad()
-            pred_label, de_feat, latent_z, mu, logvar = self.model(self.node_X, self.adj_norm)
-            loss_rec = reconstruction_loss(de_feat, self.node_X)
-            
-            # Check whether annotation was provided
-            if not self.label is None:
+            pred_label, de_feat, latent_z, mu, logvar = self.model(self.node_x, self.adj_norm)
+            loss_rec = reconstruction_loss(de_feat, self.node_x)
+
+            if self.label is not None:
                 loss_pre = label_loss(pred_label, self.label)
-                loss = (self.params.rec_w * loss_rec) + (self.params.label_w * loss_pre)
+                loss = self.params.rec_w * loss_rec + self.params.label_w * loss_pre
             else:
                 loss = loss_rec
-                
+
             loss.backward()
             self.optimizer.step()
-            
-            # Update process
-            end_time = time.time()
-            batch_time = end_time - start_time
-            
-            
-            bar_str = '{} / {} | Left time: {batch_time:.2f} mins| Loss: {loss:.4f}'
-            bar.suffix = bar_str.format(epoch + 1,self.epochs,
-                                        batch_time = batch_time * (self.epochs - epoch) / 60, loss=loss.item())
+
+            batch_time = time.time() - start_time
+            left_time = batch_time * (self.epochs - epoch - 1) / 60  # in minutes
+
+            bar.suffix = f'{epoch + 1} / {self.epochs} | Left time: {left_time:.2f} mins | Loss: {loss.item():.4f}'
             bar.next()
-            
-            # Check convergence
+
             if abs(loss.item() - prev_loss) <= self.params.convergence_threshold and epoch >= 200:
-                print('\nConvergence reached. Training stopped.')
+                logger.info('\nConvergence reached. Training stopped.')
                 break
 
             prev_loss = loss.item()
-            
         bar.finish()
-    #-    
+
     def get_latent(self):
+        """Retrieve the latent representation from the model."""
         self.model.eval()
-        pred, de_fea, latent_z, mu, logvar = self.model(self.node_X, self.adj_norm)
-        latent_z = latent_z.data.cpu().numpy()
-        return latent_z
+        with torch.no_grad():
+            _, _, latent_z, _, _ = self.model(self.node_x, self.adj_norm)
+        return latent_z.cpu().numpy()

gsMap/__init__.py

@@ -2,4 +2,4 @@
 Genetics-informed pathogenic spatial mapping
 '''
 
-__version__ = '1.66'
+__version__ = '1.67'

gsMap/config.py

@@ -55,7 +55,8 @@ def add_find_latent_representations_args(parser):
     add_shared_args(parser)
     parser.add_argument('--input_hdf5_path', required=True, type=str, help='Path to the input HDF5 file.')
     parser.add_argument('--annotation', required=True, type=str, help='Name of the annotation in adata.obs to use.')
-    parser.add_argument('--data_layer', required=True, type=str, help='Data layer for gene expression (e.g., "counts", "log1p").')
+    parser.add_argument('--data_layer', type=str, default='counts', required=True,
+                        help='Data layer for gene expression (e.g., "count", "counts", "log1p").')
     parser.add_argument('--epochs', type=int, default=300, help='Number of training epochs.')
     parser.add_argument('--feat_hidden1', type=int, default=256, help='Neurons in the first hidden layer.')
     parser.add_argument('--feat_hidden2', type=int, default=128, help='Neurons in the second hidden layer.')
@@ -66,7 +67,6 @@ def add_find_latent_representations_args(parser):
     parser.add_argument('--n_neighbors', type=int, default=11, help='Number of neighbors for GAT.')
     parser.add_argument('--n_comps', type=int, default=300, help='Number of principal components for PCA.')
     parser.add_argument('--weighted_adj', action='store_true', help='Use weighted adjacency in GAT.')
-    parser.add_argument('--var', action='store_true', help='Enable variance calculations.')
     parser.add_argument('--convergence_threshold', type=float, default=1e-4, help='Threshold for convergence.')
     parser.add_argument('--hierarchically', action='store_true', help='Enable hierarchical latent representation finding.')
 
@@ -236,8 +236,8 @@ def add_run_all_mode_args(parser):
                         help='Path to the input spatial transcriptomics data (H5AD format).')
     parser.add_argument('--annotation', type=str, required=True,
                         help='Name of the annotation in adata.obs to use.')
-    parser.add_argument('--data_layer', type=str, default='X',
-                        help='Data layer of h5ad for gene expression (e.g., "counts", "log1p", "X").')
+    parser.add_argument('--data_layer', type=str, default='counts', required=True,
+                        help='Data layer for gene expression (e.g., "count", "counts", "log1p").')
 
     # GWAS Data Parameters
     parser.add_argument('--trait_name', type=str, help='Name of the trait for GWAS analysis (required if sumstats_file is provided).')

gsMap/find_latent_representation.py

@@ -1,145 +1,145 @@
 import logging
 import random
-
 import numpy as np
-import pandas as pd
 import scanpy as sc
 import torch
-from sklearn import preprocessing
-
-from gsMap.GNN_VAE.adjacency_matrix import Construct_Adjacency_Matrix
-from gsMap.GNN_VAE.train import Model_Train
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import LabelEncoder
+from gsMap.GNN_VAE.adjacency_matrix import construct_adjacency_matrix
+from gsMap.GNN_VAE.train import ModelTrainer
 from gsMap.config import FindLatentRepresentationsConfig
 
 logger = logging.getLogger(__name__)
 
+
 def set_seed(seed_value):
     """
-    Set seed for reproducibility in PyTorch.
+    Set seed for reproducibility in PyTorch and other libraries.
     """
-    torch.manual_seed(seed_value)  # Set the seed for PyTorch
-    np.random.seed(seed_value)  # Set the seed for NumPy
-    random.seed(seed_value)  # Set the seed for Python random module
+    torch.manual_seed(seed_value)
+    np.random.seed(seed_value)
+    random.seed(seed_value)
     if torch.cuda.is_available():
-        logger.info('Running use GPU')
-        torch.cuda.manual_seed(seed_value)  # Set seed for all CUDA devices
-        torch.cuda.manual_seed_all(seed_value)  # Set seed for all CUDA devices
+        logger.info('Using GPU for computations.')
+        torch.cuda.manual_seed(seed_value)
+        torch.cuda.manual_seed_all(seed_value)
     else:
-        logger.info('Running use CPU')
+        logger.info('Using CPU for computations.')
 
+def preprocess_data(adata, params):
+    """
+    Preprocess the AnnData
+    """
+    logger.info('Preprocessing data...')
+    adata.var_names_make_unique()
+
+    sc.pp.filter_genes(adata, min_cells=30)
+    if params.data_layer in adata.layers.keys():
+        adata.X = adata.layers[params.data_layer]
+    else:
+        raise ValueError(f'Invalid data layer: {params.data_layer}, please check the input data.')
 
-# The class for finding latent representations
-class Latent_Representation_Finder:
+    if params.data_layer in ['count', 'counts']:
 
-    def __init__(self, adata, args:FindLatentRepresentationsConfig):
-        self.adata = adata.copy()
-        self.Params = args
+        sc.pp.normalize_total(adata, target_sum=1e4)
+        sc.pp.log1p(adata)
 
-        # Standard process
-        if self.Params.data_layer == 'count' or self.Params.data_layer == 'counts':
-            self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, flavor="seurat_v3", n_top_genes=self.Params.feat_cell)
-            sc.pp.normalize_total(self.adata, target_sum=1e4)
-            sc.pp.log1p(self.adata)
-            sc.pp.scale(self.adata)
-        else:
-            if self.Params.data_layer != 'X':
-                self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, n_top_genes=self.Params.feat_cell)
+        # Identify highly variable genes
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat_v3",
+            n_top_genes=params.feat_cell,
+        )
 
-    def Run_GNN_VAE(self, label, verbose='whole ST data'):
+    elif params.data_layer in adata.layers.keys():
+        logger.info(f'Using {params.data_layer} data...')
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat",
+            n_top_genes=params.feat_cell,
+        )
 
-        # Construct the neighbouring graph
-        graph_dict = Construct_Adjacency_Matrix(self.adata, self.Params)
+    return adata
 
-        # Process the feature matrix
-        node_X = self.adata[:, self.adata.var.highly_variable].X
-        logger.info(f'The shape of feature matrix is {node_X.shape}.')
-        if self.Params.input_pca:
-            node_X = sc.pp.pca(node_X, n_comps=self.Params.n_comps)
+
+class LatentRepresentationFinder:
+    def __init__(self, adata, args: FindLatentRepresentationsConfig):
+        self.params = args
+
+        self.expression_array = adata[:, adata.var.highly_variable].X.copy()
+
+        if self.params.data_layer in ['count', 'counts']:
+            self.expression_array = sc.pp.scale(self.expression_array, max_value=10)
+
+        # Construct the neighboring graph
+        self.graph_dict = construct_adjacency_matrix(adata, self.params)
+
+    def compute_pca(self):
+        self.latent_pca = PCA(n_components=self.params.feat_cell).fit_transform(self.expression_array)
+        return self.latent_pca
+
+    def run_gnn_vae(self, label, verbose='whole ST data'):
+
+        # Use PCA if specified
+        if self.params.input_pca:
+            node_X = self.compute_pca()
+        else:
+            node_X = self.expression_array
 
         # Update the input shape
-        self.Params.n_nodes = node_X.shape[0]
-        self.Params.feat_cell = node_X.shape[1]
+        self.params.n_nodes = node_X.shape[0]
+        self.params.feat_cell = node_X.shape[1]
 
-        # Run GNN-VAE
-        logger.info(f'------Finding latent representations for {verbose}...')
-        gvae = Model_Train(node_X, graph_dict, self.Params, label)
+        # Run GNN
+        logger.info(f'Finding latent representations for {verbose}...')
+        gvae = ModelTrainer(node_X, self.graph_dict, self.params, label)
         gvae.run_train()
 
-        return gvae.get_latent()
+        del self.graph_dict
 
-    def Run_PCA(self):
-        sc.tl.pca(self.adata)
-        return self.adata.obsm['X_pca'][:, 0:self.Params.n_comps]
+        return gvae.get_latent()
 
 
-def run_find_latent_representation(args:FindLatentRepresentationsConfig):
+def run_find_latent_representation(args: FindLatentRepresentationsConfig):
     set_seed(2024)
-    num_features = args.feat_cell
-    args.hdf5_with_latent_path.parent.mkdir(parents=True, exist_ok=True,mode=0o755)
+
     # Load the ST data
-    logger.info(f'------Loading ST data of {args.sample_name}...')
-    adata = sc.read_h5ad(f'{args.input_hdf5_path}')
-    adata.var_names_make_unique()
-    adata.X = adata.layers[args.data_layer] if args.data_layer in adata.layers.keys() else adata.X
-    logger.info('The ST data contains %d cells, %d genes.' % (adata.shape[0], adata.shape[1]))
+    logger.info(f'Loading ST data of {args.sample_name}...')
+    adata = sc.read_h5ad(args.input_hdf5_path)
+    logger.info(f'The ST data contains {adata.shape[0]} cells, {adata.shape[1]} genes.')
+
     # Load the cell type annotation
-    if not args.annotation is None:
-        # remove cells without enough annotations
-        adata = adata[~pd.isnull(adata.obs[args.annotation]), :]
+    if args.annotation is not None:
+        # Remove cells without enough annotations
+        adata = adata[~adata.obs[args.annotation].isnull()]
         num = adata.obs[args.annotation].value_counts()
-        adata = adata[adata.obs[args.annotation].isin(num[num >= 30].index.to_list())]
+        valid_annotations = num[num >= 30].index.to_list()
+        adata = adata[adata.obs[args.annotation].isin(valid_annotations)]
 
-        le = preprocessing.LabelEncoder()
-        le.fit(adata.obs[args.annotation])
-        adata.obs['categorical_label'] = le.transform(adata.obs[args.annotation])
-        label = adata.obs['categorical_label'].to_list()
+        le = LabelEncoder()
+        adata.obs['categorical_label'] = le.fit_transform(adata.obs[args.annotation])
+        label = adata.obs['categorical_label'].to_numpy()
     else:
         label = None
-    # Find latent representations
-    latent_rep = Latent_Representation_Finder(adata, args)
-    latent_GVAE = latent_rep.Run_GNN_VAE(label)
-    latent_PCA = latent_rep.Run_PCA()
-    # Add latent representations to the spe data
-    logger.info(f'------Adding latent representations...')
-    adata.obsm["latent_GVAE"] = latent_GVAE
-    adata.obsm["latent_PCA"] = latent_PCA
-    # Run umap based on latent representations
-    for name in ['latent_GVAE', 'latent_PCA']:
-        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
-        sc.tl.umap(adata)
-        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
 
-        # Find the latent representations hierarchically (optionally)
-    if not args.annotation is None and args.hierarchically:
-        logger.info(f'------Finding latent representations hierarchically...')
-        PCA_all = pd.DataFrame()
-        GVAE_all = pd.DataFrame()
+    # Preprocess data
+    adata = preprocess_data(adata, args)
 
-        for ct in adata.obs[args.annotation].unique():
-            adata_part = adata[adata.obs[args.annotation] == ct, :]
-            logger.info(adata_part.shape)
+    latent_rep = LatentRepresentationFinder(adata, args)
+    latent_gvae = latent_rep.run_gnn_vae(label)
+    latent_pca = latent_rep.compute_pca()
 
-            # Find latent representations for the selected ct
-            latent_rep = Latent_Representation_Finder(adata_part, args)
+    # Add latent representations to the AnnData object
+    logger.info('Adding latent representations...')
+    adata.obsm["latent_GVAE"] = latent_gvae
+    adata.obsm["latent_PCA"] = latent_pca
 
-            latent_PCA_part = pd.DataFrame(latent_rep.Run_PCA())
-            if adata_part.shape[0] <= args.n_comps:
-                latent_GVAE_part = latent_PCA_part
-            else:
-                latent_GVAE_part = pd.DataFrame(latent_rep.Run_GNN_VAE(label=None, verbose=ct))
-
-            latent_GVAE_part.index = adata_part.obs_names
-            latent_PCA_part.index = adata_part.obs_names
-
-            GVAE_all = pd.concat((GVAE_all, latent_GVAE_part), axis=0)
-            PCA_all = pd.concat((PCA_all, latent_PCA_part), axis=0)
-
-            args.feat_cell = num_features
+    # Run UMAP based on latent representations
+    for name in ['latent_GVAE', 'latent_PCA']:
+        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
+        sc.tl.umap(adata)
+        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
 
-            adata.obsm["latent_GVAE_hierarchy"] = np.array(GVAE_all.loc[adata.obs_names,])
-            adata.obsm["latent_PCA_hierarchy"] = np.array(PCA_all.loc[adata.obs_names,])
-    logger.info(f'------Saving ST data...')
+    # Save the AnnData object
+    logger.info('Saving ST data...')
     adata.write(args.hdf5_with_latent_path)
-

gsMap/format_sumstats.py

@@ -150,10 +150,10 @@ def gwas_checkname(gwas, config):
         'Pos': 'SNP positions.'
     }
 
-    print(f'\nIterpreting column names as follows:')
+    logger.info(f'\nIterpreting column names as follows:')
     for key, value in interpreting.items():
         if key in new_name:
-            print(f'{name_dict[key]}: {interpreting[key]}')
+            logger.info(f'{name_dict[key]}: {interpreting[key]}')
 
     return gwas
 
@@ -242,7 +242,7 @@ def gwas_qc(gwas, config):
     Filter out SNPs based on INFO, FRQ, MAF, N, and Genotypes. 
     '''
     old = len(gwas)
-    print(f'\nFiltering SNPs as follows:')
+    logger.info(f'\nFiltering SNPs as follows:')
     # filter: SNPs with missing values
     drops = {'NA': 0, 'P': 0, 'INFO': 0, 'FRQ': 0, 'A': 0, 'SNP': 0, 'Dup': 0, 'N': 0}
 
@@ -250,28 +250,28 @@ def gwas_qc(gwas, config):
         lambda x: x != 'INFO', gwas.columns)).reset_index(drop=True)
 
     drops['NA'] = old - len(gwas)
-    print(f'Removed {drops["NA"]} SNPs with missing values.')
+    logger.info(f'Removed {drops["NA"]} SNPs with missing values.')
 
     # filter: SNPs with Info < 0.9
     if 'INFO' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_info(gwas['INFO'], config)]
         drops['INFO'] = old - len(gwas)
-        print(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
+        logger.info(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
 
     # filter: SNPs with MAF <= 0.01
     if 'FRQ' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_frq(gwas['FRQ'], config)]
         drops['FRQ'] += old - len(gwas)
-        print(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
+        logger.info(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
 
     # filter: P-value that out-of-bounds [0,1]
     if 'P' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_pvals(gwas['P'], config)]
         drops['P'] += old - len(gwas)
-        print(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
+        logger.info(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
 
     # filter: Variants that are strand-ambiguous
     if 'A1' in gwas.columns and 'A2' in gwas.columns:
@@ -279,21 +279,21 @@ def gwas_qc(gwas, config):
         gwas.A2 = gwas.A2.str.upper()
         gwas = gwas.loc[filter_alleles(gwas.A1 + gwas.A2)]
         drops['A'] += old - len(gwas)
-        print(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
+        logger.info(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
 
     # filter: Duplicated rs numbers
     if 'SNP' in gwas.columns:
         old = len(gwas)
         gwas = gwas.drop_duplicates(subset='SNP').reset_index(drop=True)
         drops['Dup'] += old - len(gwas)
-        print(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
+        logger.info(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
 
     # filter:Sample size
     n_min = gwas.N.quantile(0.9) / 1.5
     old = len(gwas)
     gwas = gwas[gwas.N >= n_min].reset_index(drop=True)
     drops['N'] += old - len(gwas)
-    print(f'Removed {drops["N"]} SNPs with N < {n_min}.')
+    logger.info(f'Removed {drops["N"]} SNPs with N < {n_min}.')
 
     return gwas
 
@@ -302,7 +302,7 @@ def variant_to_rsid(gwas, config):
     '''
     Convert variant id (Chr, Pos) to rsid
     '''
-    print("\nConverting the SNP position to rsid. This process may take some time.")
+    logger.info("\nConverting the SNP position to rsid. This process may take some time.")
     unique_ids = set(gwas['id'])
     chr_format = gwas['Chr'].unique().astype(str)
     chr_format = [re.sub(r'\d+', '', value) for value in chr_format][1]
@@ -347,7 +347,7 @@ def clean_SNP_id(gwas, config):
             gwas = gwas.loc[matching_id.id]
             gwas['SNP'] = matching_id.dbsnp
             num_fail = old - len(gwas)
-            print(f'Removed {num_fail} SNPs that did not convert to rsid.')
+            logger.info(f'Removed {num_fail} SNPs that did not convert to rsid.')
 
     return gwas
 
@@ -356,27 +356,27 @@ def gwas_metadata(gwas, config):
     '''
     Report key features of GWAS data
     '''
-    print('\nMetadata:')
+    logger.info('\nSummary of GWAS data:')
     CHISQ = (gwas.Z ** 2)
     mean_chisq = CHISQ.mean()
-    print('Mean chi^2 = ' + str(round(mean_chisq, 3)))
+    logger.info('Mean chi^2 = ' + str(round(mean_chisq, 3)))
     if mean_chisq < 1.02:
         logger.warning("Mean chi^2 may be too small.")
 
-    print('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
-    print('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
-    print('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
+    logger.info('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
+    logger.info('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
+    logger.info('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
 
 
 def gwas_format(config: FormatSumstatsConfig):
     '''
     Format GWAS data
     '''
-    print(f'------Formating gwas data for {config.sumstats}...')
+    logger.info(f'------Formating gwas data for {config.sumstats}...')
     compression_type = get_compression(config.sumstats)
     gwas = pd.read_csv(config.sumstats, delim_whitespace=True, header=0, compression=compression_type,
                        na_values=['.', 'NA'])
-    print(f'Read {len(gwas)} SNPs from {config.sumstats}.')
+    logger.info(f'Read {len(gwas)} SNPs from {config.sumstats}.')
 
     # Check name and format
     gwas = gwas_checkname(gwas, config)
@@ -402,6 +402,6 @@ def gwas_format(config: FormatSumstatsConfig):
     gwas = gwas[keep]
     out_name = config.out + appendix + '.gz'
 
-    print(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
+    logger.info(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
     gwas.to_csv(out_name, sep="\t", index=False,
                 float_format='%.3f', compression='gzip')

gsMap/latent_to_gene.py

@@ -4,12 +4,10 @@ from pathlib import Path
 import numpy as np
 import pandas as pd
 import scanpy as sc
-from scipy.sparse import csr_matrix
 from scipy.stats import gmean
 from scipy.stats import rankdata
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.neighbors import NearestNeighbors
-from joblib import Parallel, delayed
 from tqdm import tqdm
 
 from gsMap.config import LatentToGeneConfig
@@ -152,11 +150,6 @@ def run_latent_to_gene(config: LatentToGeneConfig):
         adata.var_names = homologs.loc[adata.var_names, 'HUMAN_GENE_SYM'].values
         adata = adata[:, ~adata.var_names.duplicated()]
 
-    # Remove cells and genes that are not expressed
-    logger.info(f'Number of cells, genes of the input data: {adata.shape[0]},{adata.shape[1]}')
-    adata = adata[adata.X.sum(axis=1) > 0, adata.X.sum(axis=0) > 0]
-    logger.info(f'Number of cells, genes after transformation: {adata.shape[0]},{adata.shape[1]}')
-
     # Create mappings
     n_cells = adata.n_obs
     n_genes = adata.n_vars

gsMap/spatial_ldsc_multiple_sumstats.py

@@ -20,8 +20,6 @@ logger = logging.getLogger('gsMap.spatial_ldsc')
 
 # %%
 def _coef_new(jknife):
-    # return coef[0], coef_se[0], z[0]]
-    # est_ = jknife.est[0, 0] / Nbar
     est_ = jknife.jknife_est[0, 0] / Nbar
     se_ = jknife.jknife_se[0, 0] / Nbar
     return est_, se_

{gsmap-1.66.dist-info → gsmap-1.67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gsMap
-Version: 1.66
+Version: 1.67
 Summary: Genetics-informed pathogenic spatial mapping
 Author-email: liyang <songliyang@westlake.edu.cn>, wenhao <chenwenhao@westlake.edu.cn>
 Requires-Python: >=3.8
@@ -27,7 +27,7 @@ Requires-Dist: pyfiglet
 Requires-Dist: plotly
 Requires-Dist: kaleido
 Requires-Dist: jinja2
-Requires-Dist: scanpy
+Requires-Dist: scanpy >=1.8.0
 Requires-Dist: zarr
 Requires-Dist: bitarray
 Requires-Dist: pyarrow

{gsmap-1.66.dist-info → gsmap-1.67.dist-info}/RECORD

@@ -1,22 +1,22 @@
-gsMap/__init__.py,sha256=eQ-mfdcGTJtKS2KIu5PEQMqgx_9j9W5KKTBVr-iI4yo,78
+gsMap/__init__.py,sha256=dO26Gwp-UoxhNp0ZHWuBVMGsjg6EVy09d93LGJ7pjqA,78
 gsMap/__main__.py,sha256=jR-HT42Zzfj2f-7kFJy0bkWjNxcV1MyfQHXFpef2nSE,62
 gsMap/cauchy_combination_test.py,sha256=zBPR7DOaNkr7rRoua4tAjRZL7ArjCyMRSQlPSUdHNSE,5694
-gsMap/config.py,sha256=hMUvlwlKZXeRdTJZfMINz_8DadVhEIT6X6fyJf11M9E,41134
+gsMap/config.py,sha256=xHSIAdt5_4Vdr_2CA-YR1Wbn7i0FW70Eokd0cbi4hqU,41109
 gsMap/diagnosis.py,sha256=pp3ONVaWCOoNCog1_6eud38yicBFxL-XhH7D8iTBgF4,13220
-gsMap/find_latent_representation.py,sha256=BVv4dyTolrlciHG3I-vwNDh2ruPpTf9jiT1hMKZnpto,6044
-gsMap/format_sumstats.py,sha256=9OBxuunoOLml3LKZvvRsPEEjQvT1Cuqb0w6lqsRIYPw,13714
+gsMap/find_latent_representation.py,sha256=SOGSEHyi7XAPJnTUIRWHv-fCXHR3K9UYcyJI_nuPHFY,4834
+gsMap/format_sumstats.py,sha256=00VLNbfjXRqIgFn44WM-mMIboLTYTRn_3JiEtqJDfeQ,13846
 gsMap/generate_ldscore.py,sha256=2JfQoMWeQ0-B-zRHakmwq8ovkeewlnWHUCnih6od6ZE,29089
-gsMap/latent_to_gene.py,sha256=MwoGQd0EDvDmvpuMoVD83SI1EeGJXXzMW8YZp_6wxI8,10082
+gsMap/latent_to_gene.py,sha256=oRYtRcq1TLXpzkoP_l93CwPDPc6LZvw6_MGrdPJhC_o,9686
 gsMap/main.py,sha256=skyBtESdjvuXd9HNq5c83OPxQTNgLVErkYhwuJm8tE4,1285
 gsMap/report.py,sha256=H0uYAru2L5-d41_LFHPPdoJbtiTzP4f8kX-mirUNAfc,6963
 gsMap/run_all_mode.py,sha256=sPEct9fRw7aAQuU7BNChxk-I8YQcXuq--mtBn-2wTTY,8388
 gsMap/setup.py,sha256=eOoPalGAHTY06_7a35nvbOaKQhq0SBE5GK4pc4bp3wc,102
-gsMap/spatial_ldsc_multiple_sumstats.py,sha256=09j2zG98JUjQvSHPaRIDQVMZZLYDd1JBFaZTkW7tdvY,18470
+gsMap/spatial_ldsc_multiple_sumstats.py,sha256=qyKLeWz-_78WMjkpnMtoSJyJcDLnfp-armyY_mzlwkI,18391
 gsMap/visualize.py,sha256=FLIRHHj1KntLMFDjAhg1jZnJUdvrncR74pCW2Kj5pus,7453
 gsMap/GNN_VAE/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gsMap/GNN_VAE/adjacency_matrix.py,sha256=pmSfK9TTwdrsWTmHvCqVrbRE0PAiq1lvgmxzrdQgpiU,3500
-gsMap/GNN_VAE/model.py,sha256=Fixl8-2zN3T5MmMtpMIvxsBYydM3QVR4uC3Hhsg0DzI,3349
-gsMap/GNN_VAE/train.py,sha256=KnvZYHImRzTwJl1H0dkZZqWASdZ5VgYTCifQVW8TavM,3389
+gsMap/GNN_VAE/adjacency_matrix.py,sha256=TJzf5JGPDfC0-50xRP1Tdf7xqEPBLBl9E9VvQq5FE1g,3333
+gsMap/GNN_VAE/model.py,sha256=6c_zw_PTg9uBnRng9fNoU3HlbrjEP1j5cxFzwPJEW5s,2959
+gsMap/GNN_VAE/train.py,sha256=Etro9QKA5jU14uO17wI_gzV0Nl4YFubk9A--jpRrM4w,3095
 gsMap/templates/report_template.html,sha256=pdxHFl_W0W351NUzuJTf_Ay_BfKlEbD_fztNabAGmmg,8214
 gsMap/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gsMap/utils/generate_r2_matrix.py,sha256=A1BrUnlTrYjRwEKxK0I1FbZ5SCQzcviWVM-JzFHHRkw,29352
@@ -24,8 +24,8 @@ gsMap/utils/jackknife.py,sha256=nEDPVQJOPQ_uqfUCGX_v5cQwokgCqUmJTT_8rVFuIQo,1824
 gsMap/utils/make_annotations.py,sha256=lCbtahT27WFOwLgZrEUE5QcNRuMXmAFYUfsFR-cT-m0,22197
 gsMap/utils/manhattan_plot.py,sha256=k3n-NNgMsov9-8UQrirVqG560FUfJ4d6wNG8C0OeCjY,26235
 gsMap/utils/regression_read.py,sha256=n_hZZzQXHU-CSLvSofXmQM5Jw4Zpufv3U2HoUW344ko,8768
-gsmap-1.66.dist-info/entry_points.txt,sha256=s_P2Za22O077tc1FPLKMinbdRVXaN_HTcDBgWMYpqA4,41
-gsmap-1.66.dist-info/LICENSE,sha256=Ni2F-lLSv_H1xaVT3CoSrkiKzMvlgwh-dq8PE1esGyI,1094
-gsmap-1.66.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-gsmap-1.66.dist-info/METADATA,sha256=HXeRNmaP_UPzG2Qjn5s-jcLBvrfLgPYl7qVGDAKJG5Y,3376
-gsmap-1.66.dist-info/RECORD,,
+gsmap-1.67.dist-info/entry_points.txt,sha256=s_P2Za22O077tc1FPLKMinbdRVXaN_HTcDBgWMYpqA4,41
+gsmap-1.67.dist-info/LICENSE,sha256=Ni2F-lLSv_H1xaVT3CoSrkiKzMvlgwh-dq8PE1esGyI,1094
+gsmap-1.67.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+gsmap-1.67.dist-info/METADATA,sha256=YgGfS9s8cnWLy7fWpptkcMUKsMqcFdMod_V_RlMp0dM,3384
+gsmap-1.67.dist-info/RECORD,,