gsMap 1.66__tar.gz → 1.67__tar.gz

{gsmap-1.66 → gsmap-1.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gsMap
-Version: 1.66
+Version: 1.67
 Summary: Genetics-informed pathogenic spatial mapping
 Author-email: liyang <songliyang@westlake.edu.cn>, wenhao <chenwenhao@westlake.edu.cn>
 Requires-Python: >=3.8
@@ -27,7 +27,7 @@ Requires-Dist: pyfiglet
 Requires-Dist: plotly
 Requires-Dist: kaleido
 Requires-Dist: jinja2
-Requires-Dist: scanpy
+Requires-Dist: scanpy >=1.8.0
 Requires-Dist: zarr
 Requires-Dist: bitarray
 Requires-Dist: pyarrow

{gsmap-1.66 → gsmap-1.67}/docs/source/mouse_example.md

@@ -209,7 +209,7 @@ gsmap run_cauchy_combination \
     --annotation 'annotation'
 ```
 
-### 6. report generation
+### 6. report generation (optional)
 
 **Objective**: Generate gsMap reports, including visualizations of mapping results and diagnostic plots.
 

{gsmap-1.66 → gsmap-1.67}/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     'plotly',
     'kaleido',
     'jinja2',
-    'scanpy',
+    'scanpy >=1.8.0',
     'zarr',
     'bitarray',
     'pyarrow',

gsmap-1.67/src/gsMap/GNN_VAE/adjacency_matrix.py

@@ -0,0 +1,75 @@
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+from sklearn.neighbors import NearestNeighbors
+import torch
+
+def cal_spatial_net(adata, n_neighbors=5, verbose=True):
+    """Construct the spatial neighbor network."""
+    if verbose:
+        print('------Calculating spatial graph...')
+    coor = pd.DataFrame(adata.obsm['spatial'], index=adata.obs.index)
+    nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(coor)
+    distances, indices = nbrs.kneighbors(coor)
+    n_cells, n_neighbors = indices.shape
+    cell_indices = np.arange(n_cells)
+    cell1 = np.repeat(cell_indices, n_neighbors)
+    cell2 = indices.flatten()
+    distance = distances.flatten()
+    knn_df = pd.DataFrame({'Cell1': cell1, 'Cell2': cell2, 'Distance': distance})
+    knn_df = knn_df[knn_df['Distance'] > 0].copy()
+    cell_id_map = dict(zip(cell_indices, coor.index))
+    knn_df['Cell1'] = knn_df['Cell1'].map(cell_id_map)
+    knn_df['Cell2'] = knn_df['Cell2'].map(cell_id_map)
+    return knn_df
+
+def sparse_mx_to_torch_sparse_tensor(sparse_mx):
+    """Convert a scipy sparse matrix to a torch sparse tensor."""
+    sparse_mx = sparse_mx.tocoo().astype(np.float32)
+    indices = torch.from_numpy(
+        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64)
+    )
+    values = torch.from_numpy(sparse_mx.data)
+    shape = torch.Size(sparse_mx.shape)
+    return torch.sparse.FloatTensor(indices, values, shape)
+
+def preprocess_graph(adj):
+    """Symmetrically normalize the adjacency matrix."""
+    adj = sp.coo_matrix(adj)
+    adj_ = adj + sp.eye(adj.shape[0])
+    rowsum = np.array(adj_.sum(1)).flatten()
+    degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5))
+    adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo()
+    return sparse_mx_to_torch_sparse_tensor(adj_normalized)
+
+def construct_adjacency_matrix(adata, params, verbose=True):
+    """Construct the adjacency matrix from spatial data."""
+    spatial_net = cal_spatial_net(adata, n_neighbors=params.n_neighbors, verbose=verbose)
+    if verbose:
+        num_edges = spatial_net.shape[0]
+        num_cells = adata.n_obs
+        print(f'The graph contains {num_edges} edges, {num_cells} cells.')
+        print(f'{num_edges / num_cells:.2f} neighbors per cell on average.')
+    cell_ids = {cell: idx for idx, cell in enumerate(adata.obs.index)}
+    spatial_net['Cell1'] = spatial_net['Cell1'].map(cell_ids)
+    spatial_net['Cell2'] = spatial_net['Cell2'].map(cell_ids)
+    if params.weighted_adj:
+        distance_normalized = spatial_net['Distance'] / (spatial_net['Distance'].max() + 1)
+        weights = np.exp(-0.5 * distance_normalized ** 2)
+        adj_org = sp.coo_matrix(
+            (weights, (spatial_net['Cell1'], spatial_net['Cell2'])),
+            shape=(adata.n_obs, adata.n_obs)
+        )
+    else:
+        adj_org = sp.coo_matrix(
+            (np.ones(spatial_net.shape[0]), (spatial_net['Cell1'], spatial_net['Cell2'])),
+            shape=(adata.n_obs, adata.n_obs)
+        )
+    adj_norm = preprocess_graph(adj_org)
+    norm_value = adj_org.shape[0] ** 2 / ((adj_org.shape[0] ** 2 - adj_org.sum()) * 2)
+    graph_dict = {
+        "adj_org": adj_org,
+        "adj_norm": adj_norm,
+        "norm_value": norm_value
+    }
+    return graph_dict

gsmap-1.67/src/gsMap/GNN_VAE/model.py

@@ -0,0 +1,89 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch_geometric.nn import GATConv
+
+def full_block(in_features, out_features, p_drop):
+    return nn.Sequential(
+        nn.Linear(in_features, out_features),
+        nn.BatchNorm1d(out_features),
+        nn.ELU(),
+        nn.Dropout(p=p_drop)
+    )
+
+class GATModel(nn.Module):
+    def __init__(self, input_dim, params, num_classes=1):
+        super().__init__()
+        self.var = params.var
+        self.num_classes = num_classes
+        self.params = params
+
+        # Encoder
+        self.encoder = nn.Sequential(
+            full_block(input_dim, params.feat_hidden1, params.p_drop),
+            full_block(params.feat_hidden1, params.feat_hidden2, params.p_drop)
+        )
+
+        # GAT Layers
+        self.gat1 = GATConv(
+            in_channels=params.feat_hidden2,
+            out_channels=params.gat_hidden1,
+            heads=params.nheads,
+            dropout=params.p_drop
+        )
+        self.gat2 = GATConv(
+            in_channels=params.gat_hidden1 * params.nheads,
+            out_channels=params.gat_hidden2,
+            heads=1,
+            concat=False,
+            dropout=params.p_drop
+        )
+        if self.var:
+            self.gat3 = GATConv(
+                in_channels=params.gat_hidden1 * params.nheads,
+                out_channels=params.gat_hidden2,
+                heads=1,
+                concat=False,
+                dropout=params.p_drop
+            )
+
+        # Decoder
+        self.decoder = nn.Sequential(
+            full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop),
+            full_block(params.feat_hidden2, params.feat_hidden1, params.p_drop),
+            nn.Linear(params.feat_hidden1, input_dim)
+        )
+
+        # Clustering Layer
+        self.cluster = nn.Sequential(
+            full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop),
+            nn.Linear(params.feat_hidden2, self.num_classes)
+        )
+
+    def encode(self, x, edge_index):
+        x = self.encoder(x)
+        x = self.gat1(x, edge_index)
+        x = F.relu(x)
+        x = F.dropout(x, p=self.params.p_drop, training=self.training)
+
+        mu = self.gat2(x, edge_index)
+        if self.var:
+            logvar = self.gat3(x, edge_index)
+            return mu, logvar
+        else:
+            return mu, None
+
+    def reparameterize(self, mu, logvar):
+        if self.training and logvar is not None:
+            std = torch.exp(0.5 * logvar)
+            eps = torch.randn_like(std)
+            return eps * std + mu
+        else:
+            return mu
+
+    def forward(self, x, edge_index):
+        mu, logvar = self.encode(x, edge_index)
+        z = self.reparameterize(mu, logvar)
+        x_reconstructed = self.decoder(z)
+        pred_label = F.softmax(self.cluster(z), dim=1)
+        return pred_label, x_reconstructed, z, mu, logvar

gsmap-1.67/src/gsMap/GNN_VAE/train.py

@@ -0,0 +1,86 @@
+import logging
+import time
+
+import torch
+import torch.nn.functional as F
+from progress.bar import Bar
+
+from gsMap.GNN_VAE.model import GATModel
+
+logger = logging.getLogger(__name__)
+
+
+def reconstruction_loss(decoded, x):
+    """Compute the mean squared error loss."""
+    return F.mse_loss(decoded, x)
+
+
+def label_loss(pred_label, true_label):
+    """Compute the cross-entropy loss."""
+    return F.cross_entropy(pred_label, true_label)
+
+class ModelTrainer:
+    def __init__(self, node_x, graph_dict, params, label=None):
+        """Initialize the ModelTrainer with data and hyperparameters."""
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.params = params
+        self.epochs = params.epochs
+        self.node_x = torch.FloatTensor(node_x).to(self.device)
+        self.adj_norm = graph_dict["adj_norm"].to(self.device).coalesce()
+        self.label = label
+        self.num_classes = 1
+
+        if self.label is not None:
+            self.label = torch.tensor(self.label).to(self.device)
+            self.num_classes = len(torch.unique(self.label))
+
+        # Set up the model
+        self.model = GATModel(self.params.feat_cell, self.params, self.num_classes).to(self.device)
+        self.optimizer = torch.optim.Adam(
+            self.model.parameters(),
+            lr=self.params.gat_lr,
+            weight_decay=self.params.gcn_decay
+        )
+
+    def run_train(self):
+        """Train the model."""
+        self.model.train()
+        prev_loss = float('inf')
+        bar = Bar('GAT-AE model train:', max=self.epochs)
+        bar.check_tty = False
+
+        logger.info('Start training...')
+        for epoch in range(self.epochs):
+            start_time = time.time()
+            self.optimizer.zero_grad()
+            pred_label, de_feat, latent_z, mu, logvar = self.model(self.node_x, self.adj_norm)
+            loss_rec = reconstruction_loss(de_feat, self.node_x)
+
+            if self.label is not None:
+                loss_pre = label_loss(pred_label, self.label)
+                loss = self.params.rec_w * loss_rec + self.params.label_w * loss_pre
+            else:
+                loss = loss_rec
+
+            loss.backward()
+            self.optimizer.step()
+
+            batch_time = time.time() - start_time
+            left_time = batch_time * (self.epochs - epoch - 1) / 60  # in minutes
+
+            bar.suffix = f'{epoch + 1} / {self.epochs} | Left time: {left_time:.2f} mins | Loss: {loss.item():.4f}'
+            bar.next()
+
+            if abs(loss.item() - prev_loss) <= self.params.convergence_threshold and epoch >= 200:
+                logger.info('\nConvergence reached. Training stopped.')
+                break
+
+            prev_loss = loss.item()
+        bar.finish()
+
+    def get_latent(self):
+        """Retrieve the latent representation from the model."""
+        self.model.eval()
+        with torch.no_grad():
+            _, _, latent_z, _, _ = self.model(self.node_x, self.adj_norm)
+        return latent_z.cpu().numpy()

{gsmap-1.66 → gsmap-1.67}/src/gsMap/__init__.py

@@ -2,4 +2,4 @@
 Genetics-informed pathogenic spatial mapping
 '''
 
-__version__ = '1.66'
+__version__ = '1.67'

{gsmap-1.66 → gsmap-1.67}/src/gsMap/config.py

@@ -55,7 +55,8 @@ def add_find_latent_representations_args(parser):
     add_shared_args(parser)
     parser.add_argument('--input_hdf5_path', required=True, type=str, help='Path to the input HDF5 file.')
     parser.add_argument('--annotation', required=True, type=str, help='Name of the annotation in adata.obs to use.')
-    parser.add_argument('--data_layer', required=True, type=str, help='Data layer for gene expression (e.g., "counts", "log1p").')
+    parser.add_argument('--data_layer', type=str, default='counts', required=True,
+                        help='Data layer for gene expression (e.g., "count", "counts", "log1p").')
     parser.add_argument('--epochs', type=int, default=300, help='Number of training epochs.')
     parser.add_argument('--feat_hidden1', type=int, default=256, help='Neurons in the first hidden layer.')
     parser.add_argument('--feat_hidden2', type=int, default=128, help='Neurons in the second hidden layer.')
@@ -66,7 +67,6 @@ def add_find_latent_representations_args(parser):
     parser.add_argument('--n_neighbors', type=int, default=11, help='Number of neighbors for GAT.')
     parser.add_argument('--n_comps', type=int, default=300, help='Number of principal components for PCA.')
     parser.add_argument('--weighted_adj', action='store_true', help='Use weighted adjacency in GAT.')
-    parser.add_argument('--var', action='store_true', help='Enable variance calculations.')
     parser.add_argument('--convergence_threshold', type=float, default=1e-4, help='Threshold for convergence.')
     parser.add_argument('--hierarchically', action='store_true', help='Enable hierarchical latent representation finding.')
 
@@ -236,8 +236,8 @@ def add_run_all_mode_args(parser):
                         help='Path to the input spatial transcriptomics data (H5AD format).')
     parser.add_argument('--annotation', type=str, required=True,
                         help='Name of the annotation in adata.obs to use.')
-    parser.add_argument('--data_layer', type=str, default='X',
-                        help='Data layer of h5ad for gene expression (e.g., "counts", "log1p", "X").')
+    parser.add_argument('--data_layer', type=str, default='counts', required=True,
+                        help='Data layer for gene expression (e.g., "count", "counts", "log1p").')
 
     # GWAS Data Parameters
     parser.add_argument('--trait_name', type=str, help='Name of the trait for GWAS analysis (required if sumstats_file is provided).')

gsmap-1.67/src/gsMap/find_latent_representation.py

@@ -0,0 +1,145 @@
+import logging
+import random
+import numpy as np
+import scanpy as sc
+import torch
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import LabelEncoder
+from gsMap.GNN_VAE.adjacency_matrix import construct_adjacency_matrix
+from gsMap.GNN_VAE.train import ModelTrainer
+from gsMap.config import FindLatentRepresentationsConfig
+
+logger = logging.getLogger(__name__)
+
+
+def set_seed(seed_value):
+    """
+    Set seed for reproducibility in PyTorch and other libraries.
+    """
+    torch.manual_seed(seed_value)
+    np.random.seed(seed_value)
+    random.seed(seed_value)
+    if torch.cuda.is_available():
+        logger.info('Using GPU for computations.')
+        torch.cuda.manual_seed(seed_value)
+        torch.cuda.manual_seed_all(seed_value)
+    else:
+        logger.info('Using CPU for computations.')
+
+def preprocess_data(adata, params):
+    """
+    Preprocess the AnnData
+    """
+    logger.info('Preprocessing data...')
+    adata.var_names_make_unique()
+
+    sc.pp.filter_genes(adata, min_cells=30)
+    if params.data_layer in adata.layers.keys():
+        adata.X = adata.layers[params.data_layer]
+    else:
+        raise ValueError(f'Invalid data layer: {params.data_layer}, please check the input data.')
+
+    if params.data_layer in ['count', 'counts']:
+
+        sc.pp.normalize_total(adata, target_sum=1e4)
+        sc.pp.log1p(adata)
+
+        # Identify highly variable genes
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat_v3",
+            n_top_genes=params.feat_cell,
+        )
+
+    elif params.data_layer in adata.layers.keys():
+        logger.info(f'Using {params.data_layer} data...')
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat",
+            n_top_genes=params.feat_cell,
+        )
+
+    return adata
+
+
+class LatentRepresentationFinder:
+    def __init__(self, adata, args: FindLatentRepresentationsConfig):
+        self.params = args
+
+        self.expression_array = adata[:, adata.var.highly_variable].X.copy()
+
+        if self.params.data_layer in ['count', 'counts']:
+            self.expression_array = sc.pp.scale(self.expression_array, max_value=10)
+
+        # Construct the neighboring graph
+        self.graph_dict = construct_adjacency_matrix(adata, self.params)
+
+    def compute_pca(self):
+        self.latent_pca = PCA(n_components=self.params.feat_cell).fit_transform(self.expression_array)
+        return self.latent_pca
+
+    def run_gnn_vae(self, label, verbose='whole ST data'):
+
+        # Use PCA if specified
+        if self.params.input_pca:
+            node_X = self.compute_pca()
+        else:
+            node_X = self.expression_array
+
+        # Update the input shape
+        self.params.n_nodes = node_X.shape[0]
+        self.params.feat_cell = node_X.shape[1]
+
+        # Run GNN
+        logger.info(f'Finding latent representations for {verbose}...')
+        gvae = ModelTrainer(node_X, self.graph_dict, self.params, label)
+        gvae.run_train()
+
+        del self.graph_dict
+
+        return gvae.get_latent()
+
+
+def run_find_latent_representation(args: FindLatentRepresentationsConfig):
+    set_seed(2024)
+
+    # Load the ST data
+    logger.info(f'Loading ST data of {args.sample_name}...')
+    adata = sc.read_h5ad(args.input_hdf5_path)
+    logger.info(f'The ST data contains {adata.shape[0]} cells, {adata.shape[1]} genes.')
+
+    # Load the cell type annotation
+    if args.annotation is not None:
+        # Remove cells without enough annotations
+        adata = adata[~adata.obs[args.annotation].isnull()]
+        num = adata.obs[args.annotation].value_counts()
+        valid_annotations = num[num >= 30].index.to_list()
+        adata = adata[adata.obs[args.annotation].isin(valid_annotations)]
+
+        le = LabelEncoder()
+        adata.obs['categorical_label'] = le.fit_transform(adata.obs[args.annotation])
+        label = adata.obs['categorical_label'].to_numpy()
+    else:
+        label = None
+
+    # Preprocess data
+    adata = preprocess_data(adata, args)
+
+    latent_rep = LatentRepresentationFinder(adata, args)
+    latent_gvae = latent_rep.run_gnn_vae(label)
+    latent_pca = latent_rep.compute_pca()
+
+    # Add latent representations to the AnnData object
+    logger.info('Adding latent representations...')
+    adata.obsm["latent_GVAE"] = latent_gvae
+    adata.obsm["latent_PCA"] = latent_pca
+
+    # Run UMAP based on latent representations
+    for name in ['latent_GVAE', 'latent_PCA']:
+        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
+        sc.tl.umap(adata)
+        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
+
+    # Save the AnnData object
+    logger.info('Saving ST data...')
+    adata.write(args.hdf5_with_latent_path)

{gsmap-1.66 → gsmap-1.67}/src/gsMap/format_sumstats.py

@@ -150,10 +150,10 @@ def gwas_checkname(gwas, config):
         'Pos': 'SNP positions.'
     }
 
-    print(f'\nIterpreting column names as follows:')
+    logger.info(f'\nIterpreting column names as follows:')
     for key, value in interpreting.items():
         if key in new_name:
-            print(f'{name_dict[key]}: {interpreting[key]}')
+            logger.info(f'{name_dict[key]}: {interpreting[key]}')
 
     return gwas
 
@@ -242,7 +242,7 @@ def gwas_qc(gwas, config):
     Filter out SNPs based on INFO, FRQ, MAF, N, and Genotypes. 
     '''
     old = len(gwas)
-    print(f'\nFiltering SNPs as follows:')
+    logger.info(f'\nFiltering SNPs as follows:')
     # filter: SNPs with missing values
     drops = {'NA': 0, 'P': 0, 'INFO': 0, 'FRQ': 0, 'A': 0, 'SNP': 0, 'Dup': 0, 'N': 0}
 
@@ -250,28 +250,28 @@ def gwas_qc(gwas, config):
         lambda x: x != 'INFO', gwas.columns)).reset_index(drop=True)
 
     drops['NA'] = old - len(gwas)
-    print(f'Removed {drops["NA"]} SNPs with missing values.')
+    logger.info(f'Removed {drops["NA"]} SNPs with missing values.')
 
     # filter: SNPs with Info < 0.9
     if 'INFO' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_info(gwas['INFO'], config)]
         drops['INFO'] = old - len(gwas)
-        print(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
+        logger.info(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
 
     # filter: SNPs with MAF <= 0.01
     if 'FRQ' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_frq(gwas['FRQ'], config)]
         drops['FRQ'] += old - len(gwas)
-        print(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
+        logger.info(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
 
     # filter: P-value that out-of-bounds [0,1]
     if 'P' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_pvals(gwas['P'], config)]
         drops['P'] += old - len(gwas)
-        print(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
+        logger.info(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
 
     # filter: Variants that are strand-ambiguous
     if 'A1' in gwas.columns and 'A2' in gwas.columns:
@@ -279,21 +279,21 @@ def gwas_qc(gwas, config):
         gwas.A2 = gwas.A2.str.upper()
         gwas = gwas.loc[filter_alleles(gwas.A1 + gwas.A2)]
         drops['A'] += old - len(gwas)
-        print(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
+        logger.info(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
 
     # filter: Duplicated rs numbers
     if 'SNP' in gwas.columns:
         old = len(gwas)
         gwas = gwas.drop_duplicates(subset='SNP').reset_index(drop=True)
         drops['Dup'] += old - len(gwas)
-        print(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
+        logger.info(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
 
     # filter:Sample size
     n_min = gwas.N.quantile(0.9) / 1.5
     old = len(gwas)
     gwas = gwas[gwas.N >= n_min].reset_index(drop=True)
     drops['N'] += old - len(gwas)
-    print(f'Removed {drops["N"]} SNPs with N < {n_min}.')
+    logger.info(f'Removed {drops["N"]} SNPs with N < {n_min}.')
 
     return gwas
 
@@ -302,7 +302,7 @@ def variant_to_rsid(gwas, config):
     '''
     Convert variant id (Chr, Pos) to rsid
     '''
-    print("\nConverting the SNP position to rsid. This process may take some time.")
+    logger.info("\nConverting the SNP position to rsid. This process may take some time.")
     unique_ids = set(gwas['id'])
     chr_format = gwas['Chr'].unique().astype(str)
     chr_format = [re.sub(r'\d+', '', value) for value in chr_format][1]
@@ -347,7 +347,7 @@ def clean_SNP_id(gwas, config):
             gwas = gwas.loc[matching_id.id]
             gwas['SNP'] = matching_id.dbsnp
             num_fail = old - len(gwas)
-            print(f'Removed {num_fail} SNPs that did not convert to rsid.')
+            logger.info(f'Removed {num_fail} SNPs that did not convert to rsid.')
 
     return gwas
 
@@ -356,27 +356,27 @@ def gwas_metadata(gwas, config):
     '''
     Report key features of GWAS data
     '''
-    print('\nMetadata:')
+    logger.info('\nSummary of GWAS data:')
     CHISQ = (gwas.Z ** 2)
     mean_chisq = CHISQ.mean()
-    print('Mean chi^2 = ' + str(round(mean_chisq, 3)))
+    logger.info('Mean chi^2 = ' + str(round(mean_chisq, 3)))
     if mean_chisq < 1.02:
         logger.warning("Mean chi^2 may be too small.")
 
-    print('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
-    print('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
-    print('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
+    logger.info('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
+    logger.info('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
+    logger.info('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
 
 
 def gwas_format(config: FormatSumstatsConfig):
     '''
     Format GWAS data
     '''
-    print(f'------Formating gwas data for {config.sumstats}...')
+    logger.info(f'------Formating gwas data for {config.sumstats}...')
     compression_type = get_compression(config.sumstats)
     gwas = pd.read_csv(config.sumstats, delim_whitespace=True, header=0, compression=compression_type,
                        na_values=['.', 'NA'])
-    print(f'Read {len(gwas)} SNPs from {config.sumstats}.')
+    logger.info(f'Read {len(gwas)} SNPs from {config.sumstats}.')
 
     # Check name and format
     gwas = gwas_checkname(gwas, config)
@@ -402,6 +402,6 @@ def gwas_format(config: FormatSumstatsConfig):
     gwas = gwas[keep]
     out_name = config.out + appendix + '.gz'
 
-    print(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
+    logger.info(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
     gwas.to_csv(out_name, sep="\t", index=False,
                 float_format='%.3f', compression='gzip')

{gsmap-1.66 → gsmap-1.67}/src/gsMap/latent_to_gene.py

@@ -4,12 +4,10 @@ from pathlib import Path
 import numpy as np
 import pandas as pd
 import scanpy as sc
-from scipy.sparse import csr_matrix
 from scipy.stats import gmean
 from scipy.stats import rankdata
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.neighbors import NearestNeighbors
-from joblib import Parallel, delayed
 from tqdm import tqdm
 
 from gsMap.config import LatentToGeneConfig
@@ -152,11 +150,6 @@ def run_latent_to_gene(config: LatentToGeneConfig):
         adata.var_names = homologs.loc[adata.var_names, 'HUMAN_GENE_SYM'].values
         adata = adata[:, ~adata.var_names.duplicated()]
 
-    # Remove cells and genes that are not expressed
-    logger.info(f'Number of cells, genes of the input data: {adata.shape[0]},{adata.shape[1]}')
-    adata = adata[adata.X.sum(axis=1) > 0, adata.X.sum(axis=0) > 0]
-    logger.info(f'Number of cells, genes after transformation: {adata.shape[0]},{adata.shape[1]}')
-
     # Create mappings
     n_cells = adata.n_obs
     n_genes = adata.n_vars

{gsmap-1.66 → gsmap-1.67}/src/gsMap/spatial_ldsc_multiple_sumstats.py

@@ -20,8 +20,6 @@ logger = logging.getLogger('gsMap.spatial_ldsc')
 
 # %%
 def _coef_new(jknife):
-    # return coef[0], coef_se[0], z[0]]
-    # est_ = jknife.est[0, 0] / Nbar
     est_ = jknife.jknife_est[0, 0] / Nbar
     se_ = jknife.jknife_se[0, 0] / Nbar
     return est_, se_

gsmap-1.66/src/gsMap/GNN_VAE/adjacency_matrix.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Jul  4 21:31:27 2023
-
-@author: songliyang
-"""
-import numpy as np
-import pandas as pd
-import scipy.sparse as sp
-import sklearn.neighbors
-import torch
-
-
-def Cal_Spatial_Net(adata, n_neighbors=5, verbose=True):
-    """\
-    Construct the spatial neighbor networks.
-    """
-    #- 
-    if verbose:
-        print('------Calculating spatial graph...')
-    coor = pd.DataFrame(adata.obsm['spatial'])
-    coor.index = adata.obs.index
-    #- 
-    nbrs = sklearn.neighbors.NearestNeighbors(n_neighbors=n_neighbors).fit(coor)
-    #- 
-    distances, indices = nbrs.kneighbors(coor, return_distance=True)
-    KNN_list = []
-    for it in range(indices.shape[0]):
-        KNN_list.append(pd.DataFrame(zip([it]*indices[it].shape[0], indices[it], distances[it])))
-    #- 
-    KNN_df = pd.concat(KNN_list)
-    KNN_df.columns = ['Cell1', 'Cell2', 'Distance']
-    #- 
-    Spatial_Net = KNN_df.copy()
-    Spatial_Net = Spatial_Net.loc[Spatial_Net['Distance']>0,]
-    id_cell_trans = dict(zip(range(coor.shape[0]), np.array(coor.index), ))
-    Spatial_Net['Cell1'] = Spatial_Net['Cell1'].map(id_cell_trans)
-    Spatial_Net['Cell2'] = Spatial_Net['Cell2'].map(id_cell_trans)
-    #- 
-    return Spatial_Net
-
-
-def sparse_mx_to_torch_sparse_tensor(sparse_mx):
-    """Convert a scipy sparse matrix to a torch sparse tensor."""
-    sparse_mx = sparse_mx.tocoo().astype(np.float32)
-    indices = torch.from_numpy(np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
-    values = torch.from_numpy(sparse_mx.data)
-    shape = torch.Size(sparse_mx.shape)
-    return torch.sparse.FloatTensor(indices, values, shape)
-
-
-def preprocess_graph(adj):
-    adj = sp.coo_matrix(adj)
-    adj_ = adj + sp.eye(adj.shape[0])
-    rowsum = np.array(adj_.sum(1))
-    degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5).flatten())
-    adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo()
-    return sparse_mx_to_torch_sparse_tensor(adj_normalized)
-
-
-
-def Construct_Adjacency_Matrix(adata,Params, verbose=True):
-    # Construct the neighbor graph 
-    Spatial_Net = Cal_Spatial_Net(adata, n_neighbors=Params.n_neighbors)
-    #- 
-    if verbose:
-        print('The graph contains %d edges, %d cells.' %(Spatial_Net.shape[0], adata.n_obs))
-        print('%.2f neighbors per cell on average.' %(Spatial_Net.shape[0]/adata.n_obs))
-    #-  
-    cells = np.array(adata.obs.index)
-    cells_id_tran = dict(zip(cells, range(cells.shape[0])))
-    #- 
-    G_df = Spatial_Net.copy()
-    G_df['Cell1'] = G_df['Cell1'].map(cells_id_tran)
-    G_df['Cell2'] = G_df['Cell2'].map(cells_id_tran)
-    #- 
-    if Params.weighted_adj:
-        distance_normalized = G_df.Distance/(max(G_df.Distance)+1)
-        adj_org = sp.coo_matrix((np.exp(-distance_normalized**2/(2)), (G_df['Cell1'], G_df['Cell2'])), shape=(adata.n_obs, adata.n_obs))
-    else:
-        adj_org = sp.coo_matrix((np.ones(G_df.shape[0]), (G_df['Cell1'], G_df['Cell2'])), shape=(adata.n_obs, adata.n_obs))
-    #- 
-    adj_m1 = adj_org
-    adj_norm_m1 = preprocess_graph(adj_m1)
-    adj_label_m1 = adj_m1 + sp.eye(adj_m1.shape[0])
-    norm_m1 = adj_m1.shape[0] * adj_m1.shape[0] / float((adj_m1.shape[0] * adj_m1.shape[0] - adj_m1.sum()) * 2)
-    #- 
-    graph_dict = {
-        "adj_org": adj_org,
-        "adj_norm": adj_norm_m1,
-        "norm_value": norm_m1
-    }
-    #- 
-    return graph_dict

gsmap-1.66/src/gsMap/GNN_VAE/model.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Jul  3 11:42:44 2023
-
-@author: songliyang
-"""
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch_geometric.nn import GATConv
-
-
-def full_block(in_features, out_features, p_drop):
-    return nn.Sequential(nn.Linear(in_features, out_features),
-                         nn.BatchNorm1d(out_features),
-                         nn.ELU(),
-                         nn.Dropout(p=p_drop))
-
-
-class GNN(nn.Module):
-    def __init__(self, in_features, out_features, dr=0, act=F.relu,heads=1):
-        super().__init__()
-        self.conv1 = GATConv(in_features, out_features,heads)
-        self.act = act
-        self.dr = dr
-    #-
-    def forward(self, x, edge_index):
-        out = self.conv1(x, edge_index)
-        out = self.act(out)
-        out = F.dropout(out, self.dr, self.training)
-        return out
-
-
-    
-class GNN_VAE_Model(nn.Module):
-    def __init__(self, input_dim,params,num_classes=1):
-        super(GNN_VAE_Model, self).__init__()
-        self.var = params.var
-        self.num_classes = num_classes
-            
-        # Encoder
-        self.encoder = nn.Sequential()
-        self.encoder.add_module('encoder_L1', full_block(input_dim, params.feat_hidden1, params.p_drop))
-        self.encoder.add_module('encoder_L2', full_block(params.feat_hidden1, params.feat_hidden2, params.p_drop))
-        
-        # GNN (GAT)
-        self.gn1 = GNN(params.feat_hidden2, params.gat_hidden1, params.p_drop, act=F.relu,heads = params.nheads)
-        self.gn2 = GNN(params.gat_hidden1*params.nheads, params.gat_hidden2, params.p_drop, act=lambda x: x)
-        self.gn3 = GNN(params.gat_hidden1*params.nheads, params.gat_hidden2, params.p_drop, act=lambda x: x) 
-        
-        # Decoder
-        self.decoder = nn.Sequential()
-        self.decoder.add_module('decoder_L1', full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop))
-        self.decoder.add_module('decoder_L2', full_block(params.feat_hidden2, params.feat_hidden1, params.p_drop))
-        self.decoder.add_module('decoder_output', nn.Sequential(nn.Linear(params.feat_hidden1, input_dim)))
-        
-        # Cluster
-        self.cluster = nn.Sequential()
-        self.cluster.add_module('cluster_L1', full_block(params.gat_hidden2, params.feat_hidden2, params.p_drop))
-        self.cluster.add_module('cluster_output', nn.Linear(params.feat_hidden2, self.num_classes))
-           
-    def encode(self, x, adj):
-        feat_x = self.encoder(x)
-        hidden1 = self.gn1(feat_x, adj)
-        mu = self.gn2(hidden1, adj)
-        if self.var:
-            logvar = self.gn3(hidden1, adj)
-            return mu, logvar
-        else:
-            return mu, None
-     
-    def reparameterize(self, mu, logvar):
-        if self.training and logvar is not None:
-            std = torch.exp(logvar)
-            eps = torch.randn_like(std)
-            return eps.mul(std).add_(mu)
-        else:
-            return mu
-    
-    def forward(self, x, adj):
-        mu, logvar = self.encode(x, adj)
-        gnn_z = self.reparameterize(mu, logvar)
-        x_reconstructed = self.decoder(gnn_z)
-        pred_label = F.softmax(self.cluster(gnn_z),dim=1)
-        return pred_label, x_reconstructed, gnn_z, mu, logvar

gsmap-1.66/src/gsMap/GNN_VAE/train.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Jul  4 19:58:58 2023
-
-@author: songliyang
-"""
-import time
-
-import torch
-from progress.bar import Bar
-
-from gsMap.GNN_VAE.model import GNN_VAE_Model
-
-
-def reconstruction_loss(decoded, x):
-    loss_fn = torch.nn.MSELoss()
-    loss = loss_fn(decoded, x)
-    return loss
-
-
-def label_loss(pred_label, true_label):
-    loss_fn = torch.nn.CrossEntropyLoss()
-    loss = loss_fn(pred_label, true_label)
-    return loss
-
-
-class Model_Train:
-    def __init__(self, node_X, graph_dict, params, label=None):
-        device = 'cuda' if torch.cuda.is_available() else 'cpu'
-        torch.cuda.empty_cache()
-
-        self.params = params
-        self.device = device
-        self.epochs = params.epochs
-        self.node_X = torch.FloatTensor(node_X.copy()).to(device)
-        self.adj_norm = graph_dict["adj_norm"].to(device).coalesce()
-        self.label = label
-        self.num_classes = 1
-        
-        if not self.label is None:
-            self.label = torch.tensor(self.label).to(self.device)
-            self.num_classes = len(self.label.unique())
-        
-        # Set Model 
-        self.model = GNN_VAE_Model(self.params.feat_cell,self.params,self.num_classes).to(device)
-        self.optimizer = torch.optim.Adam(params = list(self.model.parameters()),
-                                          lr = self.params.gat_lr, weight_decay = self.params.gcn_decay)               
-    
-    # Train    
-    def run_train(self):
-        self.model.train()
-        prev_loss = float('inf')
-        
-        bar = Bar('GAT-AE model train:', max = self.epochs)
-        bar.check_tty = False 
-        for epoch in range(self.epochs):
-            start_time = time.time()
-            self.model.train()
-            self.optimizer.zero_grad()
-            pred_label, de_feat, latent_z, mu, logvar = self.model(self.node_X, self.adj_norm)
-            loss_rec = reconstruction_loss(de_feat, self.node_X)
-            
-            # Check whether annotation was provided
-            if not self.label is None:
-                loss_pre = label_loss(pred_label, self.label)
-                loss = (self.params.rec_w * loss_rec) + (self.params.label_w * loss_pre)
-            else:
-                loss = loss_rec
-                
-            loss.backward()
-            self.optimizer.step()
-            
-            # Update process
-            end_time = time.time()
-            batch_time = end_time - start_time
-            
-            
-            bar_str = '{} / {} | Left time: {batch_time:.2f} mins| Loss: {loss:.4f}'
-            bar.suffix = bar_str.format(epoch + 1,self.epochs,
-                                        batch_time = batch_time * (self.epochs - epoch) / 60, loss=loss.item())
-            bar.next()
-            
-            # Check convergence
-            if abs(loss.item() - prev_loss) <= self.params.convergence_threshold and epoch >= 200:
-                print('\nConvergence reached. Training stopped.')
-                break
-
-            prev_loss = loss.item()
-            
-        bar.finish()
-    #-    
-    def get_latent(self):
-        self.model.eval()
-        pred, de_fea, latent_z, mu, logvar = self.model(self.node_X, self.adj_norm)
-        latent_z = latent_z.data.cpu().numpy()
-        return latent_z

gsmap-1.66/src/gsMap/find_latent_representation.py

@@ -1,145 +0,0 @@
-import logging
-import random
-
-import numpy as np
-import pandas as pd
-import scanpy as sc
-import torch
-from sklearn import preprocessing
-
-from gsMap.GNN_VAE.adjacency_matrix import Construct_Adjacency_Matrix
-from gsMap.GNN_VAE.train import Model_Train
-from gsMap.config import FindLatentRepresentationsConfig
-
-logger = logging.getLogger(__name__)
-
-def set_seed(seed_value):
-    """
-    Set seed for reproducibility in PyTorch.
-    """
-    torch.manual_seed(seed_value)  # Set the seed for PyTorch
-    np.random.seed(seed_value)  # Set the seed for NumPy
-    random.seed(seed_value)  # Set the seed for Python random module
-    if torch.cuda.is_available():
-        logger.info('Running use GPU')
-        torch.cuda.manual_seed(seed_value)  # Set seed for all CUDA devices
-        torch.cuda.manual_seed_all(seed_value)  # Set seed for all CUDA devices
-    else:
-        logger.info('Running use CPU')
-
-
-# The class for finding latent representations
-class Latent_Representation_Finder:
-
-    def __init__(self, adata, args:FindLatentRepresentationsConfig):
-        self.adata = adata.copy()
-        self.Params = args
-
-        # Standard process
-        if self.Params.data_layer == 'count' or self.Params.data_layer == 'counts':
-            self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, flavor="seurat_v3", n_top_genes=self.Params.feat_cell)
-            sc.pp.normalize_total(self.adata, target_sum=1e4)
-            sc.pp.log1p(self.adata)
-            sc.pp.scale(self.adata)
-        else:
-            if self.Params.data_layer != 'X':
-                self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, n_top_genes=self.Params.feat_cell)
-
-    def Run_GNN_VAE(self, label, verbose='whole ST data'):
-
-        # Construct the neighbouring graph
-        graph_dict = Construct_Adjacency_Matrix(self.adata, self.Params)
-
-        # Process the feature matrix
-        node_X = self.adata[:, self.adata.var.highly_variable].X
-        logger.info(f'The shape of feature matrix is {node_X.shape}.')
-        if self.Params.input_pca:
-            node_X = sc.pp.pca(node_X, n_comps=self.Params.n_comps)
-
-        # Update the input shape
-        self.Params.n_nodes = node_X.shape[0]
-        self.Params.feat_cell = node_X.shape[1]
-
-        # Run GNN-VAE
-        logger.info(f'------Finding latent representations for {verbose}...')
-        gvae = Model_Train(node_X, graph_dict, self.Params, label)
-        gvae.run_train()
-
-        return gvae.get_latent()
-
-    def Run_PCA(self):
-        sc.tl.pca(self.adata)
-        return self.adata.obsm['X_pca'][:, 0:self.Params.n_comps]
-
-
-def run_find_latent_representation(args:FindLatentRepresentationsConfig):
-    set_seed(2024)
-    num_features = args.feat_cell
-    args.hdf5_with_latent_path.parent.mkdir(parents=True, exist_ok=True,mode=0o755)
-    # Load the ST data
-    logger.info(f'------Loading ST data of {args.sample_name}...')
-    adata = sc.read_h5ad(f'{args.input_hdf5_path}')
-    adata.var_names_make_unique()
-    adata.X = adata.layers[args.data_layer] if args.data_layer in adata.layers.keys() else adata.X
-    logger.info('The ST data contains %d cells, %d genes.' % (adata.shape[0], adata.shape[1]))
-    # Load the cell type annotation
-    if not args.annotation is None:
-        # remove cells without enough annotations
-        adata = adata[~pd.isnull(adata.obs[args.annotation]), :]
-        num = adata.obs[args.annotation].value_counts()
-        adata = adata[adata.obs[args.annotation].isin(num[num >= 30].index.to_list())]
-
-        le = preprocessing.LabelEncoder()
-        le.fit(adata.obs[args.annotation])
-        adata.obs['categorical_label'] = le.transform(adata.obs[args.annotation])
-        label = adata.obs['categorical_label'].to_list()
-    else:
-        label = None
-    # Find latent representations
-    latent_rep = Latent_Representation_Finder(adata, args)
-    latent_GVAE = latent_rep.Run_GNN_VAE(label)
-    latent_PCA = latent_rep.Run_PCA()
-    # Add latent representations to the spe data
-    logger.info(f'------Adding latent representations...')
-    adata.obsm["latent_GVAE"] = latent_GVAE
-    adata.obsm["latent_PCA"] = latent_PCA
-    # Run umap based on latent representations
-    for name in ['latent_GVAE', 'latent_PCA']:
-        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
-        sc.tl.umap(adata)
-        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
-
-        # Find the latent representations hierarchically (optionally)
-    if not args.annotation is None and args.hierarchically:
-        logger.info(f'------Finding latent representations hierarchically...')
-        PCA_all = pd.DataFrame()
-        GVAE_all = pd.DataFrame()
-
-        for ct in adata.obs[args.annotation].unique():
-            adata_part = adata[adata.obs[args.annotation] == ct, :]
-            logger.info(adata_part.shape)
-
-            # Find latent representations for the selected ct
-            latent_rep = Latent_Representation_Finder(adata_part, args)
-
-            latent_PCA_part = pd.DataFrame(latent_rep.Run_PCA())
-            if adata_part.shape[0] <= args.n_comps:
-                latent_GVAE_part = latent_PCA_part
-            else:
-                latent_GVAE_part = pd.DataFrame(latent_rep.Run_GNN_VAE(label=None, verbose=ct))
-
-            latent_GVAE_part.index = adata_part.obs_names
-            latent_PCA_part.index = adata_part.obs_names
-
-            GVAE_all = pd.concat((GVAE_all, latent_GVAE_part), axis=0)
-            PCA_all = pd.concat((PCA_all, latent_PCA_part), axis=0)
-
-            args.feat_cell = num_features
-
-            adata.obsm["latent_GVAE_hierarchy"] = np.array(GVAE_all.loc[adata.obs_names,])
-            adata.obsm["latent_PCA_hierarchy"] = np.array(PCA_all.loc[adata.obs_names,])
-    logger.info(f'------Saving ST data...')
-    adata.write(args.hdf5_with_latent_path)
-